diff --git a/README.md b/README.md index 4b869a9..3b52d50 100644 --- a/README.md +++ b/README.md @@ -39,6 +39,7 @@ local setup cost: | Non-persistent topics | Dispatch-or-drop runtime semantics with coverage for flow control, disconnect/reconnect, ordering, dynamic consumers, and KeyShared routing. | | Persistent topics | RocksDB-backed managed-ledger style storage is available behind the `rocksdb-storage` feature. | | Subscription modes | Shared, Failover, Exclusive, and KeyShared are covered by Rust and Python integration tests. | +| Monitoring | Prometheus metrics on `GET /metrics` (port 8080): broker/topic/subscription counters, backlog and unacked gauges, storage latency and entry-size histograms, process metrics. Optional Grafana stack under `grafana/`. | | Partitioned topics | Default partition metadata and partition topic routing are supported for local testing. | | Python package | Provides a small helper SDK that can start and manage a local broker process. | @@ -110,6 +111,39 @@ Stop the broker: ../rust/pulsar-lite.sh stop ``` +## Monitoring + +The broker exposes Prometheus metrics on `GET /metrics` (default +`0.0.0.0:8080`, same port model as native Pulsar's web service). Metric +families reuse native Pulsar names and labels (`pulsar_rate_in`, +`pulsar_subscription_back_log`, `pulsar_storage_write_latency`, ...), so +existing dashboards and PromQL translate directly; extensions unique to +Pulsar Lite use the `pulsar_lite_*` prefix. + +Quick check: + +```bash +curl -s localhost:8080/metrics | grep -E '^pulsar_(broker|subscription)' +``` + +Configuration (`rust/pulsar-lite.toml`): + +```toml +[metrics] +enabled = true # false: no listener, no scrape aggregation +addr = "0.0.0.0:8080" +cluster = "pulsar-lite" # `cluster` label value on every family +rate_window_secs = 60 # window for pulsar_rate_in-style gauges +``` + +A ready-to-run Prometheus + Grafana stack (with provisioned dashboards for +topics and broker overview) lives under [`grafana/`](grafana/README.md): + +```bash +docker compose -f grafana/docker-compose.yml up -d +# Grafana http://localhost:3000 (admin/admin), Prometheus http://localhost:9090 +``` + ## Embedded Python Usage The Python helper can start a local broker for short-lived tests or examples: diff --git a/grafana/README.md b/grafana/README.md new file mode 100644 index 0000000..61380d6 --- /dev/null +++ b/grafana/README.md @@ -0,0 +1,68 @@ +# Pulsar Lite Observability Stack + +A one-command Prometheus + Grafana stack that scrapes the broker's +`GET /metrics` endpoint (default `0.0.0.0:8080`). + +## Quick start + +```bash +# 1. Start the broker (metrics are on by default) +../rust/target/release/pulsar-lite --config ../rust/pulsar-lite.toml + +# 2. Start the observability stack +docker compose up -d + +# 3. Open +# Grafana http://localhost:3000 (admin/admin; anonymous read enabled) +# Prometheus http://localhost:9090/targets — the pulsar-lite job must be UP +``` + +Dashboards are provisioned automatically (folder `Pulsar Lite`): + +- **Pulsar Lite / Topics** — publish/deliver rates and throughput, entity + counts, subscription backlog, unacked gate state, redelivery, storage + size, end-to-end and ledger write latency (P50/P99), entry-size + distribution. +- **Pulsar Lite / Broker** — broker-level rates (counter `rate()` and + window-gauge views), connections and rejection reasons, backlog and + storage totals, write-queue batch sizes, process RSS/CPU. + +## Live view during perf tests + +The perf harness (`tests/perf/`) starts real brokers, each with a private +metrics port derived from its protocol port (6651/6652 → 8081/8082, +6661/6662 → 8091/8092, 6671/6672 → 8101/8102, bound on `0.0.0.0`). This +stack scrapes all of them (`job="pulsar-lite-perf"`), so with the stack up +you can watch a run live: + +```bash +docker compose up -d # once +python tests/perf/run_persistent_stress.py ... # then run any scenario +# Grafana http://localhost:3000 — dashboards show the running broker's series +``` + +Perf targets show DOWN in Prometheus `/targets` while no run is active — +that is expected. Docker-backed perf runs use `--network host`, so the same +ports apply. + +## Metric naming conventions + +- `pulsar_*` families reproduce native Apache Pulsar names and label sets + verbatim, so official dashboards and PromQL translate directly. +- `pulsar_lite_*` families are extensions with no native counterpart + (error reasons, redelivery counters, write-queue batch metrics). +- Histograms use the standard Prometheus shape (`_bucket{le=...}` + + `_sum` + `_count`); query percentiles with `histogram_quantile()`. + +## Broker configuration + +```toml +[metrics] +enabled = true # false: no listener, no scrape aggregation +addr = "0.0.0.0:8080" # /metrics path; must be reachable by the scraper +cluster = "pulsar-lite" # cluster label value on every family +rate_window_secs = 60 # window for pulsar_rate_in-style gauges +``` + +Remote broker: change the target in `prometheus/prometheus.yml` from +`host.docker.internal:8080` to your broker address. diff --git a/grafana/dashboards/pulsar-lite-broker.json b/grafana/dashboards/pulsar-lite-broker.json new file mode 100644 index 0000000..79dc8d8 --- /dev/null +++ b/grafana/dashboards/pulsar-lite-broker.json @@ -0,0 +1,223 @@ +{ + "uid": "pulsar-lite-broker", + "title": "Pulsar Lite / Broker", + "tags": ["pulsar-lite"], + "timezone": "browser", + "schemaVersion": 39, + "version": 1, + "editable": true, + "refresh": "10s", + "time": { "from": "now-30m", "to": "now" }, + "templating": { + "list": [ + { + "name": "cluster", + "label": "Cluster", + "type": "query", + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "query": "label_values(pulsar_broker_rate_in, cluster)", + "refresh": 2, + "includeAll": false, + "multi": false, + "current": { "text": "pulsar-lite", "value": "pulsar-lite" } + } + ] + }, + "panels": [ + { + "id": 1, + "type": "timeseries", + "title": "Broker message rate (in / out)", + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 0 }, + "targets": [ + { + "expr": "pulsar_broker_rate_in{cluster=\"$cluster\"}", + "legendFormat": "in" + }, + { + "expr": "pulsar_broker_rate_out{cluster=\"$cluster\"}", + "legendFormat": "out" + } + ] + }, + { + "id": 2, + "type": "timeseries", + "title": "Broker throughput (bytes/s)", + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 0 }, + "targets": [ + { + "expr": "pulsar_broker_throughput_in{cluster=\"$cluster\"}", + "legendFormat": "in" + }, + { + "expr": "pulsar_broker_throughput_out{cluster=\"$cluster\"}", + "legendFormat": "out" + } + ] + }, + { + "id": 3, + "type": "timeseries", + "title": "Accepted / delivered totals (rate over counters)", + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 8 }, + "targets": [ + { + "expr": "rate(pulsar_broker_in_messages_total{cluster=\"$cluster\"}[1m])", + "legendFormat": "published msg/s" + }, + { + "expr": "rate(pulsar_broker_out_messages_total{cluster=\"$cluster\"}[1m])", + "legendFormat": "delivered msg/s" + }, + { + "expr": "rate(pulsar_broker_in_bytes_total{cluster=\"$cluster\"}[1m])", + "legendFormat": "published B/s" + }, + { + "expr": "rate(pulsar_broker_out_bytes_total{cluster=\"$cluster\"}[1m])", + "legendFormat": "delivered B/s" + } + ] + }, + { + "id": 4, + "type": "timeseries", + "title": "Entity counts", + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "gridPos": { "h": 8, "w": 6, "x": 12, "y": 8 }, + "targets": [ + { + "expr": "pulsar_broker_topics_count{cluster=\"$cluster\"}", + "legendFormat": "topics" + }, + { + "expr": "pulsar_broker_subscriptions_count{cluster=\"$cluster\"}", + "legendFormat": "subscriptions" + }, + { + "expr": "pulsar_broker_producers_count{cluster=\"$cluster\"}", + "legendFormat": "producers" + }, + { + "expr": "pulsar_broker_consumers_count{cluster=\"$cluster\"}", + "legendFormat": "consumers" + } + ] + }, + { + "id": 5, + "type": "timeseries", + "title": "Connections", + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "gridPos": { "h": 8, "w": 6, "x": 18, "y": 8 }, + "targets": [ + { + "expr": "pulsar_active_connections{cluster=\"$cluster\"}", + "legendFormat": "active" + }, + { + "expr": "rate(pulsar_connection_created_total_count{cluster=\"$cluster\"}[5m])", + "legendFormat": "created /s" + }, + { + "expr": "rate(pulsar_connection_closed_total_count{cluster=\"$cluster\"}[5m])", + "legendFormat": "closed /s" + } + ] + }, + { + "id": 6, + "type": "timeseries", + "title": "Rejections by reason (/s)", + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 16 }, + "targets": [ + { + "expr": "sum by (reason) (rate(pulsar_lite_broker_errors_total{cluster=\"$cluster\"}[5m]))", + "legendFormat": "{{reason}}" + } + ] + }, + { + "id": 7, + "type": "timeseries", + "title": "Backlog + storage size", + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "gridPos": { "h": 8, "w": 6, "x": 12, "y": 16 }, + "targets": [ + { + "expr": "pulsar_broker_msg_backlog{cluster=\"$cluster\"}", + "legendFormat": "backlog entries" + }, + { + "expr": "pulsar_broker_storage_size{cluster=\"$cluster\"}", + "legendFormat": "stored bytes" + } + ] + }, + { + "id": 8, + "type": "timeseries", + "title": "Publish rate-limit rejections (/s)", + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "gridPos": { "h": 8, "w": 6, "x": 18, "y": 16 }, + "targets": [ + { + "expr": "sum by (topic) (rate(pulsar_publish_rate_limit_times{cluster=\"$cluster\"}[5m]))", + "legendFormat": "{{topic}}" + } + ] + }, + { + "id": 9, + "type": "timeseries", + "title": "Write-queue batch size (avg / P99)", + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "gridPos": { "h": 8, "w": 8, "x": 0, "y": 24 }, + "targets": [ + { + "expr": "rate(pulsar_lite_write_queue_batch_messages_total[1m]) / rate(pulsar_lite_write_queue_batches_total[1m])", + "legendFormat": "avg batch" + }, + { + "expr": "histogram_quantile(0.99, sum by (le) (rate(pulsar_lite_write_queue_batch_size_bucket[5m])))", + "legendFormat": "P99 batch" + } + ] + }, + { + "id": 10, + "type": "timeseries", + "title": "Broker RSS", + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "gridPos": { "h": 8, "w": 8, "x": 8, "y": 24 }, + "fieldConfig": { + "defaults": { "unit": "bytes" }, + "overrides": [] + }, + "targets": [ + { + "expr": "process_resident_memory_bytes", + "legendFormat": "RSS" + } + ] + }, + { + "id": 11, + "type": "timeseries", + "title": "Broker CPU (seconds/s)", + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "gridPos": { "h": 8, "w": 8, "x": 16, "y": 24 }, + "targets": [ + { + "expr": "rate(process_cpu_seconds_total[1m])", + "legendFormat": "cpu" + } + ] + } + ] +} diff --git a/grafana/dashboards/pulsar-lite-topics.json b/grafana/dashboards/pulsar-lite-topics.json new file mode 100644 index 0000000..49cfa85 --- /dev/null +++ b/grafana/dashboards/pulsar-lite-topics.json @@ -0,0 +1,253 @@ +{ + "uid": "pulsar-lite-topics", + "title": "Pulsar Lite / Topics", + "tags": ["pulsar-lite"], + "timezone": "browser", + "schemaVersion": 39, + "version": 1, + "editable": true, + "refresh": "10s", + "time": { "from": "now-30m", "to": "now" }, + "templating": { + "list": [ + { + "name": "cluster", + "label": "Cluster", + "type": "query", + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "query": "label_values(pulsar_rate_in, cluster)", + "refresh": 2, + "includeAll": false, + "multi": false, + "current": { "text": "pulsar-lite", "value": "pulsar-lite" } + }, + { + "name": "namespace", + "label": "Namespace", + "type": "query", + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "query": "label_values(pulsar_rate_in{cluster=\"$cluster\"}, namespace)", + "refresh": 2, + "includeAll": true, + "multi": true, + "current": { "text": "All", "value": "$__all" } + }, + { + "name": "topic", + "label": "Topic", + "type": "query", + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "query": "label_values(pulsar_rate_in{cluster=\"$cluster\", namespace=~\"$namespace\"}, topic)", + "refresh": 2, + "includeAll": true, + "multi": true, + "current": { "text": "All", "value": "$__all" } + }, + { + "name": "subscription", + "label": "Subscription", + "type": "query", + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "query": "label_values(pulsar_subscription_msg_rate_out{cluster=\"$cluster\", namespace=~\"$namespace\", topic=~\"$topic\"}, subscription)", + "refresh": 2, + "includeAll": true, + "multi": true, + "current": { "text": "All", "value": "$__all" } + } + ] + }, + "panels": [ + { + "id": 1, + "type": "timeseries", + "title": "Publish rate (msg/s)", + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 0 }, + "targets": [ + { + "expr": "pulsar_rate_in{cluster=\"$cluster\", namespace=~\"$namespace\", topic=~\"$topic\"}", + "legendFormat": "{{namespace}}/{{topic}} [{{partition}}]" + } + ] + }, + { + "id": 2, + "type": "timeseries", + "title": "Publish throughput (bytes/s)", + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 0 }, + "targets": [ + { + "expr": "pulsar_throughput_in{cluster=\"$cluster\", namespace=~\"$namespace\", topic=~\"$topic\"}", + "legendFormat": "{{namespace}}/{{topic}} [{{partition}}]" + } + ] + }, + { + "id": 3, + "type": "timeseries", + "title": "Delivery rate (msg/s)", + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 8 }, + "targets": [ + { + "expr": "pulsar_subscription_msg_rate_out{cluster=\"$cluster\", namespace=~\"$namespace\", topic=~\"$topic\", subscription=~\"$subscription\"}", + "legendFormat": "{{topic}}/{{subscription}}" + } + ] + }, + { + "id": 4, + "type": "timeseries", + "title": "Delivery throughput (bytes/s)", + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 8 }, + "targets": [ + { + "expr": "pulsar_subscription_msg_throughput_out{cluster=\"$cluster\", namespace=~\"$namespace\", topic=~\"$topic\", subscription=~\"$subscription\"}", + "legendFormat": "{{topic}}/{{subscription}}" + } + ] + }, + { + "id": 5, + "type": "timeseries", + "title": "Producers / Consumers / Subscriptions", + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "gridPos": { "h": 8, "w": 8, "x": 0, "y": 16 }, + "targets": [ + { + "expr": "sum(pulsar_producers_count{cluster=\"$cluster\", namespace=~\"$namespace\", topic=~\"$topic\"})", + "legendFormat": "producers" + }, + { + "expr": "sum(pulsar_consumers_count{cluster=\"$cluster\", namespace=~\"$namespace\", topic=~\"$topic\"})", + "legendFormat": "consumers" + }, + { + "expr": "sum(pulsar_subscriptions_count{cluster=\"$cluster\", namespace=~\"$namespace\", topic=~\"$topic\"})", + "legendFormat": "subscriptions" + } + ] + }, + { + "id": 6, + "type": "timeseries", + "title": "Subscription backlog", + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "gridPos": { "h": 8, "w": 8, "x": 8, "y": 16 }, + "targets": [ + { + "expr": "pulsar_subscription_back_log{cluster=\"$cluster\", namespace=~\"$namespace\", topic=~\"$topic\", subscription=~\"$subscription\"}", + "legendFormat": "{{topic}}/{{subscription}}" + } + ] + }, + { + "id": 7, + "type": "timeseries", + "title": "Unacked messages (dispatched, not acked)", + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "gridPos": { "h": 8, "w": 8, "x": 16, "y": 16 }, + "targets": [ + { + "expr": "pulsar_subscription_unacked_messages{cluster=\"$cluster\", namespace=~\"$namespace\", topic=~\"$topic\", subscription=~\"$subscription\"}", + "legendFormat": "{{topic}}/{{subscription}}" + }, + { + "expr": "pulsar_subscription_blocked_on_unacked_messages{cluster=\"$cluster\", namespace=~\"$namespace\", topic=~\"$topic\", subscription=~\"$subscription\"}", + "legendFormat": "{{topic}}/{{subscription}} blocked" + } + ] + }, + { + "id": 8, + "type": "timeseries", + "title": "Storage size (bytes)", + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 24 }, + "targets": [ + { + "expr": "sum(pulsar_storage_size{cluster=\"$cluster\", namespace=~\"$namespace\", topic=~\"$topic\"})", + "legendFormat": "stored bytes" + } + ] + }, + { + "id": 9, + "type": "timeseries", + "title": "Redelivery rate (msg/s)", + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 24 }, + "targets": [ + { + "expr": "pulsar_subscription_msg_rate_redeliver{cluster=\"$cluster\", namespace=~\"$namespace\", topic=~\"$topic\", subscription=~\"$subscription\"}", + "legendFormat": "{{topic}}/{{subscription}}" + } + ] + }, + { + "id": 10, + "type": "timeseries", + "title": "Durable publish latency (e2e, P50/P99)", + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 32 }, + "fieldConfig": { + "defaults": { "unit": "s" }, + "overrides": [] + }, + "targets": [ + { + "expr": "histogram_quantile(0.5, sum by (le) (rate(pulsar_storage_write_latency_bucket[1m])))", + "legendFormat": "P50" + }, + { + "expr": "histogram_quantile(0.99, sum by (le) (rate(pulsar_storage_write_latency_bucket[1m])))", + "legendFormat": "P99" + } + ] + }, + { + "id": 11, + "type": "timeseries", + "title": "Ledger append latency (P50/P99)", + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "gridPos": { "h": 8, "w": 6, "x": 12, "y": 32 }, + "fieldConfig": { + "defaults": { "unit": "s" }, + "overrides": [] + }, + "targets": [ + { + "expr": "histogram_quantile(0.5, sum by (le) (rate(pulsar_storage_ledger_write_latency_bucket[1m])))", + "legendFormat": "P50" + }, + { + "expr": "histogram_quantile(0.99, sum by (le) (rate(pulsar_storage_ledger_write_latency_bucket[1m])))", + "legendFormat": "P99" + } + ] + }, + { + "id": 12, + "type": "timeseries", + "title": "Entry size distribution (P50/P99/max)", + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "gridPos": { "h": 8, "w": 6, "x": 18, "y": 32 }, + "fieldConfig": { + "defaults": { "unit": "bytes" }, + "overrides": [] + }, + "targets": [ + { + "expr": "histogram_quantile(0.5, sum by (le) (rate(pulsar_entry_size_bucket[5m])))", + "legendFormat": "P50" + }, + { + "expr": "histogram_quantile(0.99, sum by (le) (rate(pulsar_entry_size_bucket[5m])))", + "legendFormat": "P99" + } + ] + } + ] +} diff --git a/grafana/docker-compose.yml b/grafana/docker-compose.yml new file mode 100644 index 0000000..860acb4 --- /dev/null +++ b/grafana/docker-compose.yml @@ -0,0 +1,31 @@ +services: + prometheus: + image: prom/prometheus:v3.1.0 + container_name: pulsar-lite-prometheus + command: + - --config.file=/etc/prometheus/prometheus.yml + - --storage.tsdb.retention.time=7d + volumes: + - ./prometheus:/etc/prometheus:ro + ports: + - "9090:9090" + extra_hosts: + - "host.docker.internal:host-gateway" + restart: unless-stopped + + grafana: + image: grafana/grafana:11.5.2 + container_name: pulsar-lite-grafana + environment: + - GF_SECURITY_ADMIN_USER=admin + - GF_SECURITY_ADMIN_PASSWORD=admin + - GF_AUTH_ANONYMOUS_ENABLED=true + - GF_AUTH_ANONYMOUS_ORG_ROLE=Viewer + volumes: + - ./grafana/provisioning:/etc/grafana/provisioning:ro + - ./dashboards:/var/lib/grafana/dashboards:ro + ports: + - "7070:3000" + depends_on: + - prometheus + restart: unless-stopped diff --git a/grafana/grafana/provisioning/dashboards/provider.yml b/grafana/grafana/provisioning/dashboards/provider.yml new file mode 100644 index 0000000..2aea31a --- /dev/null +++ b/grafana/grafana/provisioning/dashboards/provider.yml @@ -0,0 +1,13 @@ +apiVersion: 1 + +providers: + - name: pulsar-lite + orgId: 1 + folder: Pulsar Lite + type: file + disableDeletion: false + updateIntervalSeconds: 30 + allowUiUpdates: true + options: + path: /var/lib/grafana/dashboards + foldersFromFilesStructure: false diff --git a/grafana/grafana/provisioning/datasources/prometheus.yml b/grafana/grafana/provisioning/datasources/prometheus.yml new file mode 100644 index 0000000..00f9915 --- /dev/null +++ b/grafana/grafana/provisioning/datasources/prometheus.yml @@ -0,0 +1,10 @@ +apiVersion: 1 + +datasources: + - name: Prometheus + uid: prometheus + type: prometheus + access: proxy + url: http://prometheus:9090 + isDefault: true + editable: false diff --git a/grafana/prometheus/prometheus.yml b/grafana/prometheus/prometheus.yml new file mode 100644 index 0000000..539de14 --- /dev/null +++ b/grafana/prometheus/prometheus.yml @@ -0,0 +1,27 @@ +global: + scrape_interval: 5s + evaluation_interval: 5s + +scrape_configs: + - job_name: pulsar-lite + metrics_path: /metrics + static_configs: + # The broker's metrics endpoint (default 0.0.0.0:8080, same model as + # native Pulsar's web service port). host.docker.internal resolves to + # the host via the compose extra_hosts entry. + - targets: ["host.docker.internal:8080"] + + # Perf harness brokers (tests/perf/lib/broker.py derives each broker's + # metrics port from its protocol port: 6651/6652 -> 8081/8082, + # 6661/6662 -> 8091/8092, 6671/6672 -> 8101/8102, bound on 0.0.0.0). + # Targets show DOWN while no run is active — expected. + - job_name: pulsar-lite-perf + metrics_path: /metrics + static_configs: + - targets: + - "host.docker.internal:8081" + - "host.docker.internal:8082" + - "host.docker.internal:8091" + - "host.docker.internal:8092" + - "host.docker.internal:8101" + - "host.docker.internal:8102" diff --git a/rust/Cargo.lock b/rust/Cargo.lock index 866777f..192c82c 100644 --- a/rust/Cargo.lock +++ b/rust/Cargo.lock @@ -71,7 +71,7 @@ version = "1.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" dependencies = [ - "windows-sys", + "windows-sys 0.61.2", ] [[package]] @@ -82,7 +82,7 @@ checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" dependencies = [ "anstyle", "once_cell_polyfill", - "windows-sys", + "windows-sys 0.61.2", ] [[package]] @@ -91,6 +91,15 @@ version = "1.0.102" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f202df86484c868dbad7eaa557ef785d5c66295e41b460ef922eca0723b842c" +[[package]] +name = "arc-swap" +version = "1.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c049c0be4daef0b145cb3555416b3b8ef5b7888a38aea1a3a155801fe7b0810b" +dependencies = [ + "rustversion", +] + [[package]] name = "async-trait" version = "0.1.89" @@ -99,9 +108,15 @@ checksum = "9035ad2d096bed7955a320ee7e2230574d28fd3c3a0f186cbea1ff3c7eed5dbb" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] +[[package]] +name = "atomic-waker" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" + [[package]] name = "bincode" version = "1.3.3" @@ -126,7 +141,7 @@ dependencies = [ "regex", "rustc-hash", "shlex", - "syn", + "syn 2.0.117", ] [[package]] @@ -226,7 +241,7 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -283,7 +298,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys", + "windows-sys 0.61.2", ] [[package]] @@ -304,6 +319,12 @@ version = "0.5.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1d674e81391d1e1ab681a28d99df07927c6d4aa5b027d7da16ba32d1d21ecd99" +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + [[package]] name = "foldhash" version = "0.1.5" @@ -366,7 +387,7 @@ checksum = "e835b70203e41293343137df5c0664546da5745f82ec9b84d40be8336958447b" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -450,6 +471,91 @@ version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" +[[package]] +name = "hex" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" + +[[package]] +name = "http" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "918d3568bebf352712bc2ef3d46a8bcf1a75b373be6539de198e9105cbbf9ce0" +dependencies = [ + "bytes", + "itoa", +] + +[[package]] +name = "http-body" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca2a8f2913ee65f60facd6a5905613afaa448497a0230cc41ce022d93290bc2c" +dependencies = [ + "bytes", + "http", +] + +[[package]] +name = "http-body-util" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23169fe34a5fbcdd3f3862e78fb9b6fccd5f02a6dc6f732547005d45631ce71c" +dependencies = [ + "bytes", + "futures-core", + "http", + "http-body", + "pin-project-lite", +] + +[[package]] +name = "httparse" +version = "1.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87" + +[[package]] +name = "httpdate" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" + +[[package]] +name = "hyper" +version = "1.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d22053281f852e11534f5198498373cbb59295120a20771d90f7ed1897490a72" +dependencies = [ + "atomic-waker", + "bytes", + "futures-channel", + "futures-core", + "http", + "http-body", + "httparse", + "httpdate", + "itoa", + "pin-project-lite", + "smallvec", + "tokio", +] + +[[package]] +name = "hyper-util" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96547c2556ec9d12fb1578c4eaf448b04993e7fb79cbaad930a656880a6bdfa0" +dependencies = [ + "bytes", + "http", + "http-body", + "hyper", + "pin-project-lite", + "tokio", +] + [[package]] name = "id-arena" version = "2.3.0" @@ -519,7 +625,7 @@ checksum = "2a8c8b344124222efd714b73bb41f8b5120b27a7cc1c75593a6ff768d9d05aa4" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -542,6 +648,12 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "lazy_static" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" + [[package]] name = "leb128fmt" version = "0.1.0" @@ -590,6 +702,12 @@ dependencies = [ "vcpkg", ] +[[package]] +name = "linux-raw-sys" +version = "0.4.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d26c52dbd32dccf2d10cac7725f8eae5296885fb5703b261f7d0a0739ec807ab" + [[package]] name = "linux-raw-sys" version = "0.12.1" @@ -641,7 +759,7 @@ checksum = "02bd0af71c67b473010cbbc60715ee815645a4dc942899111f494b4b737d6fda" dependencies = [ "libc", "wasi", - "windows-sys", + "windows-sys 0.61.2", ] [[package]] @@ -739,7 +857,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b" dependencies = [ "proc-macro2", - "syn", + "syn 2.0.117", ] [[package]] @@ -751,6 +869,59 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "procfs" +version = "0.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc5b72d8145275d844d4b5f6d4e1eef00c8cd889edb6035c21675d1bb1f45c9f" +dependencies = [ + "bitflags", + "hex", + "procfs-core", + "rustix 0.38.44", +] + +[[package]] +name = "procfs-core" +version = "0.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "239df02d8349b06fc07398a3a1697b06418223b1c7725085e801e7c0fc6a12ec" +dependencies = [ + "bitflags", + "hex", +] + +[[package]] +name = "prometheus" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ca5326d8d0b950a9acd87e6a3f94745394f62e4dae1b1ee22b2bc0c394af43a" +dependencies = [ + "cfg-if", + "fnv", + "lazy_static", + "libc", + "memchr", + "parking_lot", + "procfs", + "thiserror 2.0.20", +] + +[[package]] +name = "prometheus-hyper" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "60d6eeac44b972d6f552e8aaec7f869200aef42f14ddffdddde308b94d0c066e" +dependencies = [ + "bytes", + "http-body-util", + "hyper", + "hyper-util", + "prometheus", + "tokio", + "tracing", +] + [[package]] name = "prost" version = "0.13.5" @@ -777,7 +948,7 @@ dependencies = [ "prost", "prost-types", "regex", - "syn", + "syn 2.0.117", "tempfile", ] @@ -791,7 +962,7 @@ dependencies = [ "itertools 0.14.0", "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -815,9 +986,13 @@ dependencies = [ "env_logger", "futures", "log", + "prometheus", + "prometheus-hyper", "prost", "prost-build", "prost-types", + "pulsar-lite-metrics", + "pulsar-lite-proto", "pulsar-lite-storage", "pulsar-lite-storage-managed-ledger", "pulsar-lite-storage-managed-ledger-rocksdb", @@ -826,7 +1001,9 @@ dependencies = [ "serde", "serde_json", "tempfile", - "thiserror", + "thiserror 1.0.69", + "tikv-jemalloc-sys", + "tikv-jemallocator", "tokio", "tokio-stream", "tokio-util", @@ -834,12 +1011,34 @@ dependencies = [ "uuid", ] +[[package]] +name = "pulsar-lite-metrics" +version = "0.1.0" +dependencies = [ + "log", + "prometheus", +] + +[[package]] +name = "pulsar-lite-proto" +version = "0.1.0" +dependencies = [ + "bytes", + "prost", + "prost-build", + "prost-types", + "tokio-util", +] + [[package]] name = "pulsar-lite-storage" version = "0.1.0" dependencies = [ "anyhow", "log", + "prometheus", + "prost", + "pulsar-lite-proto", "pulsar-lite-storage-managed-ledger", "pulsar-lite-storage-managed-ledger-rocksdb", "pulsar-lite-storage-metadata", @@ -865,10 +1064,13 @@ name = "pulsar-lite-storage-managed-ledger-rocksdb" version = "0.1.0" dependencies = [ "anyhow", + "arc-swap", "bincode", "log", "prost", "prost-build", + "pulsar-lite-metrics", + "pulsar-lite-proto", "pulsar-lite-storage-managed-ledger", "rocksdb", "serde", @@ -974,6 +1176,19 @@ version = "2.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "94300abf3f1ae2e2b8ffb7b58043de3d399c73fa6f4b73826402a5c457614dbe" +[[package]] +name = "rustix" +version = "0.38.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fdb5bc1ae2baa591800df16c9ca78619bf65c0488b41b96ccec5d11220d8c154" +dependencies = [ + "bitflags", + "errno", + "libc", + "linux-raw-sys 0.4.15", + "windows-sys 0.59.0", +] + [[package]] name = "rustix" version = "1.1.4" @@ -983,8 +1198,8 @@ dependencies = [ "bitflags", "errno", "libc", - "linux-raw-sys", - "windows-sys", + "linux-raw-sys 0.12.1", + "windows-sys 0.61.2", ] [[package]] @@ -1032,7 +1247,7 @@ checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -1092,7 +1307,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3a766e1110788c36f4fa1c2b71b387a7815aa65f88ce0229841826633d93723e" dependencies = [ "libc", - "windows-sys", + "windows-sys 0.61.2", ] [[package]] @@ -1112,6 +1327,17 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "syn" +version = "3.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53e9bae58849f64dfa4f5d5ae372c8341f7305f82a3868709269343628b659a3" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + [[package]] name = "tempfile" version = "3.26.0" @@ -1121,8 +1347,8 @@ dependencies = [ "fastrand", "getrandom 0.4.2", "once_cell", - "rustix", - "windows-sys", + "rustix 1.1.4", + "windows-sys 0.61.2", ] [[package]] @@ -1131,7 +1357,16 @@ version = "1.0.69" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6aaf5339b578ea85b50e080feb250a3e8ae8cfcdff9a461c9ec2904bc923f52" dependencies = [ - "thiserror-impl", + "thiserror-impl 1.0.69", +] + +[[package]] +name = "thiserror" +version = "2.0.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec86235f5fcc2a73650310756d2ac5b138a5780bbbdfae3eeccec992c435ba4f" +dependencies = [ + "thiserror-impl 2.0.20", ] [[package]] @@ -1142,7 +1377,38 @@ checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", +] + +[[package]] +name = "thiserror-impl" +version = "2.0.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc04cd3e1236dd4a98afca4569f2deb3f120e5422a4023be2cb683f8486292af" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.3", +] + +[[package]] +name = "tikv-jemalloc-sys" +version = "0.6.1+5.3.0-1-ge13ca993e8ccb9ba9847cc330696e02839f328f7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd8aa5b2ab86a2cefa406d889139c162cbb230092f7d1d7cbc1716405d852a3b" +dependencies = [ + "cc", + "libc", +] + +[[package]] +name = "tikv-jemallocator" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0359b4327f954e0567e69fb191cf1436617748813819c94b8cd4a431422d053a" +dependencies = [ + "libc", + "tikv-jemalloc-sys", ] [[package]] @@ -1159,7 +1425,7 @@ dependencies = [ "signal-hook-registry", "socket2", "tokio-macros", - "windows-sys", + "windows-sys 0.61.2", ] [[package]] @@ -1170,7 +1436,7 @@ checksum = "385a6cb71ab9ab790c5fe8d67f1645e6c450a7ce006a33de03daa956cf70a496" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -1238,6 +1504,37 @@ version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5d99f8c9a7727884afe522e9bd5edbfc91a3312b36a77b5fb8926e4c31a41801" +[[package]] +name = "tracing" +version = "0.1.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63e71662fa4b2a2c3a26f570f037eb95bb1f85397f3cd8076caed2f026a6d100" +dependencies = [ + "pin-project-lite", + "tracing-attributes", + "tracing-core", +] + +[[package]] +name = "tracing-attributes" +version = "0.1.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "tracing-core" +version = "0.1.36" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db97caf9d906fbde555dd62fa95ddba9eecfd14cb388e4f491a66d74cd5fb79a" +dependencies = [ + "once_cell", +] + [[package]] name = "unicode-ident" version = "1.0.24" @@ -1329,7 +1626,7 @@ dependencies = [ "bumpalo", "proc-macro2", "quote", - "syn", + "syn 2.0.117", "wasm-bindgen-shared", ] @@ -1382,6 +1679,15 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" +[[package]] +name = "windows-sys" +version = "0.59.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b" +dependencies = [ + "windows-targets", +] + [[package]] name = "windows-sys" version = "0.61.2" @@ -1391,6 +1697,70 @@ dependencies = [ "windows-link", ] +[[package]] +name = "windows-targets" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" +dependencies = [ + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_gnullvm", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" + +[[package]] +name = "windows_i686_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" + +[[package]] +name = "windows_i686_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" + [[package]] name = "winnow" version = "0.7.15" @@ -1430,7 +1800,7 @@ dependencies = [ "heck", "indexmap", "prettyplease", - "syn", + "syn 2.0.117", "wasm-metadata", "wit-bindgen-core", "wit-component", @@ -1446,7 +1816,7 @@ dependencies = [ "prettyplease", "proc-macro2", "quote", - "syn", + "syn 2.0.117", "wit-bindgen-core", "wit-bindgen-rust", ] diff --git a/rust/Cargo.toml b/rust/Cargo.toml index 75a1382..06d8f6a 100644 --- a/rust/Cargo.toml +++ b/rust/Cargo.toml @@ -1,11 +1,13 @@ [workspace] members = [ ".", + "metrics", "storage/core", "storage/metadata", "storage/resources", "storage/managed-ledger", "storage/managed-ledger-rocksdb", + "proto", ] resolver = "2" @@ -21,6 +23,10 @@ thiserror = "1.0" tokio = { version = "1.35", features = ["full"] } anyhow = "1.0" log = "0.4" +tikv-jemallocator = { version = "0.6", features = ["unprefixed_malloc_on_supported_platforms"] } +tikv-jemalloc-sys = { version = "0.6", features = ["profiling", "unprefixed_malloc_on_supported_platforms"] } +prometheus = { version = "0.14", default-features = false, features = ["process"] } +prometheus-hyper = "0.2" [package] name = "pulsar-lite" @@ -45,6 +51,7 @@ pulsar-lite-storage-metadata = { path = "storage/metadata" } pulsar-lite-storage-resources = { path ="storage/resources" } pulsar-lite-storage-managed-ledger = { path = "storage/managed-ledger" } pulsar-lite-storage-managed-ledger-rocksdb = { path = "storage/managed-ledger-rocksdb", optional = true } +pulsar-lite-proto = { path = "proto" } # 异步运行时 tokio = { version = "1.35", features = ["full"] } @@ -81,6 +88,14 @@ uuid = { version = "1.6", features = ["v4"] } # 流处理 tokio-stream = "0.1" futures = "0.3" +# Metrics: Prometheus families and HTTP export (GET /metrics) +prometheus.workspace = true +prometheus-hyper.workspace = true +pulsar-lite-metrics = { path = "metrics" } + +# 内存分配器:jemalloc(缓解 glibc arena 内存滞留导致的 RSS 高水位) +tikv-jemallocator.workspace = true +tikv-jemalloc-sys.workspace = true [build-dependencies] prost-build = "0.13" diff --git a/rust/metrics/Cargo.toml b/rust/metrics/Cargo.toml new file mode 100644 index 0000000..beaac8f --- /dev/null +++ b/rust/metrics/Cargo.toml @@ -0,0 +1,9 @@ +[package] +name = "pulsar-lite-metrics" +version = "0.1.0" +edition.workspace = true +license.workspace = true + +[dependencies] +prometheus.workspace = true +log.workspace = true diff --git a/rust/metrics/src/broker.rs b/rust/metrics/src/broker.rs new file mode 100644 index 0000000..f3f29d5 --- /dev/null +++ b/rust/metrics/src/broker.rs @@ -0,0 +1,581 @@ +/* + * Broker Prometheus Metric Families + * + * Families are created once at startup and registered into the shared + * registry from `pulsar_lite_storage::metrics`. Hot paths only ever touch + * pre-resolved handles (plain atomic adds); label resolution happens once + * at entity creation, never per message. + * + * Naming contract: families that reproduce native Pulsar semantics keep + * the exact `pulsar_*` names and label sets; extensions with no native + * counterpart use the `pulsar_lite_*` prefix. + */ + +use std::sync::{Arc, OnceLock}; + +use prometheus::{Gauge, GaugeVec, IntCounter, IntCounterVec, IntGauge, IntGaugeVec, Opts}; +use crate::global_registry; + +/// Topic-labeled families shared by every `TopicMetrics` handle. +#[derive(Debug, Clone)] +pub struct TopicFamilies { + /// pulsar_in_messages_total{cluster, namespace, topic, partition} + pub in_messages: IntCounterVec, + /// pulsar_in_bytes_total{...} + pub in_bytes: IntCounterVec, + /// pulsar_publish_rate_limit_times{...} + pub publish_rate_limit: IntCounterVec, + /// pulsar_rate_in{...} (scrape-derived gauge) + pub rate_in: GaugeVec, + /// pulsar_throughput_in{...} + pub throughput_in: GaugeVec, + /// pulsar_average_msg_size{...} + pub average_msg_size: IntGaugeVec, + /// pulsar_storage_size{...} + pub storage_size: IntGaugeVec, + /// pulsar_subscriptions_count{...} + pub subscriptions_count: IntGaugeVec, + /// pulsar_producers_count{...} + pub producers_count: IntGaugeVec, + /// pulsar_consumers_count{...} + pub consumers_count: IntGaugeVec, +} + +/// Subscription-labeled families shared by every `SubscriptionMetrics`. +#[derive(Debug, Clone)] +pub struct SubscriptionFamilies { + /// pulsar_out_messages_total{cluster, namespace, topic, partition, subscription} + pub out_messages: IntCounterVec, + /// pulsar_out_bytes_total{...} + pub out_bytes: IntCounterVec, + /// pulsar_lite_subscription_redelivered_total{...} + pub redelivered: IntCounterVec, + /// pulsar_lite_subscription_dropped_messages_total{...} + pub dropped: IntCounterVec, + /// pulsar_lite_subscription_acked_messages_total{...} + pub acked: IntCounterVec, + /// pulsar_subscription_msg_rate_out{...} + pub msg_rate_out: GaugeVec, + /// pulsar_subscription_msg_throughput_out{...} + pub msg_throughput_out: GaugeVec, + /// pulsar_subscription_msg_ack_rate{...} + pub msg_ack_rate: GaugeVec, + /// pulsar_subscription_msg_rate_redeliver{...} + pub msg_rate_redeliver: GaugeVec, + /// pulsar_subscription_msg_drop_rate{...} + pub msg_drop_rate: GaugeVec, + /// pulsar_subscription_back_log{...} + pub back_log: IntGaugeVec, + /// pulsar_subscription_unacked_messages{...} + pub unacked_messages: IntGaugeVec, + /// pulsar_subscription_blocked_on_unacked_messages{...} + pub blocked_on_unacked: IntGaugeVec, + /// pulsar_subscription_consumers_count{...} + pub consumers_count: IntGaugeVec, + /// pulsar_subscription_last_acked_timestamp{...} (unix seconds) + pub last_acked_timestamp: GaugeVec, + /// pulsar_subscription_last_consumed_timestamp{...} (unix seconds) + pub last_consumed_timestamp: GaugeVec, +} + +impl SubscriptionFamilies { + fn new(registry: &prometheus::Registry) -> Result { + let out_messages = IntCounterVec::new( + Opts::new( + "pulsar_out_messages_total", + "Total messages delivered to consumers", + ), + SUBSCRIPTION_LABELS, + )?; + let out_bytes = IntCounterVec::new( + Opts::new( + "pulsar_out_bytes_total", + "Total bytes delivered to consumers", + ), + SUBSCRIPTION_LABELS, + )?; + let redelivered = IntCounterVec::new( + Opts::new( + "pulsar_lite_subscription_redelivered_total", + "Messages queued for redelivery", + ), + SUBSCRIPTION_LABELS, + )?; + let dropped = IntCounterVec::new( + Opts::new( + "pulsar_lite_subscription_dropped_messages_total", + "Non-persistent messages dropped with no writable consumer", + ), + SUBSCRIPTION_LABELS, + )?; + let acked = IntCounterVec::new( + Opts::new( + "pulsar_lite_subscription_acked_messages_total", + "Messages acknowledged by consumers", + ), + SUBSCRIPTION_LABELS, + )?; + let msg_rate_out = GaugeVec::new( + Opts::new( + "pulsar_subscription_msg_rate_out", + "Delivered message rate (window average)", + ), + SUBSCRIPTION_LABELS, + )?; + let msg_throughput_out = GaugeVec::new( + Opts::new( + "pulsar_subscription_msg_throughput_out", + "Delivered byte rate (window average)", + ), + SUBSCRIPTION_LABELS, + )?; + let msg_ack_rate = GaugeVec::new( + Opts::new( + "pulsar_subscription_msg_ack_rate", + "Acknowledge rate (window average)", + ), + SUBSCRIPTION_LABELS, + )?; + let msg_rate_redeliver = GaugeVec::new( + Opts::new( + "pulsar_subscription_msg_rate_redeliver", + "Redelivery rate (window average)", + ), + SUBSCRIPTION_LABELS, + )?; + let msg_drop_rate = GaugeVec::new( + Opts::new( + "pulsar_subscription_msg_drop_rate", + "Non-persistent drop rate (window average)", + ), + SUBSCRIPTION_LABELS, + )?; + let back_log = IntGaugeVec::new( + Opts::new( + "pulsar_subscription_back_log", + "Unacknowledged stored entries", + ), + SUBSCRIPTION_LABELS, + )?; + let unacked_messages = IntGaugeVec::new( + Opts::new( + "pulsar_subscription_unacked_messages", + "Dispatched-but-unacknowledged messages", + ), + SUBSCRIPTION_LABELS, + )?; + let blocked_on_unacked = IntGaugeVec::new( + Opts::new( + "pulsar_subscription_blocked_on_unacked_messages", + "1 while dispatch is blocked by the unacked gate", + ), + SUBSCRIPTION_LABELS, + )?; + let consumers_count = IntGaugeVec::new( + Opts::new("pulsar_subscription_consumers_count", "Connected consumers"), + SUBSCRIPTION_LABELS, + )?; + let last_acked_timestamp = GaugeVec::new( + Opts::new( + "pulsar_subscription_last_acked_timestamp", + "Last acknowledge time (unix seconds)", + ), + SUBSCRIPTION_LABELS, + )?; + let last_consumed_timestamp = GaugeVec::new( + Opts::new( + "pulsar_subscription_last_consumed_timestamp", + "Last dispatch time (unix seconds)", + ), + SUBSCRIPTION_LABELS, + )?; + + registry.register(Box::new(out_messages.clone()))?; + registry.register(Box::new(out_bytes.clone()))?; + registry.register(Box::new(redelivered.clone()))?; + registry.register(Box::new(dropped.clone()))?; + registry.register(Box::new(acked.clone()))?; + registry.register(Box::new(msg_rate_out.clone()))?; + registry.register(Box::new(msg_throughput_out.clone()))?; + registry.register(Box::new(msg_ack_rate.clone()))?; + registry.register(Box::new(msg_rate_redeliver.clone()))?; + registry.register(Box::new(msg_drop_rate.clone()))?; + registry.register(Box::new(back_log.clone()))?; + registry.register(Box::new(unacked_messages.clone()))?; + registry.register(Box::new(blocked_on_unacked.clone()))?; + registry.register(Box::new(consumers_count.clone()))?; + registry.register(Box::new(last_acked_timestamp.clone()))?; + registry.register(Box::new(last_consumed_timestamp.clone()))?; + + Ok(Self { + out_messages, + out_bytes, + redelivered, + dropped, + acked, + msg_rate_out, + msg_throughput_out, + msg_ack_rate, + msg_rate_redeliver, + msg_drop_rate, + back_log, + unacked_messages, + blocked_on_unacked, + consumers_count, + last_acked_timestamp, + last_consumed_timestamp, + }) + } +} + +const SUBSCRIPTION_LABELS: &[&str] = + &["cluster", "namespace", "topic", "partition", "subscription"]; +const TOPIC_LABELS: &[&str] = &["cluster", "namespace", "topic", "partition"]; + +impl TopicFamilies { + fn new(registry: &prometheus::Registry) -> Result { + let in_messages = IntCounterVec::new( + Opts::new( + "pulsar_in_messages_total", + "Total messages accepted for publish", + ), + TOPIC_LABELS, + )?; + let in_bytes = IntCounterVec::new( + Opts::new("pulsar_in_bytes_total", "Total bytes accepted for publish"), + TOPIC_LABELS, + )?; + let publish_rate_limit = IntCounterVec::new( + Opts::new( + "pulsar_publish_rate_limit_times", + "Publishes rejected by the topic rate limiter", + ), + TOPIC_LABELS, + )?; + let rate_in = GaugeVec::new( + Opts::new("pulsar_rate_in", "Accepted message rate (window average)"), + TOPIC_LABELS, + )?; + let throughput_in = GaugeVec::new( + Opts::new( + "pulsar_throughput_in", + "Accepted byte rate (window average)", + ), + TOPIC_LABELS, + )?; + let average_msg_size = IntGaugeVec::new( + Opts::new( + "pulsar_average_msg_size", + "Average accepted message size in bytes", + ), + TOPIC_LABELS, + )?; + let storage_size = IntGaugeVec::new( + Opts::new("pulsar_storage_size", "Bytes stored for the topic"), + TOPIC_LABELS, + )?; + let subscriptions_count = IntGaugeVec::new( + Opts::new( + "pulsar_subscriptions_count", + "Active subscriptions on the topic", + ), + TOPIC_LABELS, + )?; + let producers_count = IntGaugeVec::new( + Opts::new("pulsar_producers_count", "Connected producers on the topic"), + TOPIC_LABELS, + )?; + let consumers_count = IntGaugeVec::new( + Opts::new("pulsar_consumers_count", "Connected consumers on the topic"), + TOPIC_LABELS, + )?; + + registry.register(Box::new(in_messages.clone()))?; + registry.register(Box::new(in_bytes.clone()))?; + registry.register(Box::new(publish_rate_limit.clone()))?; + registry.register(Box::new(rate_in.clone()))?; + registry.register(Box::new(throughput_in.clone()))?; + registry.register(Box::new(average_msg_size.clone()))?; + registry.register(Box::new(storage_size.clone()))?; + registry.register(Box::new(subscriptions_count.clone()))?; + registry.register(Box::new(producers_count.clone()))?; + registry.register(Box::new(consumers_count.clone()))?; + + Ok(Self { + in_messages, + in_bytes, + publish_rate_limit, + rate_in, + throughput_in, + average_msg_size, + storage_size, + subscriptions_count, + producers_count, + consumers_count, + }) + } +} + +/// Broker-scoped metric families. +#[derive(Debug)] +pub struct BrokerMetrics { + /// `cluster` label value applied to every family (config-injected). + cluster: String, + + /// Topic-labeled families (per-`TopicMetrics` handles resolve from these). + pub topics: TopicFamilies, + /// Subscription-labeled families (per-`SubscriptionMetrics` handles). + pub subscriptions: SubscriptionFamilies, + + /// pulsar_active_connections{cluster} + pub active_connections: IntGauge, + /// pulsar_connection_created_total_count{cluster} + pub connection_created: IntCounter, + /// pulsar_connection_closed_total_count{cluster} + pub connection_closed: IntCounter, + /// pulsar_lite_broker_errors_total{cluster, reason} + pub errors: IntCounterVec, + /// pulsar_broker_in_messages_total{cluster} + pub broker_in_messages: IntCounter, + /// pulsar_broker_in_bytes_total{cluster} + pub broker_in_bytes: IntCounter, + /// pulsar_broker_out_messages_total{cluster} + pub broker_out_messages: IntCounter, + /// pulsar_broker_out_bytes_total{cluster} + pub broker_out_bytes: IntCounter, + /// pulsar_broker_topics_count{cluster} (scrape-set) + pub broker_topics_count: IntGauge, + /// pulsar_broker_subscriptions_count{cluster} (scrape-set) + pub broker_subscriptions_count: IntGauge, + /// pulsar_broker_producers_count{cluster} (scrape-set) + pub broker_producers_count: IntGauge, + /// pulsar_broker_consumers_count{cluster} (scrape-set) + pub broker_consumers_count: IntGauge, + /// pulsar_broker_rate_in{cluster} (scrape-derived) + pub broker_rate_in: Gauge, + /// pulsar_broker_throughput_in{cluster} + pub broker_throughput_in: Gauge, + /// pulsar_broker_rate_out{cluster} + pub broker_rate_out: Gauge, + /// pulsar_broker_throughput_out{cluster} + pub broker_throughput_out: Gauge, + /// pulsar_broker_msg_backlog{cluster} + pub broker_msg_backlog: IntGauge, + /// pulsar_broker_storage_size{cluster} + pub broker_storage_size: IntGauge, +} + +impl BrokerMetrics { + fn new(cluster: &str, registry: &prometheus::Registry) -> Result { + let topics = TopicFamilies::new(registry)?; + let subscriptions = SubscriptionFamilies::new(registry)?; + + let active_connections = IntGauge::new( + "pulsar_active_connections", + "Currently open client connections", + )?; + let connection_created = IntCounter::new( + "pulsar_connection_created_total_count", + "Total accepted connections", + )?; + let connection_closed = IntCounter::new( + "pulsar_connection_closed_total_count", + "Total closed connections", + )?; + let errors = IntCounterVec::new( + Opts::new( + "pulsar_lite_broker_errors_total", + "Rejected or failed operations by reason", + ), + &["cluster", "reason"], + )?; + let version_info = IntGaugeVec::new( + Opts::new("pulsar_version_info", "Broker version constant"), + &["cluster", "version"], + )?; + let broker_in_messages = IntCounter::new( + "pulsar_broker_in_messages_total", + "Total messages accepted for publish", + )?; + let broker_in_bytes = IntCounter::new( + "pulsar_broker_in_bytes_total", + "Total bytes accepted for publish", + )?; + let broker_out_messages = IntCounter::new( + "pulsar_broker_out_messages_total", + "Total messages delivered to consumers", + )?; + let broker_out_bytes = IntCounter::new( + "pulsar_broker_out_bytes_total", + "Total bytes delivered to consumers", + )?; + let broker_topics_count = + IntGauge::new("pulsar_broker_topics_count", "Topics hosted by the broker")?; + let broker_subscriptions_count = IntGauge::new( + "pulsar_broker_subscriptions_count", + "Subscriptions hosted by the broker", + )?; + let broker_producers_count = IntGauge::new( + "pulsar_broker_producers_count", + "Connected producers on the broker", + )?; + let broker_consumers_count = IntGauge::new( + "pulsar_broker_consumers_count", + "Connected consumers on the broker", + )?; + let broker_rate_in = Gauge::new( + "pulsar_broker_rate_in", + "Accepted message rate across all topics (window average)", + )?; + let broker_throughput_in = Gauge::new( + "pulsar_broker_throughput_in", + "Accepted byte rate across all topics (window average)", + )?; + let broker_rate_out = Gauge::new( + "pulsar_broker_rate_out", + "Delivered message rate across all topics (window average)", + )?; + let broker_throughput_out = Gauge::new( + "pulsar_broker_throughput_out", + "Delivered byte rate across all topics (window average)", + )?; + let broker_msg_backlog = + IntGauge::new("pulsar_broker_msg_backlog", "Total backlog entries")?; + let broker_storage_size = + IntGauge::new("pulsar_broker_storage_size", "Total stored bytes")?; + + registry.register(Box::new(active_connections.clone()))?; + registry.register(Box::new(connection_created.clone()))?; + registry.register(Box::new(connection_closed.clone()))?; + registry.register(Box::new(errors.clone()))?; + registry.register(Box::new(version_info.clone()))?; + registry.register(Box::new(broker_in_messages.clone()))?; + registry.register(Box::new(broker_in_bytes.clone()))?; + registry.register(Box::new(broker_out_messages.clone()))?; + registry.register(Box::new(broker_out_bytes.clone()))?; + registry.register(Box::new(broker_topics_count.clone()))?; + registry.register(Box::new(broker_subscriptions_count.clone()))?; + registry.register(Box::new(broker_producers_count.clone()))?; + registry.register(Box::new(broker_consumers_count.clone()))?; + registry.register(Box::new(broker_rate_in.clone()))?; + registry.register(Box::new(broker_throughput_in.clone()))?; + registry.register(Box::new(broker_rate_out.clone()))?; + registry.register(Box::new(broker_throughput_out.clone()))?; + registry.register(Box::new(broker_msg_backlog.clone()))?; + registry.register(Box::new(broker_storage_size.clone()))?; + // The registry keeps the gauge alive; we only pin the value once. + version_info + .with_label_values(&[cluster, env!("CARGO_PKG_VERSION")]) + .set(1); + Ok(Self { + cluster: cluster.to_string(), + topics, + subscriptions, + active_connections, + connection_created, + connection_closed, + errors, + broker_in_messages, + broker_in_bytes, + broker_out_messages, + broker_out_bytes, + broker_topics_count, + broker_subscriptions_count, + broker_producers_count, + broker_consumers_count, + broker_rate_in, + broker_throughput_in, + broker_rate_out, + broker_throughput_out, + broker_msg_backlog, + broker_storage_size, + }) + } + + /// The configured `cluster` label value. + pub fn cluster(&self) -> &str { + &self.cluster + } + + /// Pre-resolves an error counter for `reason` (one label lookup, then + /// stored by the caller; `inc()` is a single atomic add). + pub fn error_counter(&self, reason: &str) -> IntCounter { + self.errors + .with_label_values(&[self.cluster.as_str(), reason]) + } + + /// Fallback constructor used only when registration failed: identical + /// handles, not attached to the served registry. + fn unregistered(cluster: &str) -> Self { + // Static family names are proven valid by tests; registration into a + // fresh registry cannot fail for any other reason. + Self::new(cluster, &prometheus::Registry::new()) + .expect("static broker metric definitions are valid") + } +} + +static BROKER_METRICS: OnceLock> = OnceLock::new(); + +/// Initializes broker metric families into the shared global registry. +/// +/// Idempotent: the first call wins, so unit tests and production startup +/// share one code path. Construction only fails on invalid static names +/// (a programming error caught by tests); in that case we log once and +/// fall back to unregistered families so counters stay callable. +/// Initializes broker metric families into the shared global registry. +/// +/// Idempotent: the first call wins, so unit tests and production startup +/// share one code path. +pub fn init(cluster: &str) -> Arc { + BROKER_METRICS + .get_or_init(|| match BrokerMetrics::new(cluster, &global_registry()) { + Ok(metrics) => Arc::new(metrics), + Err(error) => { + log::error!("Failed to register broker metrics: {}", error); + Arc::new(BrokerMetrics::unregistered(cluster)) + } + }) + .clone() +} + +/// Returns the initialized metrics, auto-initializing with the default +/// cluster label for callers (tests, library embedders) that skipped +/// explicit startup wiring. +pub fn get() -> Arc { + init("pulsar-lite") +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn init_is_idempotent_and_counters_tick() { + let metrics = init("test-cluster"); + assert!(Arc::ptr_eq(&metrics, &init("other-cluster"))); + + let before = metrics.connection_created.get(); + metrics.connection_created.inc(); + // Unique label: the served registry is process-global and other + // tests (oversized-message handling) also bump error counters. + let rejected = metrics.error_counter("registry_unit_test"); + rejected.inc(); + assert!(metrics.connection_created.get() > before); + assert_eq!(rejected.get(), 1); + // Cluster label itself is racy under parallel tests (first init + // wins process-wide); ptr_eq above already proves idempotency. + } + + #[test] + fn topic_families_resolve_same_cell_for_same_labels() { + let families = &get().topics; + let a = families + .in_messages + .with_label_values(&["c", "public/default", "t", "-1"]); + let b = families + .in_messages + .with_label_values(&["c", "public/default", "t", "-1"]); + a.inc(); + assert_eq!(a.get(), b.get()); + } +} diff --git a/rust/metrics/src/lib.rs b/rust/metrics/src/lib.rs new file mode 100644 index 0000000..6adb1c5 --- /dev/null +++ b/rust/metrics/src/lib.rs @@ -0,0 +1,92 @@ +//! Pulsar Lite metrics crate. +//! +//! One home for every Prometheus family this broker exports: +//! +//! - [`global_registry`] — the process-wide registry served on +//! `GET /metrics` (via `prometheus-hyper` in the broker binary); +//! - [`storage`] — families observed from the managed-ledger write path +//! (`pulsar_storage_write_latency`, `pulsar_entry_size`, +//! write-queue batch metrics); +//! - [`broker`] — broker-scoped families and the topic/subscription +//! label families (`pulsar_broker_*`, `pulsar_in/out_*`, +//! `pulsar_subscription_*`); +//! - [`topic`] / [`subscription`] — pre-resolved per-entity handles whose +//! label lookup happens exactly once at entity creation; +//! - [`observer`] — the `PublishCommitObserver` hook the RocksDB +//! write-queue worker invokes per committed batch. +//! +//! Naming contract: families reproducing native Pulsar semantics keep the +//! exact `pulsar_*` names and label sets; extensions use `pulsar_lite_*`. +//! +//! Hot paths only ever touch pre-resolved handles (plain atomic adds). +//! [`init`] registers everything once; before it runs, accessors return +//! no-op handles so unit tests never panic. + +use std::sync::{Arc, LazyLock}; + +use prometheus::Registry; + +pub mod broker; +pub mod observer; +pub mod storage; +pub mod subscription; +pub mod topic; + +pub use broker::{get, BrokerMetrics}; +pub use observer::PublishCommitObserver; +pub use storage::{storage_metrics, StorageMetrics, StorageMetricsHandle}; +pub use subscription::SubscriptionMetrics; +pub use topic::{parse_topic_labels, TopicLabels, TopicMetrics}; + +static GLOBAL: LazyLock> = LazyLock::new(|| { + let registry = Registry::new(); + // The process collector (RSS / CPU gauges) is registered best-effort: a + // missing /proc mount only drops those gauges, never breaks collection. + match registry.register(Box::new( + prometheus::process_collector::ProcessCollector::for_self(), + )) { + Ok(()) => {} + Err(error) => log::warn!("Failed to register process collector: {}", error), + } + Arc::new(registry) +}); + +/// Returns the process-wide metrics registry, creating it on first use. +pub fn global_registry() -> Arc { + GLOBAL.clone() +} + +/// Registers broker families (idempotent) and storage families. +/// +/// `broker::init` is separately idempotent so it can also serve as the +/// lazy entry point for embedders that never call this function. +pub fn init(cluster: &str) -> Arc { + let metrics = broker::init(cluster); + let _ = storage::init(cluster); + metrics +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn global_registry_is_stable_across_calls() { + assert!(Arc::ptr_eq(&global_registry(), &global_registry())); + } + + #[test] + fn init_registers_all_families_once() { + let metrics = init("crate-test-cluster"); + // Unique labels: the registry is process-global and sibling tests + // also bump shared counters. + let cell = metrics + .topics + .in_messages + .with_label_values(&["c", "ns", "lib-init-check", "-1"]); + cell.inc(); + let storage = storage::storage_metrics(); + storage.observe_entry_size(10.0); + assert_eq!(cell.get(), 1); + } +} diff --git a/rust/metrics/src/observer.rs b/rust/metrics/src/observer.rs new file mode 100644 index 0000000..84fe76f --- /dev/null +++ b/rust/metrics/src/observer.rs @@ -0,0 +1,16 @@ +//! Publish accounting hook implemented by the broker. + +use std::sync::Arc; + +/// Per-topic publish accounting hook implemented by the broker. +/// +/// The write-queue worker calls it once per successfully committed group +/// with the group's message and byte totals, so broker counters fold whole +/// batches into single atomic updates while storage crates stay free of +/// any metrics-family knowledge. +pub trait PublishCommitObserver: Send + Sync { + fn on_commit(&self, messages: u64, bytes: u64); +} + +/// Convenience alias for pre-resolved observer handles carried per request. +pub type SharedObserver = Arc; diff --git a/rust/metrics/src/storage.rs b/rust/metrics/src/storage.rs new file mode 100644 index 0000000..169d512 --- /dev/null +++ b/rust/metrics/src/storage.rs @@ -0,0 +1,228 @@ +//! Storage-side metric families observed from the managed-ledger write path. + +use std::sync::{Arc, LazyLock, OnceLock}; + +use prometheus::{Histogram, HistogramOpts, IntCounter, Registry}; + +use crate::global_registry; + +/// Native Pulsar's addEntry latency bucket layout, in seconds +/// ({0.5, 1, 5, 10, 20, 50, 100, 200, 1000} ms). +const WRITE_LATENCY_BUCKETS: &[f64] = &[ + 0.0005, 0.001, 0.005, 0.01, 0.02, 0.05, 0.1, 0.2, 1.0, +]; + +/// Native Pulsar's entry-size bucket layout, in bytes +/// ({128, 512, 1K, 2K, 4K, 16K, 100K, 1M}). +const ENTRY_SIZE_BUCKETS: &[f64] = &[ + 128.0, 512.0, 1024.0, 2048.0, 4096.0, 16384.0, 102400.0, 1048576.0, +]; + +/// Write-queue batch sizes (MAX_BATCH today is 64). +const BATCH_SIZE_BUCKETS: &[f64] = &[1.0, 2.0, 4.0, 8.0, 16.0, 32.0, 64.0]; + +/// Storage-side histogram/counter families. +/// +/// `pulsar_storage_write_latency` is the end-to-end durable-publish view +/// (enqueue → committed batch), the equivalent of native addEntry latency +/// including queue wait. `pulsar_storage_ledger_write_latency` covers only +/// the ledger append (native bookie-write view). Write-queue families +/// replace the removed 1 Hz summary log. +#[derive(Debug)] +pub struct StorageMetrics { + write_latency: Histogram, + ledger_write_latency: Histogram, + entry_size: Histogram, + wq_batches: IntCounter, + wq_batch_messages: IntCounter, + wq_batch_size: Histogram, +} + +static STORAGE_METRICS: OnceLock> = OnceLock::new(); + +impl StorageMetrics { + fn new(cluster: &str, registry: &Registry) -> Result { + let write_latency = Histogram::with_opts( + HistogramOpts::new( + "pulsar_storage_write_latency", + "End-to-end durable publish latency (enqueue to committed batch)", + ) + .buckets(WRITE_LATENCY_BUCKETS.to_vec()), + )?; + let ledger_write_latency = Histogram::with_opts( + HistogramOpts::new( + "pulsar_storage_ledger_write_latency", + "Managed-ledger append latency per committed group", + ) + .buckets(WRITE_LATENCY_BUCKETS.to_vec()), + )?; + let entry_size = Histogram::with_opts( + HistogramOpts::new("pulsar_entry_size", "Accepted entry size in bytes") + .buckets(ENTRY_SIZE_BUCKETS.to_vec()), + )?; + let wq_batches = IntCounter::new( + "pulsar_lite_write_queue_batches_total", + "Write-queue batches drained", + )?; + let wq_batch_messages = IntCounter::new( + "pulsar_lite_write_queue_batch_messages_total", + "Messages passed through the write queue", + )?; + let wq_batch_size = Histogram::with_opts( + HistogramOpts::new( + "pulsar_lite_write_queue_batch_size", + "Write-queue batch size in messages", + ) + .buckets(BATCH_SIZE_BUCKETS.to_vec()), + )?; + + registry.register(Box::new(write_latency.clone()))?; + registry.register(Box::new(ledger_write_latency.clone()))?; + registry.register(Box::new(entry_size.clone()))?; + registry.register(Box::new(wq_batches.clone()))?; + registry.register(Box::new(wq_batch_messages.clone()))?; + registry.register(Box::new(wq_batch_size.clone()))?; + let _ = cluster; // families are broker-global; reserved for future labels + + Ok(Self { + write_latency, + ledger_write_latency, + entry_size, + wq_batches, + wq_batch_messages, + wq_batch_size, + }) + } +} + +impl StorageMetrics { + /// End-to-end durable-publish latency (seconds). + pub fn observe_write_latency(&self, seconds: f64) { + self.write_latency.observe(seconds); + } + + /// Ledger-append latency (seconds). + pub fn observe_ledger_write_latency(&self, seconds: f64) { + self.ledger_write_latency.observe(seconds); + } + + /// Accepted entry size (metadata + payload bytes). + pub fn observe_entry_size(&self, bytes: f64) { + self.entry_size.observe(bytes); + } + + /// One drained write-queue batch of `messages` entries. + pub fn observe_batch(&self, messages: u64) { + self.wq_batches.inc(); + self.wq_batch_messages.inc_by(messages); + self.wq_batch_size.observe(messages as f64); + } +} + +/// A no-op stand-in used before [`init`]; keeps call sites branch-free. +#[derive(Debug)] +pub struct DisabledStorageMetrics; + +impl DisabledStorageMetrics { + fn observe_write_latency(&self, _seconds: f64) {} + fn observe_ledger_write_latency(&self, _seconds: f64) {} + fn observe_entry_size(&self, _bytes: f64) {} + fn observe_batch(&self, _messages: u64) {} +} + +/// Unified accessor so workers can record without checking initialization. +#[derive(Debug, Clone)] +pub enum StorageMetricsHandle { + Enabled(Arc), + Disabled(Arc), +} + +impl StorageMetricsHandle { + pub fn observe_write_latency(&self, seconds: f64) { + match self { + Self::Enabled(metrics) => metrics.observe_write_latency(seconds), + Self::Disabled(metrics) => metrics.observe_write_latency(seconds), + } + } + + pub fn observe_ledger_write_latency(&self, seconds: f64) { + match self { + Self::Enabled(metrics) => metrics.observe_ledger_write_latency(seconds), + Self::Disabled(metrics) => metrics.observe_ledger_write_latency(seconds), + } + } + + pub fn observe_entry_size(&self, bytes: f64) { + match self { + Self::Enabled(metrics) => metrics.observe_entry_size(bytes), + Self::Disabled(metrics) => metrics.observe_entry_size(bytes), + } + } + + pub fn observe_batch(&self, messages: u64) { + match self { + Self::Enabled(metrics) => metrics.observe_batch(messages), + Self::Disabled(metrics) => metrics.observe_batch(messages), + } + } +} + +static HANDLE: LazyLock = LazyLock::new(|| { + match STORAGE_METRICS.get() { + Some(metrics) => StorageMetricsHandle::Enabled(Arc::clone(metrics)), + None => StorageMetricsHandle::Disabled(Arc::new(DisabledStorageMetrics)), + } +}); + +/// Returns the storage metrics handle (no-op before [`init`]). +pub fn storage_metrics() -> StorageMetricsHandle { + HANDLE.clone() +} + +/// Registers storage metric families into the shared global registry. +/// +/// Idempotent: the first call wins. Called by broker startup; unit tests +/// that never call it get no-op handles. +pub fn init(cluster: &str) -> Arc { + STORAGE_METRICS + .get_or_init(|| match StorageMetrics::new(cluster, &global_registry()) { + Ok(metrics) => Arc::new(metrics), + Err(error) => { + log::error!("Failed to register storage metrics: {}", error); + // Names are static and test-proven; this branch is unreachable + // unless the definitions themselves are invalid. + Arc::new( + StorageMetrics::new(cluster, &Registry::new()) + .expect("static storage metric definitions are valid"), + ) + } + }) + .clone() +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn global_registry_is_stable_across_calls() { + assert!(Arc::ptr_eq(&global_registry(), &global_registry())); + } + + #[test] + fn handle_before_init_is_noop() { + // Do not call init(): the disabled handle must swallow observations. + let handle = storage_metrics(); + handle.observe_write_latency(0.1); + handle.observe_ledger_write_latency(0.2); + handle.observe_entry_size(42.0); + handle.observe_batch(7); + } + + #[test] + fn init_registers_enabled_handle() { + let metrics = init("test-cluster"); + metrics.observe_write_latency(0.001); + assert_eq!(metrics.write_latency.get_sample_count(), 1); + } +} diff --git a/rust/metrics/src/subscription.rs b/rust/metrics/src/subscription.rs new file mode 100644 index 0000000..307a5de --- /dev/null +++ b/rust/metrics/src/subscription.rs @@ -0,0 +1,224 @@ +/* + * Subscription-level metric handles + * + * One `SubscriptionMetrics` per Subscription entity, resolved at creation + * from the global families (labels `{cluster, namespace, topic, partition, + * subscription}`). Hot paths touch pre-resolved atomic handles only; + * gauges are set by the scrape aggregation loop. + */ + +use std::collections::VecDeque; +use std::sync::{Arc, Mutex}; +use std::time::{Instant, SystemTime, UNIX_EPOCH}; + +use prometheus::{Gauge, IntCounter, IntGauge}; + +use crate::broker::BrokerMetrics; +use crate::topic::parse_topic_labels; + +/// (timestamp, out_messages, out_bytes, acked, redelivered, dropped). +type RateSample = (Instant, u64, u64, u64, u64, u64); +type RateSampleWindow = VecDeque; + +/// Pre-resolved per-subscription handles. +pub struct SubscriptionMetrics { + out_messages: IntCounter, + out_bytes: IntCounter, + redelivered: IntCounter, + dropped: IntCounter, + acked: IntCounter, + msg_rate_out: Gauge, + msg_throughput_out: Gauge, + msg_ack_rate: Gauge, + msg_rate_redeliver: Gauge, + msg_drop_rate: Gauge, + back_log: IntGauge, + consumers_count: IntGauge, + unacked_messages: IntGauge, + blocked_on_unacked: IntGauge, + last_acked_timestamp: Gauge, + last_consumed_timestamp: Gauge, + broker: Arc, + /// Samples for derived rates; touched only by the scrape task. Seeded + /// with a zero baseline. + rate_samples: Mutex, +} + +impl SubscriptionMetrics { + /// Resolves all handles for `(topic_name, subscription_name)`. + pub fn new(topic_name: &str, subscription_name: &str) -> Self { + let broker = crate::get(); + let labels = parse_topic_labels(topic_name); + let cluster = broker.cluster(); + let partition = labels.partition.to_string(); + let values: Vec<&str> = vec![ + cluster, + labels.namespace.as_str(), + labels.topic.as_str(), + partition.as_str(), + subscription_name, + ]; + + let families = &broker.subscriptions; + let mut samples: RateSampleWindow = VecDeque::new(); + samples.push_back((Instant::now(), 0, 0, 0, 0, 0)); + Self { + out_messages: families.out_messages.with_label_values(&values), + out_bytes: families.out_bytes.with_label_values(&values), + redelivered: families.redelivered.with_label_values(&values), + dropped: families.dropped.with_label_values(&values), + acked: families.acked.with_label_values(&values), + msg_rate_out: families.msg_rate_out.with_label_values(&values), + msg_throughput_out: families.msg_throughput_out.with_label_values(&values), + msg_ack_rate: families.msg_ack_rate.with_label_values(&values), + msg_rate_redeliver: families.msg_rate_redeliver.with_label_values(&values), + msg_drop_rate: families.msg_drop_rate.with_label_values(&values), + back_log: families.back_log.with_label_values(&values), + consumers_count: families.consumers_count.with_label_values(&values), + unacked_messages: families.unacked_messages.with_label_values(&values), + blocked_on_unacked: families.blocked_on_unacked.with_label_values(&values), + last_acked_timestamp: families.last_acked_timestamp.with_label_values(&values), + last_consumed_timestamp: families.last_consumed_timestamp.with_label_values(&values), + broker, + rate_samples: Mutex::new(samples), + } + } + + /// Records dispatched messages (batch-aware count) and their bytes. + pub fn record_dispatched(&self, messages: u64, bytes: u64) { + self.out_messages.inc_by(messages); + self.out_bytes.inc_by(bytes); + self.broker.broker_out_messages.inc_by(messages); + self.broker.broker_out_bytes.inc_by(bytes); + self.last_consumed_timestamp.set(unix_epoch_seconds()); + } + + /// Records an acknowledged message. + pub fn record_acked(&self) { + self.acked.inc(); + self.last_acked_timestamp.set(unix_epoch_seconds()); + } + + /// Records messages queued for redelivery. + pub fn record_redelivered(&self, messages: u64) { + self.redelivered.inc_by(messages); + } + + /// Records a non-persistent message dropped (no writable consumer). + pub fn record_dropped(&self) { + self.dropped.inc(); + } + + /// Records `count` non-persistent drops (batch drop accounting). + pub fn record_dropped_n(&self, count: u64) { + self.dropped.inc_by(count); + } + + /// Scrape task: refreshes derived rate gauges over `window_secs`. + /// Returns (out_messages, out_bytes) deltas for broker-level folding. + pub fn update_rates(&self, window_secs: u64) -> (u64, u64) { + let now = Instant::now(); + let window = std::time::Duration::from_secs(window_secs.max(1)); + let current = ( + self.out_messages.get(), + self.out_bytes.get(), + self.acked.get(), + self.redelivered.get(), + self.dropped.get(), + ); + + let mut samples = match self.rate_samples.lock() { + Ok(guard) => guard, + Err(poisoned) => poisoned.into_inner(), + }; + samples.push_back((now, current.0, current.1, current.2, current.3, current.4)); + while let Some((t, ..)) = samples.front() { + if now.duration_since(*t) > window { + samples.pop_front(); + } else { + break; + } + } + + let deltas = match (samples.front(), samples.back()) { + (Some((t0, m0, b0, a0, r0, d0)), Some((t1, m1, b1, a1, r1, d1))) => { + let elapsed = t1.duration_since(*t0).as_secs_f64(); + ( + m1.saturating_sub(*m0), + b1.saturating_sub(*b0), + a1.saturating_sub(*a0), + r1.saturating_sub(*r0), + d1.saturating_sub(*d0), + elapsed, + ) + } + _ => (0, 0, 0, 0, 0, 0.0), + }; + let (d_out, d_bytes, d_acked, d_redelivered, d_dropped, elapsed) = deltas; + + if elapsed >= 1.0 { + self.msg_rate_out.set(d_out as f64 / elapsed); + self.msg_throughput_out.set(d_bytes as f64 / elapsed); + self.msg_ack_rate.set(d_acked as f64 / elapsed); + self.msg_rate_redeliver.set(d_redelivered as f64 / elapsed); + self.msg_drop_rate.set(d_dropped as f64 / elapsed); + } + (d_out, d_bytes) + } + + /// Scrape task: state gauges for this subscription. `back_log: None` + /// means "unknown this round" (busy storage lock) and keeps the previous + /// value instead of reporting a spurious zero. + pub fn set_state( + &self, + back_log: Option, + unacked: i64, + blocked: bool, + consumers: i64, + ) { + if let Some(back_log) = back_log { + self.back_log.set(back_log); + } + self.unacked_messages.set(unacked); + self.blocked_on_unacked.set(blocked as i64); + self.consumers_count.set(consumers); + } +} + +impl std::fmt::Debug for SubscriptionMetrics { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("SubscriptionMetrics") + .finish_non_exhaustive() + } +} + +fn unix_epoch_seconds() -> f64 { + SystemTime::now() + .duration_since(UNIX_EPOCH) + .map(|d| d.as_secs_f64()) + .unwrap_or(0.0) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn dispatched_acked_and_liveness_tick() { + let metrics = + SubscriptionMetrics::new("persistent://public/subns/sub-topic", "sub-unit-test"); + metrics.record_dispatched(1, 120); + metrics.record_dispatched(1, 30); + metrics.record_acked(); + metrics.record_redelivered(2); + metrics.record_dropped(); + + let (d_out, d_bytes) = metrics.update_rates(60); + assert_eq!(d_out, 2); + assert_eq!(d_bytes, 150); + + metrics.set_state(Some(3), 1, true, 2); + // State gauges are read through the registry in integration checks; + // here we only assert the counters did not panic and rates reported. + } +} diff --git a/rust/metrics/src/topic.rs b/rust/metrics/src/topic.rs new file mode 100644 index 0000000..2ae7037 --- /dev/null +++ b/rust/metrics/src/topic.rs @@ -0,0 +1,260 @@ +/* + * Topic-level metric handles + * + * One `TopicMetrics` per Topic entity, created at `Topic::new` time. All + * label resolution happens exactly once here; hot paths afterwards only + * touch the resolved `IntCounter`/`IntGauge` handles (plain atomic ops). + * + * Label conventions mirror native Pulsar: `{cluster, namespace, topic, + * partition}`, with `partition="-1"` for non-partitioned topics and the + * `-partition-N` suffix stripped from the exported topic label. + */ + +use std::collections::VecDeque; +use std::sync::{Arc, Mutex}; +use std::time::Instant; + +use prometheus::{Gauge, IntCounter, IntGauge}; + +use crate::broker::BrokerMetrics; + +/// Native-style decomposition of a full topic URI. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct TopicLabels { + /// `tenant/namespace` (native namespace label format). + pub namespace: String, + /// Local topic name without domain or partition suffix. + pub topic: String, + /// Partition index, `-1` for non-partitioned topics. + pub partition: i32, +} + +/// Splits `persistent://tenant/ns/local[-partition-N]` into label values. +/// +/// Malformed names degrade to `namespace="unknown"` instead of failing: +/// metrics must never reject traffic the broker already accepted. +pub fn parse_topic_labels(name: &str) -> TopicLabels { + let body = match name.split_once("://") { + Some((_, rest)) => rest, + None => name, + }; + + let mut parts = body.splitn(3, '/'); + let tenant = parts.next().unwrap_or_default(); + let namespace_part = parts.next().unwrap_or_default(); + let local = parts.next().unwrap_or(body); + + let namespace = if tenant.is_empty() || namespace_part.is_empty() { + "unknown".to_string() + } else { + format!("{}/{}", tenant, namespace_part) + }; + + let (topic, partition) = match local.rsplit_once("-partition-") { + Some((base, digits)) + if !base.is_empty() + && !digits.is_empty() + && digits.bytes().all(|b| b.is_ascii_digit()) => + { + (base.to_string(), digits.parse::().unwrap_or(-1)) + } + _ => (local.to_string(), -1), + }; + + TopicLabels { + namespace, + topic, + partition, + } +} + +/// Pre-resolved per-topic handles (counters) + scrape-set gauges. +pub struct TopicMetrics { + in_messages: IntCounter, + in_bytes: IntCounter, + publish_rate_limit: IntCounter, + rate_in: Gauge, + throughput_in: Gauge, + average_msg_size: IntGauge, + storage_size: IntGauge, + subscriptions_count: IntGauge, + producers_count: IntGauge, + consumers_count: IntGauge, + broker: Arc, + /// (timestamp, cumulative messages, cumulative bytes) samples for the + /// scrape-derived rate gauges; only the scrape task touches this. + /// Seeded with a zero baseline so the first update already reports + /// the full window delta. + rate_samples: Mutex>, +} + +impl TopicMetrics { + /// Resolves all handles for `topic_name` from the global families. + pub fn new(topic_name: &str) -> Self { + let broker = crate::get(); + let labels = parse_topic_labels(topic_name); + let cluster = broker.cluster(); + let values = [cluster, labels.namespace.as_str(), labels.topic.as_str()]; + let partition = labels.partition.to_string(); + let mut with_partition = values.to_vec(); + with_partition.push(partition.as_str()); + let vals: Vec<&str> = with_partition; + + let families = &broker.topics; + Self { + in_messages: families.in_messages.with_label_values(&vals), + in_bytes: families.in_bytes.with_label_values(&vals), + publish_rate_limit: families.publish_rate_limit.with_label_values(&vals), + rate_in: families.rate_in.with_label_values(&vals), + throughput_in: families.throughput_in.with_label_values(&vals), + average_msg_size: families.average_msg_size.with_label_values(&vals), + storage_size: families.storage_size.with_label_values(&vals), + subscriptions_count: families.subscriptions_count.with_label_values(&vals), + producers_count: families.producers_count.with_label_values(&vals), + consumers_count: families.consumers_count.with_label_values(&vals), + broker, + rate_samples: { + let mut samples = VecDeque::new(); + samples.push_back((Instant::now(), 0, 0)); + Mutex::new(samples) + }, + } + } + + /// Records accepted publishes (messages and payload+metadata bytes). + /// + /// Called from single-writer contexts (non-persistent fan-out worker, + /// in-process publish); the durable write-queue path goes through the + /// `PublishCommitObserver` impl below instead. + pub fn record_publish(&self, messages: u64, bytes: u64) { + self.in_messages.inc_by(messages); + self.in_bytes.inc_by(bytes); + self.broker.broker_in_messages.inc_by(messages); + self.broker.broker_in_bytes.inc_by(bytes); + } + + /// Records a publish-rate-limit rejection. + pub fn record_rate_limit_reject(&self) { + self.publish_rate_limit.inc(); + } + + /// Scrape task: refreshes derived rate gauges over `window_secs`. + /// + /// Returns the window delta (messages, bytes) so the caller can fold + /// broker-level rate gauges in the same pass. + pub fn update_rates(&self, window_secs: u64) -> (u64, u64) { + let (messages, bytes) = (self.in_messages.get(), self.in_bytes.get()); + let now = Instant::now(); + let window = std::time::Duration::from_secs(window_secs.max(1)); + + let mut samples = match self.rate_samples.lock() { + Ok(guard) => guard, + Err(poisoned) => poisoned.into_inner(), + }; + samples.push_back((now, messages, bytes)); + while let Some((t, _, _)) = samples.front() { + if now.duration_since(*t) > window { + samples.pop_front(); + } else { + break; + } + } + + let (d_messages, d_bytes, elapsed) = match (samples.front(), samples.back()) { + (Some((t0, m0, b0)), Some((t1, m1, b1))) => { + let elapsed = t1.duration_since(*t0).as_secs_f64(); + (m1.saturating_sub(*m0), b1.saturating_sub(*b0), elapsed) + } + _ => (0, 0, 0.0), + }; + + if elapsed >= 1.0 { + self.rate_in.set(d_messages as f64 / elapsed); + self.throughput_in.set(d_bytes as f64 / elapsed); + self.average_msg_size + .set(d_bytes.checked_div(d_messages).unwrap_or(0) as i64); + } + (d_messages, d_bytes) + } + + /// Scrape task: entity count gauges for this topic. + pub fn set_entity_counts(&self, subscriptions: i64, producers: i64, consumers: i64) { + self.subscriptions_count.set(subscriptions); + self.producers_count.set(producers); + self.consumers_count.set(consumers); + } + + /// Scrape task: stored bytes for this topic. + pub fn set_storage_size(&self, bytes: i64) { + self.storage_size.set(bytes); + } +} + +impl std::fmt::Debug for TopicMetrics { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("TopicMetrics").finish_non_exhaustive() + } +} +/// Durable-batch accounting for the write-queue worker (single writer). +impl crate::observer::PublishCommitObserver for TopicMetrics { + fn on_commit(&self, messages: u64, bytes: u64) { + self.in_messages.inc_by(messages); + self.in_bytes.inc_by(bytes); + self.broker.broker_in_messages.inc_by(messages); + self.broker.broker_in_bytes.inc_by(bytes); + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn parses_persistent_topic_uri() { + let labels = parse_topic_labels("persistent://public/default/orders"); + assert_eq!(labels.namespace, "public/default"); + assert_eq!(labels.topic, "orders"); + assert_eq!(labels.partition, -1); + } + + #[test] + fn parses_partition_suffix() { + let labels = parse_topic_labels("non-persistent://tenant/ns/queue-partition-7"); + assert_eq!(labels.namespace, "tenant/ns"); + assert_eq!(labels.topic, "queue"); + assert_eq!(labels.partition, 7); + } + + #[test] + fn malformed_names_degrade_to_unknown_namespace() { + let labels = parse_topic_labels("bare-topic"); + assert_eq!(labels.namespace, "unknown"); + assert_eq!(labels.topic, "bare-topic"); + assert_eq!(labels.partition, -1); + } + + #[test] + fn non_numeric_partition_suffix_is_kept_in_topic() { + let labels = parse_topic_labels("persistent://t/n/v2-partition-x"); + assert_eq!(labels.topic, "v2-partition-x"); + assert_eq!(labels.partition, -1); + } + + #[test] + fn topic_metrics_counters_and_rates_tick() { + let metrics = TopicMetrics::new("persistent://public/tickns/metrics-tick"); + metrics.record_publish(3, 300); + metrics.record_rate_limit_reject(); + + // First update reports the full delta from the zero baseline + // (the elapsed guard only suppresses the rate gauges, not the + // returned delta used for broker-level folding). + let (d_msgs, d_bytes) = metrics.update_rates(60); + assert_eq!(d_msgs, 3); + assert_eq!(d_bytes, 300); + + let labels = parse_topic_labels("persistent://public/tickns/metrics-tick"); + assert_eq!(labels.namespace, "public/tickns"); + assert_eq!(labels.topic, "metrics-tick"); + } +} diff --git a/rust/proto/Cargo.toml b/rust/proto/Cargo.toml new file mode 100644 index 0000000..7e68933 --- /dev/null +++ b/rust/proto/Cargo.toml @@ -0,0 +1,14 @@ +[package] +name = "pulsar-lite-proto" +version = "0.1.0" +edition = "2021" + +[dependencies] +# Protobuf +prost = "0.13" +prost-types = "0.13" +bytes = "1.5" +tokio-util = { version = "0.7", features = ["codec"] } + +[build-dependencies] +prost-build = "0.13" \ No newline at end of file diff --git a/rust/build.rs b/rust/proto/build.rs similarity index 70% rename from rust/build.rs rename to rust/proto/build.rs index f1029b9..4905ff7 100644 --- a/rust/build.rs +++ b/rust/proto/build.rs @@ -4,10 +4,7 @@ fn main() -> Result<(), Box> { prost_build::Config::new() .out_dir(&out_dir) .protoc_arg("--experimental_allow_proto3_optional") - .compile_protos( - &["proto/PulsarApi.proto", "proto/MLDataFormats.proto"], - &["proto"], - )?; + .compile_protos(&["PulsarApi.proto", "MLDataFormats.proto"], &["proto"])?; Ok(()) } diff --git a/rust/src/protocol/codec.rs b/rust/proto/src/codec.rs similarity index 95% rename from rust/src/protocol/codec.rs rename to rust/proto/src/codec.rs index 0b8fb9c..42be572 100644 --- a/rust/src/protocol/codec.rs +++ b/rust/proto/src/codec.rs @@ -21,7 +21,6 @@ pub mod proto { // Import ServerCommand from command module use super::command::ServerCommand; -use crate::broker::service::PendingMessage; /// Magic number for checksum verification (0x0e01) const MAGIC_NUMBER: u16 = 0x0e01; @@ -305,31 +304,9 @@ impl Encoder for PulsarFrameCodec { } } -impl Encoder<(u64, PendingMessage)> for PulsarFrameCodec { - type Error = io::Error; - - fn encode( - &mut self, - item: (u64, PendingMessage), - dst: &mut BytesMut, - ) -> Result<(), Self::Error> { - let (consumer_id, msg) = item; - self.encode_message( - consumer_id, - msg.message_id.ledger, - msg.message_id.entry, - msg.message_id.partition, - &msg.metadata, - &msg.payload, - msg.redelivery_count, - dst, - ) - } -} - impl PulsarFrameCodec { /// Encode a Message command with payload - fn encode_message( + pub fn encode_message( &self, consumer_id: u64, ledger_id: u64, @@ -358,11 +335,24 @@ impl PulsarFrameCodec { } } +/// Number of client-visible messages carried by one entry's metadata. +/// Client-visible semantics (Apache Pulsar): a batch entry of N messages +/// counts as N messages for permit, rate, and counter accounting. Entries +/// without batch metadata (or undecodable metadata) count as one message. +pub fn messages_in_batch(metadata: &[u8]) -> u32 { + proto::pulsar::MessageMetadata::decode(metadata) + .ok() + .and_then(|m| m.num_messages_in_batch) + .filter(|n| *n > 0) + .map(|n| n as u32) + .unwrap_or(1) +} + #[cfg(test)] mod tests { - use super::proto::pulsar::{BaseCommand, CompressionType, KeyValue, MessageMetadata}; use super::*; - use crate::protocol::command::ServerCommand; + use crate::codec::proto::pulsar::{BaseCommand, CompressionType, KeyValue, MessageMetadata}; + use crate::command::ServerCommand; use bytes::Bytes; use prost::Message; diff --git a/rust/src/protocol/command.rs b/rust/proto/src/command.rs similarity index 100% rename from rust/src/protocol/command.rs rename to rust/proto/src/command.rs diff --git a/rust/src/protocol/mod.rs b/rust/proto/src/lib.rs similarity index 100% rename from rust/src/protocol/mod.rs rename to rust/proto/src/lib.rs diff --git a/rust/pulsar-lite.toml b/rust/pulsar-lite.toml index eddce3f..1dde7e4 100644 --- a/rust/pulsar-lite.toml +++ b/rust/pulsar-lite.toml @@ -26,3 +26,12 @@ connection_liveness_check_timeout_secs = 10 # Connection limits (0 = unlimited) max_connections = 0 max_connections_per_ip = 0 + +# Prometheus metrics endpoint (GET /metrics) +[metrics] +enabled = true +addr = "0.0.0.0:8080" +# Value of the `cluster` label on every exported family +cluster = "pulsar-lite" +# Window (seconds) for derived rate gauges (pulsar_rate_in etc.) +rate_window_secs = 60 diff --git a/rust/src/broker/broker_service.rs b/rust/src/broker/broker_service.rs index b4dcb97..b0d8633 100644 --- a/rust/src/broker/broker_service.rs +++ b/rust/src/broker/broker_service.rs @@ -8,7 +8,7 @@ use crate::broker::service::topic::{ PartitionedTopic, PartitionedTopicStats, SharedPartitionedTopic, Topic, TopicPublishRate, TopicStats, }; -use crate::storage::Storage; +use pulsar_lite_storage::Storage; use std::collections::HashMap; use std::sync::Arc; use tokio::sync::{Mutex, RwLock}; @@ -169,8 +169,6 @@ impl BrokerService { self.topics.len() } - // ==================== Partitioned Topic Management ==================== - /// Get or create a partitioned topic with the specified number of partitions /// /// This will create the partitioned topic if it doesn't exist diff --git a/rust/src/broker/connection_limiter.rs b/rust/src/broker/connection_limiter.rs index 93b1066..70966cc 100644 --- a/rust/src/broker/connection_limiter.rs +++ b/rust/src/broker/connection_limiter.rs @@ -63,6 +63,15 @@ impl ConnectionLimiter { }) } + /// Current number of tracked connections (for the + /// `pulsar_active_connections` gauge). + pub fn active_connections(&self) -> usize { + match self.inner.lock() { + Ok(state) => state.total_connections, + Err(poisoned) => poisoned.into_inner().total_connections, + } + } + fn release(&self, ip: IpAddr) { let mut state = self.inner.lock().unwrap_or_else(|e| e.into_inner()); diff --git a/rust/src/broker/dispatcher/enums.rs b/rust/src/broker/dispatcher/enums.rs index d43d0b6..d8dc3c1 100644 --- a/rust/src/broker/dispatcher/enums.rs +++ b/rust/src/broker/dispatcher/enums.rs @@ -8,7 +8,7 @@ use super::{ExclusiveDispatcher, FailoverDispatcher, KeySharedDispatcher, Shared use crate::broker::dispatcher::redelivery_controller::RedeliveryEntry; use crate::broker::service::topic::{KeySharedPolicy, SubscriptionType}; use crate::broker::service::{Consumer, SharedStorage}; -use crate::storage::{ManagedLedgerPosition, MessageId}; +use pulsar_lite_storage_managed_ledger::{ManagedLedgerPosition, MessageId}; use std::sync::Arc; /// Dispatcher enum - holds the concrete dispatcher implementation diff --git a/rust/src/broker/dispatcher/exclusive.rs b/rust/src/broker/dispatcher/exclusive.rs index de90be9..2ea7dd1 100644 --- a/rust/src/broker/dispatcher/exclusive.rs +++ b/rust/src/broker/dispatcher/exclusive.rs @@ -10,7 +10,7 @@ use super::read_position::{commit_read_position, next_unacked_candidate}; use crate::broker::dispatcher::Dispatcher; use crate::broker::service::topic::SubscriptionType; use crate::broker::service::{Consumer, SharedStorage}; -use crate::storage::ManagedLedgerPosition; +use pulsar_lite_storage_managed_ledger::ManagedLedgerPosition; use std::sync::atomic::{AtomicU32, Ordering}; use std::sync::{Arc, RwLock}; @@ -138,6 +138,8 @@ impl Dispatcher for ExclusiveDispatcher { .await?; if let Some(candidate) = candidate { + let batch_count = + crate::broker::dispatcher::messages_in_batch(&candidate.metadata); if consumer .enqueue_message( candidate.message_id, @@ -148,7 +150,7 @@ impl Dispatcher for ExclusiveDispatcher { { commit_read_position(&self.read_position, candidate.next_position); consumer - .record_message_dispatched(candidate.payload.len()) + .record_message_dispatched(batch_count, candidate.payload.len()) .await; dispatched += 1; } else { diff --git a/rust/src/broker/dispatcher/failover.rs b/rust/src/broker/dispatcher/failover.rs index 25b8c3e..86c7930 100644 --- a/rust/src/broker/dispatcher/failover.rs +++ b/rust/src/broker/dispatcher/failover.rs @@ -10,7 +10,7 @@ use super::rewind_read_position; use crate::broker::dispatcher::Dispatcher; use crate::broker::service::topic::SubscriptionType; use crate::broker::service::{Consumer, SharedStorage}; -use crate::storage::ManagedLedgerPosition; +use pulsar_lite_storage_managed_ledger::ManagedLedgerPosition; use std::sync::atomic::{AtomicU32, Ordering}; use std::sync::{Arc, RwLock}; @@ -274,6 +274,8 @@ impl Dispatcher for FailoverDispatcher { .await?; if let Some(candidate) = candidate { + let batch_count = + crate::broker::dispatcher::messages_in_batch(&candidate.metadata); if primary_consumer .enqueue_message( candidate.message_id, @@ -284,7 +286,7 @@ impl Dispatcher for FailoverDispatcher { { commit_read_position(&self.read_position, candidate.next_position); primary_consumer - .record_message_dispatched(candidate.payload.len()) + .record_message_dispatched(batch_count, candidate.payload.len()) .await; dispatched += 1; } else { @@ -325,7 +327,7 @@ impl Dispatcher for FailoverDispatcher { mod tests { use super::*; use crate::broker::service::topic::Subscription; - use crate::storage::Storage; + use pulsar_lite_storage::Storage; use std::path::Path; use tokio::sync::{mpsc, Mutex, RwLock}; diff --git a/rust/src/broker/dispatcher/key_shared.rs b/rust/src/broker/dispatcher/key_shared.rs index 58f7a4e..ab402f7 100644 --- a/rust/src/broker/dispatcher/key_shared.rs +++ b/rust/src/broker/dispatcher/key_shared.rs @@ -11,16 +11,16 @@ use crate::broker::service::topic::{ KeySharedHashRange, KeySharedMode, KeySharedPolicy, SubscriptionType, }; use crate::broker::service::{Consumer, SharedStorage}; -use crate::storage::{ManagedLedgerPosition, MessageId}; +use pulsar_lite_storage_managed_ledger::{ManagedLedgerPosition, MessageId}; use std::collections::HashMap; -use std::sync::atomic::{AtomicU32, Ordering}; +use std::sync::atomic::{AtomicI32, Ordering}; use std::sync::{Arc, RwLock}; const DISPATCHER_MAX_ROUND_ROBIN_BATCH_SIZE: u32 = 20; type DispatchableRedelivery = Option<( RedeliveryEntry, - crate::storage::StoredMessage, + pulsar_lite_storage_managed_ledger::StoredMessage, Arc, )>; @@ -29,7 +29,7 @@ pub struct KeySharedDispatcher { auto_split_assignments: Vec<(KeySharedHashRange, Arc)>, sticky_assignments: Vec<(KeySharedHashRange, Arc)>, key_shared_policy: KeySharedPolicy, - total_available_permits: AtomicU32, + total_available_permits: AtomicI32, read_position: RwLock>, redelivery_controller: RwLock, } @@ -47,7 +47,7 @@ impl KeySharedDispatcher { auto_split_assignments: Vec::new(), sticky_assignments: Vec::new(), key_shared_policy, - total_available_permits: AtomicU32::new(0), + total_available_permits: AtomicI32::new(0), read_position: RwLock::new(None), redelivery_controller: RwLock::new(RedeliveryController::new(block_hashes)), } @@ -64,7 +64,7 @@ impl KeySharedDispatcher { let _ = self.total_available_permits.fetch_update( Ordering::Relaxed, Ordering::Relaxed, - |current| Some(current.saturating_sub(permits)), + |current| Some(current.saturating_sub(permits as i32)), ); } @@ -154,7 +154,7 @@ impl KeySharedDispatcher { fn pop_dispatchable_redelivery_message( &self, - storage: &crate::storage::Storage, + storage: &pulsar_lite_storage::Storage, topic: &str, subscription: &str, ) -> Result> { @@ -326,12 +326,12 @@ impl KeySharedDispatcher { let max_batch = self .total_available_permits .load(Ordering::Relaxed) - .min(DISPATCHER_MAX_ROUND_ROBIN_BATCH_SIZE); - if max_batch == 0 { + .min(DISPATCHER_MAX_ROUND_ROBIN_BATCH_SIZE as i32); + if max_batch <= 0 { return Ok(0); } - let mut remaining_dispatches = max_batch; + let mut remaining_dispatches = max_batch as u32; let mut progress = 0u32; while remaining_dispatches > 0 { @@ -344,7 +344,8 @@ impl KeySharedDispatcher { let sticky_key_hash = redelivery .sticky_key_hash .unwrap_or_else(|| sticky_key_hash_from_metadata(&entry.metadata)); - if !consumer.use_permit().await { + let batch_count = crate::broker::dispatcher::messages_in_batch(&entry.metadata); + if !consumer.try_use_permits(batch_count).await { self.restore_redelivery_message(RedeliveryEntry { sticky_key_hash: Some(sticky_key_hash), ..redelivery @@ -352,7 +353,7 @@ impl KeySharedDispatcher { break; } remaining_dispatches -= 1; - self.subtract_total_permits(1); + self.subtract_total_permits(batch_count); if consumer .send_message_with_sticky_hash( @@ -365,7 +366,7 @@ impl KeySharedDispatcher { .await { consumer - .record_message_dispatched(entry.payload.len()) + .record_message_dispatched(batch_count, entry.payload.len()) .await; progress += 1; } else { @@ -374,11 +375,11 @@ impl KeySharedDispatcher { redelivery_count: redelivery.redelivery_count + 1, sticky_key_hash: Some(sticky_key_hash), }); - consumer.add_permits(1).await; - self.total_available_permits.fetch_add(1, Ordering::Relaxed); + consumer.add_permits(batch_count).await; + self.total_available_permits + .fetch_add(batch_count as i32, Ordering::Relaxed); break; } - continue; } let Some(candidate) = self @@ -419,11 +420,12 @@ impl KeySharedDispatcher { let Some(consumer) = self.select_consumer_for_hash(sticky_key_hash) else { break; }; - if !consumer.use_permit().await { + let batch_count = crate::broker::dispatcher::messages_in_batch(&candidate.metadata); + if !consumer.try_use_permits(batch_count).await { break; } remaining_dispatches -= 1; - self.subtract_total_permits(1); + self.subtract_total_permits(batch_count); if consumer .send_message_with_sticky_hash( @@ -437,7 +439,7 @@ impl KeySharedDispatcher { { commit_read_position(&self.read_position, candidate.next_position); consumer - .record_message_dispatched(candidate.payload.len()) + .record_message_dispatched(batch_count, candidate.payload.len()) .await; progress += 1; } else { @@ -447,8 +449,9 @@ impl KeySharedDispatcher { sticky_key_hash: Some(sticky_key_hash), }); commit_read_position(&self.read_position, candidate.next_position); - consumer.add_permits(1).await; - self.total_available_permits.fetch_add(1, Ordering::Relaxed); + consumer.add_permits(batch_count).await; + self.total_available_permits + .fetch_add(batch_count as i32, Ordering::Relaxed); break; } } @@ -508,7 +511,7 @@ impl Dispatcher for KeySharedDispatcher { fn consumer_flow(&self, consumer_id: u64, additional_permits: u32) { if self.consumers_by_id.contains_key(&consumer_id) { self.total_available_permits - .fetch_add(additional_permits, Ordering::Relaxed); + .fetch_add(additional_permits as i32, Ordering::Relaxed); } } @@ -527,7 +530,7 @@ impl Dispatcher for KeySharedDispatcher { if progress == 0 { break; } - if self.total_available_permits.load(Ordering::Relaxed) == 0 { + if self.total_available_permits.load(Ordering::Relaxed) <= 0 { break; } @@ -543,10 +546,11 @@ mod tests { use super::*; use crate::broker::service::topic::{Subscription, SubscriptionRuntimeMode}; use crate::broker::service::{ConnectionWriteState, PendingMessage}; - use crate::protocol::codec::proto::pulsar::MessageMetadata; - use crate::storage::{CursorInitOptions, InitialPosition, Storage}; use bytes::Bytes; use prost::Message; + use pulsar_lite_proto::codec::proto::pulsar::MessageMetadata; + use pulsar_lite_storage::Storage; + use pulsar_lite_storage_managed_ledger::{CursorInitOptions, InitialPosition}; use std::path::Path; use std::time::Duration; use tokio::sync::{mpsc, Mutex, RwLock}; diff --git a/rust/src/broker/dispatcher/mod.rs b/rust/src/broker/dispatcher/mod.rs index e09794c..2d152f8 100644 --- a/rust/src/broker/dispatcher/mod.rs +++ b/rust/src/broker/dispatcher/mod.rs @@ -10,6 +10,7 @@ mod key_shared; mod read_position; pub mod redelivery_controller; mod shared; +pub(crate) use shared::DEFAULT_MAX_UNACKED_MESSAGES_PER_CONSUMER; mod single_active; pub(crate) mod sticky_key; mod traits; @@ -21,3 +22,7 @@ pub use key_shared::KeySharedDispatcher; pub use shared::SharedDispatcher; pub use single_active::rewind_read_position; pub use traits::Dispatcher; + +/// Client-visible message count per entry (batch-aware); shared with the +/// write-queue commit accounting. See `pulsar_lite_proto::codec`. +pub use pulsar_lite_proto::codec::messages_in_batch; diff --git a/rust/src/broker/dispatcher/read_position.rs b/rust/src/broker/dispatcher/read_position.rs index 01e4652..5d982c3 100644 --- a/rust/src/broker/dispatcher/read_position.rs +++ b/rust/src/broker/dispatcher/read_position.rs @@ -1,5 +1,5 @@ use crate::broker::service::SharedStorage; -use crate::storage::{ManagedLedgerPosition, MessageId}; +use pulsar_lite_storage_managed_ledger::{ManagedLedgerPosition, MessageId}; use std::sync::RwLock; pub type DispatchError = Box; @@ -49,6 +49,7 @@ pub async fn next_unacked_candidate( let Some((entry, next_position, already_acked)) = ({ let guard = storage.lock().await; + // TODO: let batch = guard.read_entries_from(topic, &pos, 1)?; if let Some(entry) = batch.into_iter().next() { diff --git a/rust/src/broker/dispatcher/redelivery_controller.rs b/rust/src/broker/dispatcher/redelivery_controller.rs index dc56c51..72d06bb 100644 --- a/rust/src/broker/dispatcher/redelivery_controller.rs +++ b/rust/src/broker/dispatcher/redelivery_controller.rs @@ -1,4 +1,4 @@ -use crate::storage::MessageId; +use pulsar_lite_storage_managed_ledger::MessageId; use std::collections::{BTreeMap, HashMap}; #[derive(Debug, Clone, PartialEq, Eq)] diff --git a/rust/src/broker/dispatcher/shared.rs b/rust/src/broker/dispatcher/shared.rs index 90c2b7c..f135a85 100644 --- a/rust/src/broker/dispatcher/shared.rs +++ b/rust/src/broker/dispatcher/shared.rs @@ -9,9 +9,9 @@ use super::redelivery_controller::{RedeliveryController, RedeliveryEntry}; use crate::broker::dispatcher::Dispatcher; use crate::broker::service::topic::SubscriptionType; use crate::broker::service::{Consumer, SharedStorage}; -use crate::storage::{ManagedLedgerPosition, MessageId}; +use pulsar_lite_storage_managed_ledger::{ManagedLedgerPosition, MessageId}; use std::collections::HashMap; -use std::sync::atomic::{AtomicBool, AtomicU32, AtomicUsize, Ordering}; +use std::sync::atomic::{AtomicBool, AtomicI32, AtomicUsize, Ordering}; use std::sync::{Arc, RwLock}; /// Consistent with Apache Pulsar: dispatcherMaxRoundRobinBatchSize = 20 @@ -30,12 +30,24 @@ pub struct SharedDispatcher { /// Round-Robin index for consumer selection (atomic for thread safety) round_robin_index: AtomicUsize, - /// Total available permits across all consumers (atomic for thread safety) - total_available_permits: AtomicU32, + /// Total available permits across all consumers (atomic for thread safety). + /// i32: the balance may briefly go negative when a batch entry is larger + /// than the remaining permits (Apache Pulsar semantics). + total_available_permits: AtomicI32, - /// Flag to prevent reentrant dispatching + /// Flag to prevent reentrant dispatching (Pulsar readMoreEntriesInProgress). dispatch_in_progress: AtomicBool, + /// Set by triggers that arrive while a dispatch loop is already running; + /// the loop checks this at the end of each pass and runs one more pass if + /// set (Pulsar shouldRescheduleRead). No triggers are lost and no fixed + /// time constants are involved. + should_reschedule: AtomicBool, + + /// Per-consumer delivered-but-unacked budget. Dispatch pauses when a + /// consumer exceeds it (aligns with Pulsar maxUnackedMessagesPerConsumer). + max_unacked_messages: usize, + // Pending messages to redeliver. Shared does not use sticky hash blocking. redelivery_controller: Arc>, @@ -43,6 +55,9 @@ pub struct SharedDispatcher { read_position: RwLock>, } +/// Aligns with Apache Pulsar broker.conf maxUnackedMessagesPerConsumer default. +pub(crate) const DEFAULT_MAX_UNACKED_MESSAGES_PER_CONSUMER: usize = 50_000; + impl SharedDispatcher { /// Create a new SharedDispatcher pub fn new() -> Self { @@ -50,8 +65,10 @@ impl SharedDispatcher { consumers: HashMap::new(), consumer_order: Vec::new(), round_robin_index: AtomicUsize::new(0), - total_available_permits: AtomicU32::new(0), + total_available_permits: AtomicI32::new(0), dispatch_in_progress: AtomicBool::new(false), + should_reschedule: AtomicBool::new(false), + max_unacked_messages: DEFAULT_MAX_UNACKED_MESSAGES_PER_CONSUMER, redelivery_controller: Arc::new(RwLock::new(RedeliveryController::new(false))), read_position: RwLock::new(None), } @@ -168,7 +185,7 @@ impl SharedDispatcher { let _ = self.total_available_permits.fetch_update( Ordering::Relaxed, Ordering::Relaxed, - |current| Some(current.saturating_sub(permits)), + |current| Some(current.saturating_sub(permits as i32)), ); } @@ -232,7 +249,7 @@ impl SharedDispatcher { ) -> Result> { // Check if we have permits let total_permits = self.total_available_permits.load(Ordering::Relaxed); - if total_permits == 0 { + if total_permits <= 0 { log::debug!("No permits available, skipping dispatch"); return Ok(0); } @@ -240,7 +257,8 @@ impl SharedDispatcher { // Dispatch up to DISPATCHER_MAX_ROUND_ROBIN_BATCH_SIZE messages let mut dispatched = 0u32; let mut redelivered = 0u32; - let max_batch = std::cmp::min(total_permits, DISPATCHER_MAX_ROUND_ROBIN_BATCH_SIZE); + let max_batch = + std::cmp::min(total_permits, DISPATCHER_MAX_ROUND_ROBIN_BATCH_SIZE as i32) as u32; log::debug!( "Starting batch dispatch: max_batch={}, total_permits={}, consumers={}, redelivery_queue={}", @@ -260,14 +278,28 @@ impl SharedDispatcher { let consumer_id = consumer.consumer_id; - // Use one permit - if !consumer.use_permit().await { - log::warn!("Consumer {} permits exhausted during dispatch", consumer_id); + // Unacked budget gate: pause dispatch for consumers whose + // delivered-but-unacked set exceeds the budget (Pulsar + // maxUnackedMessagesPerConsumer semantics). Without this, the + // redelivery/Flow feedback loop grows permits and reads unboundedly. + // + // TODO(dispatchers): extract this gate (plus the merge-trigger in + // dispatch_messages and the send-failure break) into a shared + // module so Exclusive/Failover/KeyShared dispatchers reuse the + // same bounded-dispatch logic instead of duplicating it. + if consumer.pending_acks_len().await >= self.max_unacked_messages { + log::warn!( + "Consumer {} unacked budget exceeded ({} >= {}), pausing dispatch", + consumer_id, + consumer.pending_acks_len().await, + self.max_unacked_messages + ); break; } - // Decrease total permits - self.total_available_permits.fetch_sub(1, Ordering::Relaxed); + // Permits are consumed per client-visible message after the entry + // is fetched (batch entries carry N messages), matching Apache + // Pulsar. Nothing is consumed here at the top of the loop. // 1. Priority: get message from redelivery queue if let Some(redelivery) = self.pop_redelivery_message() { @@ -278,8 +310,6 @@ impl SharedDispatcher { guard.is_acknowledged(&topic, &subscription, &msg_id)? }; if already_acked { - consumer.add_permits(1).await; - self.total_available_permits.fetch_add(1, Ordering::Relaxed); log::debug!( "Skipping replay for already-acked message {}:{}", msg_id.ledger, @@ -295,6 +325,21 @@ impl SharedDispatcher { }; if let Some(entry) = message_opt { + let batch_count = crate::broker::dispatcher::messages_in_batch(&entry.metadata); + if !consumer.try_use_permits(batch_count).await { + self.restore_redelivery_message(RedeliveryEntry { + message_id: entry.message_id.clone(), + redelivery_count, + sticky_key_hash: redelivery.sticky_key_hash, + }); + log::warn!( + "Consumer {} permits insufficient for batch of {} messages, waiting for flow", + consumer_id, batch_count + ); + break; + } + self.total_available_permits + .fetch_sub(batch_count as i32, Ordering::Relaxed); if consumer .send_message( entry.message_id.clone(), @@ -305,7 +350,7 @@ impl SharedDispatcher { .await { consumer - .record_message_dispatched(entry.payload.len()) + .record_message_dispatched(batch_count, entry.payload.len()) .await; dispatched += 1; redelivered += 1; @@ -323,13 +368,15 @@ impl SharedDispatcher { redelivery_count: redelivery_count + 1, sticky_key_hash: None, }); - consumer.add_permits(1).await; - self.total_available_permits.fetch_add(1, Ordering::Relaxed); + consumer.add_permits(batch_count).await; + self.total_available_permits + .fetch_add(batch_count as i32, Ordering::Relaxed); + // Send failed: connection/write path is saturated. + // Stop this pass instead of immediately re-reading the + // same entry (previous `continue` caused the spin). + break; } } else { - // Message no longer exists (may have been deleted), restore permit - consumer.add_permits(1).await; - self.total_available_permits.fetch_add(1, Ordering::Relaxed); log::warn!( "Redelivery message {}:{} not found in storage", msg_id.ledger, @@ -345,6 +392,16 @@ impl SharedDispatcher { .await?; if let Some(candidate) = message_opt { + let batch_count = crate::broker::dispatcher::messages_in_batch(&candidate.metadata); + if !consumer.try_use_permits(batch_count).await { + log::warn!( + "Consumer {} permits insufficient for batch of {} messages, waiting for flow", + consumer_id, batch_count + ); + break; + } + self.total_available_permits + .fetch_sub(batch_count as i32, Ordering::Relaxed); if consumer .send_message( candidate.message_id.clone(), @@ -356,7 +413,7 @@ impl SharedDispatcher { { commit_read_position(&self.read_position, candidate.next_position); consumer - .record_message_dispatched(candidate.payload.len()) + .record_message_dispatched(batch_count, candidate.payload.len()) .await; dispatched += 1; @@ -368,13 +425,15 @@ impl SharedDispatcher { } else { self.add_to_redelivery_queue(vec![(candidate.message_id, 1)]); commit_read_position(&self.read_position, candidate.next_position); - consumer.add_permits(1).await; - self.total_available_permits.fetch_add(1, Ordering::Relaxed); + consumer.add_permits(batch_count).await; + self.total_available_permits + .fetch_add(batch_count as i32, Ordering::Relaxed); + // Send failed: connection/write path is saturated. Stop + // this pass rather than queueing the whole batch for + // redelivery and re-reading in a tight loop. + break; } } else { - // No more messages, restore permit - consumer.add_permits(1).await; - self.total_available_permits.fetch_add(1, Ordering::Relaxed); log::debug!("No more dispatchable messages"); break; } @@ -610,7 +669,7 @@ impl Dispatcher for SharedDispatcher { // Consumer-local permit state is updated by the flow handler before it // triggers dispatch. The dispatcher only tracks the aggregate count. self.total_available_permits - .fetch_add(additional_permits, Ordering::Relaxed); + .fetch_add(additional_permits as i32, Ordering::Relaxed); log::info!( "Consumer {} flowed {} permits, total={}", @@ -626,34 +685,58 @@ impl Dispatcher for SharedDispatcher { topic: String, subscription: String, ) -> Result<(), Box> { - // Prevent reentrant dispatching + // Merge-trigger (Pulsar): only one dispatch loop runs at a time. + // Concurrent triggers set the reschedule flag instead of starting a + // new loop; the running loop checks the flag at the end of each pass + // and runs one more pass. No triggers are lost, no fixed time + // constants, and empty passes never read storage. if self.dispatch_in_progress.swap(true, Ordering::Relaxed) { - log::debug!("Dispatch already in progress, skipping"); + log::debug!("Dispatch already in progress, scheduling reschedule"); + self.should_reschedule.store(true, Ordering::Relaxed); return Ok(()); } + let result = async { - // Keep dispatching batches while we still make progress and have permits. - // read/send one batch, then readMoreEntries again loop { - let dispatcher = self - .dispatch_messages_batch(storage.clone(), topic.clone(), subscription.clone()) - .await?; - if dispatcher == 0 { - break; - } - if self.total_available_permits.load(Ordering::Relaxed) == 0 { - break; + // One pass: dispatch batches while we make progress and have permits. + loop { + let dispatcher = self + .dispatch_messages_batch( + storage.clone(), + topic.clone(), + subscription.clone(), + ) + .await?; + if dispatcher == 0 { + break; + } + if self.total_available_permits.load(Ordering::Relaxed) == 0 { + break; + } + + // yield so other tasks can run on large backlogs. + tokio::task::yield_now().await; } - // yield so other tasks can run on large backlogs. - tokio::task::yield_now().await; + // Pulsar ordering: release the loop guard, then check whether + // triggers arrived during the pass. Re-acquire for the + // follow-up pass; if another trigger already grabbed the guard, + // that loop owns the reschedule handling. + self.dispatch_in_progress.store(false, Ordering::Relaxed); + if !self.should_reschedule.swap(false, Ordering::Relaxed) { + return Ok(()); + } + if self.dispatch_in_progress.swap(true, Ordering::Relaxed) { + return Ok(()); + } } - Ok(()) } .await; - // Reset flag - self.dispatch_in_progress.store(false, Ordering::Relaxed); + if result.is_err() { + // Error propagated out of the pass loop with the guard still held. + self.dispatch_in_progress.store(false, Ordering::Relaxed); + } result } @@ -663,7 +746,8 @@ impl Dispatcher for SharedDispatcher { mod tests { use super::*; use crate::broker::service::topic::Subscription; - use crate::storage::{CursorInitOptions, InitialPosition, Storage}; + use pulsar_lite_storage::Storage; + use pulsar_lite_storage_managed_ledger::{CursorInitOptions, InitialPosition}; use std::path::Path; use tokio::sync::{mpsc, Mutex, RwLock}; @@ -698,6 +782,61 @@ mod tests { )) } + #[tokio::test] + async fn shared_dispatch_pauses_when_unacked_budget_exceeded() { + let storage = create_test_storage(); + let subscription = create_test_subscription(storage.clone()); + let (tx, mut rx) = mpsc::channel(8); + let consumer = Arc::new(Consumer::new( + 1, + "consumer-1".to_string(), + subscription, + "conn-1".to_string(), + tx, + 0, + )); + consumer.add_permits(10).await; + + let mut dispatcher = SharedDispatcher::new(); + dispatcher.max_unacked_messages = 2; + dispatcher.add_consumer(consumer.clone()).unwrap(); + dispatcher.consumer_flow(1, 10); + + // Fill the delivered-but-unacked set past the budget (3 > 2). + for i in 0..3u64 { + consumer + .track_message_dispatched( + &MessageId { + ledger: 0, + entry: i, + partition: -1, + }, + 0, + ) + .await; + } + + let result = dispatcher + .dispatch_messages( + storage.clone(), + "persistent://public/default/test-topic".to_string(), + "test-sub".to_string(), + ) + .await; + assert!( + result.is_ok(), + "dispatch should not error when budget exceeded" + ); + + // Budget exceeded -> dispatch paused: nothing should be delivered. + let delivered = + tokio::time::timeout(std::time::Duration::from_millis(200), rx.recv()).await; + assert!( + delivered.is_err(), + "dispatch must pause when unacked budget is exceeded" + ); + } + #[tokio::test] async fn priority_dispatch_prefers_higher_priority_consumers() { let storage = create_test_storage(); diff --git a/rust/src/broker/dispatcher/single_active.rs b/rust/src/broker/dispatcher/single_active.rs index c01964d..d4befbc 100644 --- a/rust/src/broker/dispatcher/single_active.rs +++ b/rust/src/broker/dispatcher/single_active.rs @@ -1,5 +1,5 @@ use crate::broker::service::SharedStorage; -use crate::storage::ManagedLedgerPosition; +use pulsar_lite_storage_managed_ledger::ManagedLedgerPosition; /// Compute the read position a SingleActive dispatcher should rewind to when /// the active consumer disconnects. diff --git a/rust/src/broker/dispatcher/sticky_key.rs b/rust/src/broker/dispatcher/sticky_key.rs index 15fe42d..edc53a7 100644 --- a/rust/src/broker/dispatcher/sticky_key.rs +++ b/rust/src/broker/dispatcher/sticky_key.rs @@ -1,5 +1,5 @@ -use crate::protocol::codec::proto::pulsar::MessageMetadata; use prost::Message; +use pulsar_lite_proto::codec::proto::pulsar::MessageMetadata; const RANGE_SIZE: u32 = 2 << 15; diff --git a/rust/src/broker/dispatcher/traits.rs b/rust/src/broker/dispatcher/traits.rs index 827baa5..e4badf0 100644 --- a/rust/src/broker/dispatcher/traits.rs +++ b/rust/src/broker/dispatcher/traits.rs @@ -6,7 +6,7 @@ use crate::broker::service::topic::SubscriptionType; use crate::broker::service::{Consumer, SharedStorage}; -use crate::storage::ManagedLedgerPosition; +use pulsar_lite_storage_managed_ledger::ManagedLedgerPosition; use std::future::Future; use std::sync::Arc; diff --git a/rust/src/broker/handler/connection_handler.rs b/rust/src/broker/handler/connection_handler.rs index 8387b45..f603192 100644 --- a/rust/src/broker/handler/connection_handler.rs +++ b/rust/src/broker/handler/connection_handler.rs @@ -3,12 +3,12 @@ * Handles connection-level commands: Connect, Ping/Pong */ -use crate::protocol::codec::{ +use futures::SinkExt; +use pulsar_lite_proto::codec::{ proto::pulsar::{BaseCommand, CommandPong}, PulsarFrameCodec, }; -use crate::protocol::ServerCommand; -use futures::SinkExt; +use pulsar_lite_proto::ServerCommand; use tokio_util::codec::Framed; /// Handle Connect command diff --git a/rust/src/broker/handler/consumer_handler.rs b/rust/src/broker/handler/consumer_handler.rs index cb6542c..e8b9a37 100644 --- a/rust/src/broker/handler/consumer_handler.rs +++ b/rust/src/broker/handler/consumer_handler.rs @@ -10,10 +10,10 @@ use crate::broker::service::topic::{ }; use crate::broker::service::ConnectionWriteState; use crate::broker::service::Consumer; -use crate::protocol::codec::{proto::pulsar::BaseCommand, PulsarFrameCodec}; -use crate::protocol::ServerCommand; -use crate::storage::{CursorInitOptions, InitialPosition, MessageId}; use futures::SinkExt; +use pulsar_lite_proto::codec::{proto::pulsar::BaseCommand, PulsarFrameCodec}; +use pulsar_lite_proto::ServerCommand; +use pulsar_lite_storage_managed_ledger::{CursorInitOptions, InitialPosition, MessageId}; use std::collections::HashMap; use std::sync::Arc; use tokio::sync::mpsc; @@ -195,7 +195,10 @@ pub async fn handle_flow( consumers: &mut HashMap>, ) -> Result<(), Box> { let flow_cmd = cmd.flow.as_ref().ok_or("Missing flow command")?; - log::info!( + // Per-Flow logging is debug-level: clients send thousands of Flow + // commands per second under load; INFO floods the journal/PTY and the + // synchronous logger write can stall runtime workers under backpressure. + log::debug!( "Handling Flow command: consumer_id={}, permits={}", flow_cmd.consumer_id, flow_cmd.message_permits @@ -207,18 +210,38 @@ pub async fn handle_flow( .ok_or_else(|| format!("Unknown consumer ID: {}", flow_cmd.consumer_id))?; // Native Pulsar updates permits on the consumer and then notifies the - // dispatcher/subscription path. Non-persistent single-active dispatchers - // read consumer-local permits directly, while shared variants also keep - // dispatcher-level aggregates. - consumer.add_permits(flow_cmd.message_permits).await; - - // Flow permits to dispatcher and trigger dispatch via Subscription + // dispatcher/subscription path. Both accounting layers (consumer-local + // and dispatcher aggregate) must be applied synchronously under the + // subscription read lock: remove_consumer_with_recovery subtracts the + // consumer-local balance from the aggregate, and non-persistent dispatch + // gates on permits from other connections, so a half-applied Flow (one + // layer updated, the other still queued in a task) either zeroed the + // aggregate on consumer removal or dropped entries a consumer actually + // had permits for. Readers share this lock; only subscribe/close writers + // can briefly delay it. let consumer_id = consumer.consumer_id; + let permits = flow_cmd.message_permits; let subscription = consumer.get_subscription(); - let sub_guard = subscription.read().await; - sub_guard - .consumer_flow(consumer_id, flow_cmd.message_permits) - .await; + let needs_dispatch_trigger = { + let sub_guard = subscription.read().await; + consumer.add_permits(permits).await; + sub_guard.apply_flow_permits(consumer_id, permits); + sub_guard.is_persistent() + }; + + // Dispatch runs in its own task: a dispatch loop can keep running while + // triggers keep arriving (should_reschedule), so awaiting it here would + // block this connection's event loop and starve reads/acks (observed as + // channel growth + OOM under 8-subscription fanout). + if needs_dispatch_trigger { + let subscription = subscription.clone(); + tokio::spawn(async move { + let sub_guard = subscription.read().await; + if let Err(e) = sub_guard.dispatch_messages().await { + log::error!("Flow-triggered dispatch failed: {}", e); + } + }); + } Ok(()) } @@ -585,13 +608,13 @@ mod tests { use crate::broker::broker_service::{BrokerService, TopicRef}; use crate::broker::service::topic::{KeySharedMode, TopicRuntimeMode}; use crate::broker::service::SharedStorage; - use crate::protocol::codec::proto::pulsar::{ + use futures::StreamExt; + use prost::Message; + use pulsar_lite_proto::codec::proto::pulsar::{ base_command, CommandAck, CommandSubscribe, IntRange, KeySharedMeta, KeyValue, MessageIdData, }; - use crate::storage::Storage; - use futures::StreamExt; - use prost::Message; + use pulsar_lite_storage::Storage; use std::time::{SystemTime, UNIX_EPOCH}; use tokio::io::duplex; use tokio::sync::{Mutex, RwLock}; diff --git a/rust/src/broker/handler/lookup_handler.rs b/rust/src/broker/handler/lookup_handler.rs index 92ce922..00ba859 100644 --- a/rust/src/broker/handler/lookup_handler.rs +++ b/rust/src/broker/handler/lookup_handler.rs @@ -4,9 +4,9 @@ */ use crate::broker::SharedBrokerService; -use crate::protocol::codec::{proto::pulsar::BaseCommand, PulsarFrameCodec}; -use crate::protocol::ServerCommand; use futures::SinkExt; +use pulsar_lite_proto::codec::{proto::pulsar::BaseCommand, PulsarFrameCodec}; +use pulsar_lite_proto::ServerCommand; use tokio_util::codec::Framed; /// Handle PartitionMetadata command @@ -85,10 +85,10 @@ where mod tests { use super::*; use crate::broker::broker_service::BrokerService; - use crate::protocol::codec::proto::pulsar::{ + use pulsar_lite_proto::codec::proto::pulsar::{ base_command, CommandLookupTopic, CommandPartitionedTopicMetadata, }; - use crate::storage::Storage; + use pulsar_lite_storage::Storage; use std::path::Path; use std::sync::Arc; use std::time::Instant; diff --git a/rust/src/broker/handler/producer_handler.rs b/rust/src/broker/handler/producer_handler.rs index 13d5bb5..b8c3b68 100644 --- a/rust/src/broker/handler/producer_handler.rs +++ b/rust/src/broker/handler/producer_handler.rs @@ -6,9 +6,10 @@ use crate::broker::broker_service::{SharedBrokerService, TopicRef}; use crate::broker::service::topic::TopicRuntimeMode; use crate::broker::service::Producer; -use crate::protocol::codec::{proto::pulsar::BaseCommand, PulsarFrame, PulsarFrameCodec}; -use crate::protocol::ServerCommand; +use crate::broker::Topic; use futures::SinkExt; +use pulsar_lite_proto::codec::{proto::pulsar::BaseCommand, PulsarFrame, PulsarFrameCodec}; +use pulsar_lite_proto::ServerCommand; use std::collections::HashMap; use std::sync::Arc; use tokio_util::codec::Framed; @@ -97,7 +98,7 @@ where pub struct PublishedSend { pub producer_id: u64, pub sequence_id: u64, - pub message_id: crate::storage::MessageId, + pub message_id: pulsar_lite_storage_managed_ledger::MessageId, } /// Publish a Send command (storage + dispatch) without writing SendReceipt. @@ -129,13 +130,11 @@ pub async fn publish_send( let message_id = if is_non_persistent { producer.record_message_sent(frame.payload.len()).await; - let publish = { - let mut topic_guard = topic.write().await; - topic_guard.prepare_non_persistent_publish(frame.metadata, frame.payload)? - }; - let message_id = publish.message_id(); - publish.dispatch_sequential().await; - message_id + // Lock-free hot path (mirror of the persistent write-queue enqueue): + // short read lock for rate check + subscription snapshot, then hand + // off to the per-topic ordered fan-out worker. The connection task is + // free to read the next frame while fan-out runs. + Topic::publish_non_persistent(&topic, frame.metadata, frame.payload).await? } else { let message_id = producer .publish_message(frame.metadata, frame.payload) @@ -150,9 +149,7 @@ pub async fn publish_send( ); { - let topic = producer.get_topic(); - let mut topic_guard = topic.write().await; - topic_guard.dispatch_to_subscriptions().await; + Topic::spawn_dispatcher(producer.get_topic()); } message_id }; @@ -181,9 +178,7 @@ pub async fn publish_persistent_send( producer.get_topic_name() ); { - let topic = producer.get_topic(); - let mut topic_guard = topic.write().await; - topic_guard.dispatch_to_subscriptions().await; + Topic::spawn_dispatcher(producer.get_topic()); } Ok(PublishedSend { producer_id, diff --git a/rust/src/broker/mod.rs b/rust/src/broker/mod.rs index 81c9183..715a49d 100644 --- a/rust/src/broker/mod.rs +++ b/rust/src/broker/mod.rs @@ -20,4 +20,4 @@ pub use service::topic::{ }; pub use service::SharedStorage; pub use service::{handle_connection, ConnectionState}; -pub use stats::{BrokerMetrics, SharedMetrics}; +pub use stats::{get, init, BrokerMetrics}; diff --git a/rust/src/broker/non_persistent/dispatcher/mod.rs b/rust/src/broker/non_persistent/dispatcher/mod.rs index 624b4d6..f2e4a5f 100644 --- a/rust/src/broker/non_persistent/dispatcher/mod.rs +++ b/rust/src/broker/non_persistent/dispatcher/mod.rs @@ -11,7 +11,7 @@ mod sticky_key; use crate::broker::service::topic::{KeySharedPolicy, SubscriptionType}; use crate::broker::service::Consumer; -use crate::storage::NonPersistentEntry; +use pulsar_lite_storage_managed_ledger::NonPersistentEntry; use std::sync::Arc; pub use multiple_consumers::NonPersistentDispatcherMultipleConsumers; diff --git a/rust/src/broker/non_persistent/dispatcher/multiple_consumers.rs b/rust/src/broker/non_persistent/dispatcher/multiple_consumers.rs index 6175e62..e8ab3d4 100644 --- a/rust/src/broker/non_persistent/dispatcher/multiple_consumers.rs +++ b/rust/src/broker/non_persistent/dispatcher/multiple_consumers.rs @@ -1,6 +1,6 @@ use crate::broker::service::topic::SubscriptionType; use crate::broker::service::Consumer; -use crate::storage::{MessageId, NonPersistentEntry}; +use pulsar_lite_storage_managed_ledger::{MessageId, NonPersistentEntry}; use std::collections::HashMap; use std::sync::{ atomic::{AtomicU32, AtomicU64, AtomicUsize, Ordering}, @@ -123,22 +123,18 @@ impl NonPersistentDispatcherMultipleConsumers { let entry_count = entries.len() as u64; self.received_messages .fetch_add(entry_count, Ordering::Relaxed); - let mut pending_entries = entries.into_iter(); - while let Some(entry) = pending_entries.next() { - if self.total_available_permits.load(Ordering::Relaxed) == 0 { - log::debug!("Dropping non-persistent shared entry due to zero aggregate permits"); - self.record_drop(1); - entry.release(); - for remaining in pending_entries.by_ref() { - self.record_drop(1); - remaining.release(); - } - continue; - } - + for entry in entries { + // No aggregate-permit gate here: the aggregate can momentarily + // read zero while a Flow's permit update is still being applied + // by its spawned task, and dropping on that stale snapshot lost + // messages a consumer actually had permits for. Per-consumer + // reservations are the authority; genuine permit exhaustion + // fails `use_permit` immediately and drops below as before. let Some(start) = self.next_consumer_start_index() else { log::debug!("Dropping non-persistent shared entry due to no connected consumer"); - self.record_drop(1); + self.record_drop( + crate::broker::dispatcher::messages_in_batch(entry.metadata()) as u64, + ); entry.release(); continue; }; @@ -151,27 +147,38 @@ impl NonPersistentDispatcherMultipleConsumers { let metadata = entry.metadata_bytes(); let payload = entry.payload_bytes(); + let batch_count = crate::broker::dispatcher::messages_in_batch(entry.metadata()); let mut delivered = false; for offset in 0..self.ordered_consumers.len() { let consumer = self.ordered_consumers[(start + offset) % self.ordered_consumers.len()].clone(); if let Some(reservation) = consumer - .try_reserve_dispatch(&message_id, metadata.clone(), payload.clone(), 0) + .try_reserve_dispatch( + &message_id, + metadata.clone(), + payload.clone(), + 0, + batch_count, + ) .await { reservation.send(); - self.record_dispatched(1); - self.subtract_total_permits(1); - consumer.record_message_dispatched(entry.len()).await; + self.record_dispatched(batch_count as u64); + self.subtract_total_permits(batch_count); + consumer + .record_message_dispatched(batch_count, entry.len()) + .await; delivered = true; break; } } if !delivered { - log::debug!("Dropping non-persistent shared entry due to no writable consumer"); - self.record_drop(1); + log::debug!( + "Dropping non-persistent shared entry: no consumer with permits or writability" + ); + self.record_drop(batch_count as u64); } entry.release(); } @@ -184,8 +191,9 @@ mod tests { use super::*; use crate::broker::service::topic::Subscription; use crate::broker::service::SharedStorage; - use crate::storage::{NonPersistentEntry, Storage}; use bytes::Bytes; + use pulsar_lite_storage::Storage; + use pulsar_lite_storage_managed_ledger::NonPersistentEntry; use std::path::Path; use std::sync::Arc; use std::time::Instant; @@ -296,12 +304,15 @@ mod tests { } #[tokio::test] - async fn shared_dispatcher_limits_batch_by_aggregate_permits() { + async fn shared_dispatcher_uses_consumer_permits_when_aggregate_stale() { let subscription = create_test_subscription(); let (consumer, mut rx) = create_test_consumer(1, subscription); let mut dispatcher = NonPersistentDispatcherMultipleConsumers::new(); dispatcher.add_consumer(consumer.clone()).unwrap(); + // Reproduce the flow-application skew: consumer-local permits + // applied, dispatcher aggregate still stale. Deliveries must follow + // the consumer-local authority, not the stale aggregate. consumer.add_permits(2).await; dispatcher.consumer_flow(consumer.consumer_id, 1); @@ -315,9 +326,11 @@ mod tests { let first = rx.recv().await.expect("first message should be delivered"); assert_eq!(first.1.payload, b"first".to_vec()); + let second = rx.recv().await.expect("second message should be delivered"); + assert_eq!(second.1.payload, b"second".to_vec()); assert!(rx.try_recv().is_err()); - assert_eq!(dispatcher.dropped_messages(), 1); - assert_eq!(consumer.get_available_permits().await, 1); + assert_eq!(dispatcher.dropped_messages(), 0); + assert_eq!(consumer.get_available_permits().await, 0); } #[tokio::test] diff --git a/rust/src/broker/non_persistent/dispatcher/single_active.rs b/rust/src/broker/non_persistent/dispatcher/single_active.rs index 452cd88..da3134a 100644 --- a/rust/src/broker/non_persistent/dispatcher/single_active.rs +++ b/rust/src/broker/non_persistent/dispatcher/single_active.rs @@ -1,6 +1,6 @@ use crate::broker::service::topic::SubscriptionType; use crate::broker::service::Consumer; -use crate::storage::{MessageId, NonPersistentEntry}; +use pulsar_lite_storage_managed_ledger::{MessageId, NonPersistentEntry}; use std::sync::{ atomic::{AtomicU64, Ordering}, Arc, @@ -86,7 +86,9 @@ impl NonPersistentDispatcherExclusive { .fetch_add(entries.len() as u64, Ordering::Relaxed); let Some(consumer) = &self.consumer else { for entry in entries { - self.record_drop(1); + self.record_drop( + crate::broker::dispatcher::messages_in_batch(entry.metadata()) as u64, + ); entry.release(); } return Ok(()); @@ -101,15 +103,18 @@ impl NonPersistentDispatcherExclusive { let metadata = entry.metadata_bytes(); let payload = entry.payload_bytes(); + let batch_count = crate::broker::dispatcher::messages_in_batch(entry.metadata()); if let Some(reservation) = consumer - .try_reserve_dispatch(&message_id, metadata, payload, 0) + .try_reserve_dispatch(&message_id, metadata, payload, 0, batch_count) .await { reservation.send(); - self.record_dispatched(1); - consumer.record_message_dispatched(entry.len()).await; + self.record_dispatched(batch_count as u64); + consumer + .record_message_dispatched(batch_count, entry.len()) + .await; } else { - self.record_drop(1); + self.record_drop(batch_count as u64); } entry.release(); } @@ -357,7 +362,9 @@ impl NonPersistentDispatcherFailover { .fetch_add(entries.len() as u64, Ordering::Relaxed); let Some(active_consumer) = self.get_active_consumer() else { for entry in entries { - self.record_drop(1); + self.record_drop( + crate::broker::dispatcher::messages_in_batch(entry.metadata()) as u64, + ); entry.release(); } return Ok(()); @@ -372,15 +379,18 @@ impl NonPersistentDispatcherFailover { let metadata = entry.metadata_bytes(); let payload = entry.payload_bytes(); + let batch_count = crate::broker::dispatcher::messages_in_batch(entry.metadata()); if let Some(reservation) = active_consumer - .try_reserve_dispatch(&message_id, metadata, payload, 0) + .try_reserve_dispatch(&message_id, metadata, payload, 0, batch_count) .await { reservation.send(); - self.record_dispatched(1); - active_consumer.record_message_dispatched(entry.len()).await; + self.record_dispatched(batch_count as u64); + active_consumer + .record_message_dispatched(batch_count, entry.len()) + .await; } else { - self.record_drop(1); + self.record_drop(batch_count as u64); } entry.release(); } diff --git a/rust/src/broker/non_persistent/dispatcher/sticky_key.rs b/rust/src/broker/non_persistent/dispatcher/sticky_key.rs index 002191a..4b037fd 100644 --- a/rust/src/broker/non_persistent/dispatcher/sticky_key.rs +++ b/rust/src/broker/non_persistent/dispatcher/sticky_key.rs @@ -5,7 +5,7 @@ use crate::broker::service::topic::{ KeySharedHashRange, KeySharedMode, KeySharedPolicy, SubscriptionType, }; use crate::broker::service::Consumer; -use crate::storage::{MessageId, NonPersistentEntry}; +use pulsar_lite_storage_managed_ledger::{MessageId, NonPersistentEntry}; use std::collections::HashMap; use std::sync::{ atomic::{AtomicU32, AtomicU64, Ordering}, @@ -228,7 +228,9 @@ impl NonPersistentStickyKeyDispatcher { log::debug!( "Dropping non-persistent key-shared entry due to no available consumer" ); - self.record_drop(1); + self.record_drop( + crate::broker::dispatcher::messages_in_batch(entry.metadata()) as u64, + ); entry.release(); continue; }; @@ -241,16 +243,19 @@ impl NonPersistentStickyKeyDispatcher { let metadata = entry.metadata_bytes(); let payload = entry.payload_bytes(); + let batch_count = crate::broker::dispatcher::messages_in_batch(entry.metadata()); if let Some(reservation) = consumer - .try_reserve_dispatch(&message_id, metadata, payload, 0) + .try_reserve_dispatch(&message_id, metadata, payload, 0, batch_count) .await { reservation.send(); - self.record_dispatched(1); - self.subtract_total_permits(1); - consumer.record_message_dispatched(entry.len()).await; + self.record_dispatched(batch_count as u64); + self.subtract_total_permits(batch_count); + consumer + .record_message_dispatched(batch_count, entry.len()) + .await; } else { - self.record_drop(1); + self.record_drop(batch_count as u64); } entry.release(); @@ -263,10 +268,10 @@ impl NonPersistentStickyKeyDispatcher { mod tests { use super::*; use crate::broker::service::topic::{Subscription, SubscriptionRuntimeMode}; - use crate::protocol::codec::proto::pulsar::MessageMetadata; - use crate::storage::Storage; use bytes::Bytes; use prost::Message; + use pulsar_lite_proto::codec::proto::pulsar::MessageMetadata; + use pulsar_lite_storage::Storage; use std::path::Path; use std::time::Instant; use tokio::sync::{mpsc, Mutex, RwLock}; diff --git a/rust/src/broker/non_persistent/runtime.rs b/rust/src/broker/non_persistent/runtime.rs index 927e812..c3a1c8d 100644 --- a/rust/src/broker/non_persistent/runtime.rs +++ b/rust/src/broker/non_persistent/runtime.rs @@ -9,7 +9,7 @@ use crate::broker::non_persistent::NonPersistentDispatcherEnum; use crate::broker::service::topic::{KeySharedPolicy, SubscriptionType}; use crate::broker::service::Consumer; -use crate::storage::NonPersistentEntry; +use pulsar_lite_storage_managed_ledger::NonPersistentEntry; use std::collections::HashMap; use std::sync::Arc; diff --git a/rust/src/broker/service/connection_write_state.rs b/rust/src/broker/service/connection_write_state.rs index 6a734d4..5153fe3 100644 --- a/rust/src/broker/service/connection_write_state.rs +++ b/rust/src/broker/service/connection_write_state.rs @@ -1,5 +1,11 @@ use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; +/// Byte budget for the per-connection message channel (dispatcher -> socket). +/// Dispatch pauses when buffered-but-unencoded message bytes exceed this, +/// so fan-out scenarios (e.g. 8 subscriptions) cannot flood the channel +/// faster than the client can drain it. Roughly 64 entries of ~1MB each. +const DEFAULT_MAX_CHANNEL_BYTES: usize = 64 * 1024 * 1024; + /// Shared connection-level outbound write state. /// /// This approximates Pulsar/Netty channel writability semantics: once the @@ -12,6 +18,9 @@ pub struct ConnectionWriteState { writable: AtomicBool, high_watermark_bytes: usize, low_watermark_bytes: usize, + /// Unencoded message bytes sitting in the mpsc channel (dispatcher side). + channel_pending_bytes: AtomicUsize, + max_channel_bytes: usize, } impl ConnectionWriteState { @@ -26,6 +35,8 @@ impl ConnectionWriteState { writable: AtomicBool::new(true), high_watermark_bytes, low_watermark_bytes, + channel_pending_bytes: AtomicUsize::new(0), + max_channel_bytes: DEFAULT_MAX_CHANNEL_BYTES, } } @@ -51,6 +62,36 @@ impl ConnectionWriteState { self.writable.store(next_writable, Ordering::Release); } + /// Try to reserve `bytes` of channel budget. Returns false (and reserves + /// nothing) when the channel would exceed the byte cap. + pub fn try_reserve_channel_bytes(&self, bytes: usize) -> bool { + let current = self.channel_pending_bytes.load(Ordering::Relaxed); + loop { + let next = current.saturating_add(bytes); + if next > self.max_channel_bytes { + return false; + } + match self.channel_pending_bytes.compare_exchange_weak( + current, + next, + Ordering::Relaxed, + Ordering::Relaxed, + ) { + Ok(_) => return true, + Err(actual) => { + let current = actual; + let _ = current; + } + } + } + } + + /// Release channel budget after the run loop dequeues a message. + pub fn release_channel_bytes(&self, bytes: usize) { + self.channel_pending_bytes + .fetch_sub(bytes, Ordering::Relaxed); + } + pub fn high_watermark_bytes(&self) -> usize { self.high_watermark_bytes } diff --git a/rust/src/broker/service/consumer.rs b/rust/src/broker/service/consumer.rs index f63667f..76a1fbe 100644 --- a/rust/src/broker/service/consumer.rs +++ b/rust/src/broker/service/consumer.rs @@ -5,7 +5,7 @@ use bytes::Bytes; use std::sync::{ - atomic::{AtomicBool, AtomicI64, AtomicU32, Ordering}, + atomic::{AtomicBool, AtomicI32, AtomicI64, Ordering}, Arc, }; @@ -15,7 +15,7 @@ use tokio::sync::{mpsc, RwLock}; use super::topic::KeySharedPolicy; use super::topic::{AckCommandType, Subscription, SubscriptionType}; -use crate::storage::MessageId; +use pulsar_lite_storage_managed_ledger::MessageId; /// Consumer statistics #[derive(Debug, Default, Clone)] @@ -60,7 +60,7 @@ pub struct PendingMessage { /// Call `send()` to commit the dispatch. If dropped without sending, all /// resources are automatically rolled back (dispatch-or-drop semantics). pub struct DispatchReservation { - available_permits: Arc, + available_permits: Arc, pending_acks: Arc, owned_permit: Option>, consumer_id: u64, @@ -119,7 +119,7 @@ pub struct Consumer { /// Statistics stats: Arc>, - available_permits: Arc, + available_permits: Arc, /// Message sender channel - sends messages to ServerCnx for delivery /// Format: (consumer_id, PendingMessage) @@ -137,6 +137,11 @@ pub struct Consumer { /// Failover active-consumer view, updated by dispatcher notifications. active_consumer_id: AtomicI64, is_active_consumer: AtomicBool, + /// Pre-resolved subscription metrics. Resolved lazily via `try_read` + /// (never `.read().await`): the dispatch path may run while the + /// subscription lock is contended, and awaiting a read here can queue + /// behind a pending writer and deadlock the connection. + sub_metrics: std::sync::OnceLock>, } impl std::fmt::Debug for Consumer { @@ -192,7 +197,7 @@ impl Consumer { subscription, connection_id, stats: Arc::new(RwLock::new(ConsumerStats::default())), - available_permits: Arc::new(AtomicU32::new(0)), + available_permits: Arc::new(AtomicI32::new(0)), message_tx, connection_write_state, pending_acks: Arc::new(PendingAcksMap::new()), @@ -200,12 +205,19 @@ impl Consumer { key_shared_policy, active_consumer_id: AtomicI64::new(-1), is_active_consumer: AtomicBool::new(false), + sub_metrics: std::sync::OnceLock::new(), } } /// Update permits (flow control) pub async fn add_permits(&self, permits: u32) { - self.available_permits.fetch_add(permits, Ordering::Relaxed); + self.available_permits + .fetch_add(permits as i32, Ordering::Relaxed); + } + + /// Instrumentation/gate: dispatched-but-unacked message count for this consumer. + pub(crate) async fn pending_acks_len(&self) -> usize { + self.pending_acks.len().await } /// Use one permit when dispatching a message @@ -225,28 +237,80 @@ impl Consumer { false } + /// Use `n` permits when dispatching a batch entry. + /// + /// Permit accounting is per client-visible message (Apache Pulsar + /// semantics): a batch entry of N messages consumes N permits. Like Pulsar, + /// the balance may briefly go negative when a batch entry is larger than + /// the remaining permits — the entry is still delivered and the deficit is + /// repaid by the client's next Flow. This keeps the message stream flowing + /// (no deadlock where the client waits for messages and the broker waits + /// for Flow). Returns false only when there is no permit at all. + pub async fn try_use_permits(&self, n: u32) -> bool { + let mut current = self.available_permits.load(Ordering::Relaxed); + while current > 0 { + match self.available_permits.compare_exchange( + current, + current - n as i32, + Ordering::Relaxed, + Ordering::Relaxed, + ) { + Ok(_) => return true, + Err(observed) => current = observed, + } + } + false + } + /// Get available permits - pub async fn get_available_permits(&self) -> u32 { + pub async fn get_available_permits(&self) -> i32 { self.available_permits.load(Ordering::Relaxed) } - /// Record message dispatched to this consumer - pub async fn record_message_dispatched(&self, message_size: usize) { - let mut stats = self.stats.write().await; - stats.messages_received += 1; - stats.bytes_received += message_size as u64; + /// Returns the subscription metrics handle, resolving it once via a + /// non-blocking `try_read`. `None` only while the subscription lock is + /// continuously write-held; the next call retries. + fn subscription_metrics(&self) -> Option<&Arc> { + if let Some(metrics) = self.sub_metrics.get() { + return Some(metrics); + } + let guard = self.subscription.try_read().ok()?; + let _ = self.sub_metrics.set(Arc::clone(&guard.metrics)); + drop(guard); + self.sub_metrics.get() + } + + /// Record entries dispatched to this consumer. `messages` is the + /// client-visible message count (batch-aware; see + /// `dispatcher::messages_in_batch`), `message_size` is entry bytes. + pub async fn record_message_dispatched(&self, messages: u32, message_size: usize) { + { + let mut stats = self.stats.write().await; + stats.messages_received += messages as u64; + stats.bytes_received += message_size as u64; + } + // Fold into subscription-level counters via the cached handle; + // never awaits the subscription lock from the dispatch path. + if let Some(metrics) = self.subscription_metrics() { + metrics.record_dispatched(messages as u64, message_size as u64); + } } /// Record message acknowledged pub async fn record_message_acked(&self) { - let mut stats = self.stats.write().await; - stats.messages_acked += 1; + { + let mut stats = self.stats.write().await; + stats.messages_acked += 1; + } + if let Some(metrics) = self.subscription_metrics() { + metrics.record_acked(); + } } /// Get current statistics pub async fn get_stats(&self) -> ConsumerStats { let mut stats = self.stats.read().await.clone(); - stats.available_permits = self.available_permits.load(Ordering::Relaxed); + stats.available_permits = self.available_permits.load(Ordering::Relaxed).max(0) as u32; let active_consumer_id = self.active_consumer_id.load(Ordering::Relaxed); stats.active_consumer_id = (active_consumer_id >= 0).then_some(active_consumer_id as u64); stats.is_active_consumer = self.is_active_consumer.load(Ordering::Relaxed); @@ -354,7 +418,17 @@ impl Consumer { { let metadata = metadata.into(); let payload = payload.into(); - let wire_size = crate::protocol::codec::estimate_message_parts_size( + + // Backpressure: if this connection's outbound write buffer is backed up + // (socket cannot drain fast enough), stop dispatching instead of filling + // the message channel. The dispatcher returns the message to its + // redelivery queue and restores the permit; dispatch resumes once acks + // flow and the buffer drains below the low watermark. + if !self.is_writable() { + return false; + } + + let wire_size = pulsar_lite_proto::codec::estimate_message_parts_size( self.consumer_id, message_id.ledger, message_id.entry, @@ -424,14 +498,16 @@ impl Consumer { metadata: Bytes, payload: Bytes, redelivery_count: u32, + permits: u32, ) -> Option { - // Step 1: Acquire flow-control permit - if !self.use_permit().await { + // Step 1: Acquire flow-control permits (per client-visible message; + // see `try_use_permits`). + if !self.try_use_permits(permits).await { return None; } // Step 2: Calculate wire size - let wire_size = crate::protocol::codec::estimate_message_parts_size( + let wire_size = pulsar_lite_proto::codec::estimate_message_parts_size( self.consumer_id, message_id.ledger, message_id.entry, @@ -443,7 +519,8 @@ impl Consumer { // Step 3: Connection must currently be writable. if !self.is_writable() { - self.available_permits.fetch_add(1, Ordering::Relaxed); + self.available_permits + .fetch_add(permits as i32, Ordering::Relaxed); return None; } @@ -451,7 +528,8 @@ impl Consumer { let owned_permit = match self.message_tx.clone().try_reserve_owned() { Ok(p) => p, Err(_) => { - self.available_permits.fetch_add(1, Ordering::Relaxed); + self.available_permits + .fetch_add(permits as i32, Ordering::Relaxed); return None; } }; @@ -463,7 +541,8 @@ impl Consumer { .track_message_dispatched(message_id, redelivery_count) .await { - self.available_permits.fetch_add(1, Ordering::Relaxed); + self.available_permits + .fetch_add(permits as i32, Ordering::Relaxed); // owned_permit drops here, releasing channel slot return None; } @@ -626,7 +705,7 @@ impl Consumer { } pub fn available_permits_now(&self) -> u32 { - self.available_permits.load(Ordering::Relaxed) + self.available_permits.load(Ordering::Relaxed).max(0) as u32 } pub fn notify_active_consumer_change(&self, active_consumer_id: u64) { @@ -660,7 +739,7 @@ impl Consumer { } /// Record consumer-level ack stats (Exclusive/Failover path). - pub async fn ack_message(&self, message_id: crate::storage::MessageId) { + pub async fn ack_message(&self, message_id: pulsar_lite_storage_managed_ledger::MessageId) { log::debug!( "Consumer {} acking message {}:{}", self.consumer_id, @@ -823,7 +902,7 @@ mod tests { use super::super::topic::{Subscription, SubscriptionRuntimeMode}; use super::super::{ConnectionWriteState, SharedStorage}; use super::*; - use crate::storage::Storage; + use pulsar_lite_storage::Storage; use std::path::Path; use tokio::sync::Mutex; @@ -884,7 +963,7 @@ mod tests { assert_eq!(consumer.get_available_permits().await, 3); // Record messages - consumer.record_message_dispatched(100).await; + consumer.record_message_dispatched(1, 100).await; consumer.record_message_acked().await; let stats = consumer.get_stats().await; @@ -926,7 +1005,7 @@ mod tests { assert_eq!(consumer.get_available_permits().await, 10); // Test ack - let msg_id = crate::storage::MessageId { + let msg_id = pulsar_lite_storage_managed_ledger::MessageId { ledger: 1, entry: 1, partition: -1, @@ -1092,7 +1171,7 @@ mod tests { .await ); - let expected_wire_size = crate::protocol::codec::estimate_message_parts_size( + let expected_wire_size = pulsar_lite_proto::codec::estimate_message_parts_size( consumer.consumer_id, 1, 1, diff --git a/rust/src/broker/service/mod.rs b/rust/src/broker/service/mod.rs index c724e3f..c32cdc7 100644 --- a/rust/src/broker/service/mod.rs +++ b/rust/src/broker/service/mod.rs @@ -4,7 +4,7 @@ * Inspired by Apache Pulsar's service structure */ -use crate::storage::Storage; +use pulsar_lite_storage::Storage; use std::sync::Arc; use tokio::sync::Mutex; diff --git a/rust/src/broker/service/pending_acks.rs b/rust/src/broker/service/pending_acks.rs index 6aff968..6098ae2 100644 --- a/rust/src/broker/service/pending_acks.rs +++ b/rust/src/broker/service/pending_acks.rs @@ -1,4 +1,4 @@ -use crate::storage::MessageId; +use pulsar_lite_storage_managed_ledger::MessageId; use std::collections::BTreeMap; use std::sync::atomic::{AtomicBool, Ordering}; use std::time::Instant; diff --git a/rust/src/broker/service/persistent/persistent_subscription.rs b/rust/src/broker/service/persistent/persistent_subscription.rs index 4b82a7e..a48c471 100644 --- a/rust/src/broker/service/persistent/persistent_subscription.rs +++ b/rust/src/broker/service/persistent/persistent_subscription.rs @@ -2,7 +2,7 @@ use crate::broker::dispatcher::redelivery_controller::RedeliveryEntry; use crate::broker::dispatcher::DispatcherEnum; use crate::broker::service::topic::{KeySharedPolicy, SubscriptionType}; use crate::broker::service::{Consumer, SharedStorage}; -use crate::storage::{ManagedLedgerPosition, MessageId}; +use pulsar_lite_storage_managed_ledger::{ManagedLedgerPosition, MessageId}; use std::sync::Arc; #[derive(Debug)] @@ -213,28 +213,25 @@ impl PersistentSubscriptionRuntime { } } - pub(crate) async fn consumer_flow(&self, consumer_id: u64, additional_permits: u32) { + /// Apply a Flow's permits to the dispatcher aggregate without triggering + /// dispatch. Used by the flow handler, which applies both permit layers + /// synchronously under the subscription read lock and triggers dispatch + /// separately off the connection event loop. + pub(crate) fn apply_flow_permits(&self, consumer_id: u64, additional_permits: u32) { if let Some(dispatcher) = self.dispatcher.as_ref() { - log::debug!( - "Subscription '{}' received flow from consumer {}, permits={}", - self.name, - consumer_id, - additional_permits - ); dispatcher.consumer_flow(consumer_id, additional_permits); + } + } - if let Err(e) = dispatcher - .dispatch_messages(self.storage.clone(), self.topic.clone(), self.name.clone()) - .await - { - log::error!( - "Failed to dispatch messages for subscription '{}': {}", - self.name, - e - ); - } - } else { - log::warn!("No dispatcher found for subscription '{}'", self.name); + pub(crate) async fn consumer_flow(&self, consumer_id: u64, additional_permits: u32) { + self.apply_flow_permits(consumer_id, additional_permits); + + if let Err(e) = self.dispatch_messages().await { + log::error!( + "Failed to dispatch messages for subscription '{}': {}", + self.name, + e + ); } } } diff --git a/rust/src/broker/service/persistent/persistent_topic.rs b/rust/src/broker/service/persistent/persistent_topic.rs index c39f491..c9c2aa9 100644 --- a/rust/src/broker/service/persistent/persistent_topic.rs +++ b/rust/src/broker/service/persistent/persistent_topic.rs @@ -1,6 +1,7 @@ use crate::broker::service::SharedStorage; -use crate::storage::{CursorInitOptions, CursorOpenResult, MessageId, Storage}; use bytes::Bytes; +use pulsar_lite_storage::Storage; +use pulsar_lite_storage_managed_ledger::{CursorInitOptions, CursorOpenResult, MessageId}; #[derive(Debug, Clone)] pub(crate) struct PersistentTopicRuntime { diff --git a/rust/src/broker/service/producer.rs b/rust/src/broker/service/producer.rs index 8ea79b1..d1f43ab 100644 --- a/rust/src/broker/service/producer.rs +++ b/rust/src/broker/service/producer.rs @@ -80,7 +80,10 @@ impl Producer { &self, metadata: Option, payload: Bytes, - ) -> Result> { + ) -> Result< + pulsar_lite_storage_managed_ledger::MessageId, + Box, + > { log::debug!( "Producer {} publishing message (metadata={} bytes, payload={} bytes)", self.producer_id, @@ -96,12 +99,15 @@ impl Producer { // - Producer state validation (is_closed) // Record statistics before publishing + let published_bytes = + (metadata.as_ref().map(|value| value.len()).unwrap_or(0) + payload.len()) as u64; self.record_message_sent(payload.len()).await; // Persistent path: keep topic write lock off the storage IO wait so multiple // producers can enqueue into the write queue concurrently. // Non-persistent path still needs &mut Topic for in-memory publish/dispatch. let topic_name; + let topic_metrics; let message_id = { let is_persistent = { let topic = self.topic.read().await; @@ -109,25 +115,32 @@ impl Producer { }; if is_persistent { - let (name, partition, persistent_runtime) = { - let mut topic = self.topic.write().await; + let (name, partition, persistent_runtime, metrics) = { + let topic = self.topic.read().await; topic.validate_publish_rate_public(metadata.as_ref(), payload.len())?; ( topic.name.clone(), topic.partition, topic.persistent_runtime_handle(), + topic.metrics.clone(), ) }; topic_name = name.clone(); + topic_metrics = metrics; persistent_runtime .publish_message(&name, partition, metadata, payload) .await? } else { let mut topic = self.topic.write().await; topic_name = topic.name.clone(); + topic_metrics = topic.metrics.clone(); topic.publish_message(metadata, payload).await? } }; + // This path serves the memory backend and in-process callers; the + // RocksDB connection path counts durable batches at the write-queue + // worker instead, so there is no double counting. + topic_metrics.record_publish(1, published_bytes); log::debug!( "Producer {} published message {}:{} to topic '{}'", @@ -189,7 +202,7 @@ impl std::hash::Hash for Producer { mod tests { use super::super::topic::Topic; use super::*; - use crate::storage::Storage; + use pulsar_lite_storage::Storage; use std::path::Path; use std::sync::Arc as StdArc; use tokio::sync::Mutex; diff --git a/rust/src/broker/service/server_cnx.rs b/rust/src/broker/service/server_cnx.rs index 9402f68..af796c0 100644 --- a/rust/src/broker/service/server_cnx.rs +++ b/rust/src/broker/service/server_cnx.rs @@ -7,6 +7,7 @@ use futures::future::pending; use futures::{SinkExt, StreamExt}; use prost::Message; use std::collections::HashMap; +use std::collections::VecDeque; use std::error::Error; use std::sync::Arc; use tokio::sync::mpsc; @@ -16,19 +17,47 @@ use tokio_util::codec::Framed; use super::consumer::PendingMessage; use super::{ConnectionWriteState, Consumer, Producer, SharedStorage}; use crate::broker::broker_service::SharedBrokerService; -use crate::broker::handler; use crate::broker::service::topic::TopicPublishRateExceeded; -use crate::protocol::codec::{ +use crate::broker::{handler, Topic}; +use pulsar_lite_proto::codec::{ proto::pulsar::{base_command, BaseCommand, ProtocolVersion, ServerError}, PulsarFrame, PulsarFrameCodec, }; -use crate::protocol::ServerCommand; +use pulsar_lite_proto::ServerCommand; + +/// Local newtype so the orphan rule accepts the foreign-trait impl: the trait +/// parameter is now the local type itself (uncovered), not a tuple-wrapped one. +pub struct ConsumerMessage(pub u64, pub PendingMessage); + +/// Adapter impl: the protocol crate only knows raw encode primitives, the +/// business message type (PendingMessage) lives here in the broker crate. +impl tokio_util::codec::Encoder for PulsarFrameCodec { + type Error = std::io::Error; + + fn encode( + &mut self, + item: ConsumerMessage, + dst: &mut bytes::BytesMut, + ) -> Result<(), Self::Error> { + let ConsumerMessage(consumer_id, msg) = item; + self.encode_message( + consumer_id, + msg.message_id.ledger, + msg.message_id.entry, + msg.message_id.partition, + &msg.metadata, + &msg.payload, + msg.redelivery_count, + dst, + ) + } +} type CnxError = Box; type CnxResult = Result; #[cfg(feature = "rocksdb-storage")] -use pulsar_lite_storage_managed_ledger_rocksdb::ConnAppendResult; +use pulsar_lite_storage_managed_ledger_rocksdb::{ConcurrentAppender, ConnAppendResult}; fn to_cnx_error(error: impl ToString) -> CnxError { Box::new(std::io::Error::other(error.to_string())) @@ -81,7 +110,7 @@ const DEFAULT_MAX_PERSISTENT_IN_FLIGHT: usize = 1000; struct PersistentSendOutcome { producer_id: u64, sequence_id: u64, - result: Result, + result: Result, } /// Runtime context for a single broker connection. @@ -123,17 +152,32 @@ where /// Number of in-flight publish tasks on this connection /// (non-persistent sync path + persistent pipelined path). pending_send_requests: usize, + /// In-flight publish bytes on this connection (payload + metadata). + /// Byte-level counterpart of pending_send_requests: caps memory for + /// large-message workloads where 1000 requests could hold GBs. + pending_send_bytes: usize, + /// FIFO of in-flight persistent publish sizes, pairing enqueue order with + /// completion order (write-queue is single-worker FIFO) for exact accounting. + pending_send_sizes: VecDeque, /// Drop gate: messages are silently dropped with fake receipt when this is exceeded. /// Maps to Pulsar's maxConcurrentNonPersistentMessagePerConnection. max_concurrent_non_persistent: usize, /// TCP throttle gate: reading from framed stops when pending reaches this limit. /// Maps to Pulsar's maxPendingPublishRequestsPerConnection. max_pending_publish_requests: usize, + /// Byte-level TCP throttle high watermark (hysteresis: resume at 50%). + max_pending_publish_bytes: usize, /// Cap for pipelined persistent Send tasks on this connection. #[cfg(feature = "rocksdb-storage")] max_persistent_in_flight: usize, + /// Write-queue handle cloned once at connection setup so the per-Send hot + /// path never takes Mutex to obtain an appender. + #[cfg(feature = "rocksdb-storage")] + persistent_appender: Option, /// Resume reading threshold (hysteresis = max_pending_publish_requests / 2). resume_threshold: usize, + /// Resume reading threshold for bytes (max_pending_publish_bytes / 2). + resume_bytes_threshold: usize, /// When true, the event loop skips framed.next(), equivalent to Netty auto-read = false. read_paused: bool, /// Maximum message size accepted by the broker. @@ -157,14 +201,14 @@ where .observe_buffered_bytes(self.framed.write_buffer().len()); } - async fn write_message_batch(&mut self, batch: Vec<(u64, PendingMessage)>) -> CnxResult<()> { + async fn write_message_batch(&mut self, batch: Vec) -> CnxResult<()> { for item in batch { self.sync_connection_writable_from_framed_buffer(); self.framed.feed(item).await.map_err(to_cnx_error)?; self.sync_connection_writable_from_framed_buffer(); } - futures::sink::SinkExt::<(u64, PendingMessage)>::flush(&mut self.framed) + futures::sink::SinkExt::::flush(&mut self.framed) .await .map_err(to_cnx_error)?; self.sync_connection_writable_from_framed_buffer(); @@ -182,10 +226,12 @@ where connection_liveness_check_timeout: Duration, max_concurrent_non_persistent: usize, max_pending_publish_requests: usize, + max_pending_publish_bytes: usize, max_message_size: usize, broker_service_url: String, channel_write_buffer_high_water_mark_bytes: usize, channel_write_buffer_low_water_mark_bytes: usize, + #[cfg(feature = "rocksdb-storage")] persistent_appender: Option, ) -> Self { let mut framed = Framed::new(socket, PulsarFrameCodec::new()); framed.set_backpressure_boundary(channel_write_buffer_high_water_mark_bytes); @@ -217,11 +263,17 @@ where connection_id, topic_manager, pending_send_requests: 0, + pending_send_bytes: 0, + pending_send_sizes: VecDeque::new(), max_concurrent_non_persistent, max_pending_publish_requests, + max_pending_publish_bytes, #[cfg(feature = "rocksdb-storage")] max_persistent_in_flight: DEFAULT_MAX_PERSISTENT_IN_FLIGHT, + #[cfg(feature = "rocksdb-storage")] + persistent_appender, resume_threshold: max_pending_publish_requests / 2, + resume_bytes_threshold: max_pending_publish_bytes / 2, read_paused: false, max_message_size, #[cfg(feature = "rocksdb-storage")] @@ -262,7 +314,7 @@ where tokio::select! { // Inbound protocol commands — skipped when read_paused (TCP backpressure). - frame_result = self.framed.next(), if !self.read_paused && self.connection_write_state.is_writable() => { + frame_result = self.framed.next(), if !self.read_paused => { let Some(frame) = frame_result else { self.close_reason.get_or_insert(CloseReason::ClientClosed); break Ok(()); @@ -283,27 +335,58 @@ where } // Durable append completions from write-queue → SendReceipt / SendError. + // Batched: drain all pending completions, feed each receipt into the + // write buffer, then flush once so a batch of receipts leaves as a + // single large TCP write. With TCP_NODELAY enabled, flushing per + // receipt would emit one small segment per message (~200k pkt/s at + // max produce rate), which is what regressed bulk persistent produce. Some(append) = self.conn_append_rx.recv() => { - let outcome = PersistentSendOutcome { - producer_id: append.producer_id, - sequence_id: append.sequence_id, - result: append.result, - }; - if let Err(e) = self.complete_persistent_send(outcome).await { + let mut batch = vec![append]; + while let Ok(next) = self.conn_append_rx.try_recv() { + batch.push(next); + if batch.len() >= 128 { + break; + } + } + + let mut send_error: Option = None; + for append in batch { + let outcome = PersistentSendOutcome { + producer_id: append.producer_id, + sequence_id: append.sequence_id, + result: append.result, + }; + if let Err(e) = self.complete_persistent_send(outcome).await { + send_error = Some(e); + break; + } + } + // Single flush for the whole batch (complete_persistent_send only feeds). + if send_error.is_none() { + if let Err(e) = + futures::sink::SinkExt::::flush(&mut self.framed) + .await + .map_err(to_cnx_error) + { + send_error = Some(e); + } + } + self.sync_connection_writable_from_framed_buffer(); + + if let Some(e) = send_error { self.set_failed(CloseReason::ProtocolError(e.to_string())); log::error!("Error completing persistent send: {}", e); break Err(e); } - self.sync_connection_writable_from_framed_buffer(); } // Outbound broker messages are batched: drain all pending, encode // into the write buffer with feed(), then flush once for a single // TCP write. This amortises syscall overhead across many messages. Some((consumer_id, pending_msg)) = self.message_rx.recv() => { - let mut batch = vec![(consumer_id, pending_msg)]; + let mut batch = vec![ConsumerMessage(consumer_id, pending_msg)]; while let Ok(next) = self.message_rx.try_recv() { - batch.push(next); + batch.push(ConsumerMessage(next.0, next.1)); if batch.len() >= 128 { break; } @@ -355,7 +438,7 @@ where tokio::select! { // Inbound protocol commands — skipped when read_paused (TCP backpressure). - frame_result = self.framed.next(), if !self.read_paused && self.connection_write_state.is_writable() => { + frame_result = self.framed.next(), if !self.read_paused => { let Some(frame) = frame_result else { self.close_reason.get_or_insert(CloseReason::ClientClosed); break Ok(()); @@ -379,9 +462,9 @@ where // into the write buffer with feed(), then flush once for a single // TCP write. This amortises syscall overhead across many messages. Some((consumer_id, pending_msg)) = self.message_rx.recv() => { - let mut batch = vec![(consumer_id, pending_msg)]; + let mut batch = vec![ConsumerMessage(consumer_id, pending_msg)]; while let Ok(next) = self.message_rx.try_recv() { - batch.push(next); + batch.push(ConsumerMessage(next.0, next.1)); if batch.len() >= 128 { break; } @@ -711,20 +794,36 @@ where .map_err(to_cnx_error) } + fn maybe_pause_read(&mut self) { + if self.pending_send_requests >= self.max_pending_publish_requests + || self.pending_send_bytes >= self.max_pending_publish_bytes + { + self.read_paused = true; + } + } + fn maybe_resume_read_after_send(&mut self) { - if self.read_paused && self.pending_send_requests <= self.resume_threshold { + if self.read_paused + && self.pending_send_requests <= self.resume_threshold + && self.pending_send_bytes <= self.resume_bytes_threshold + { self.read_paused = false; } } async fn complete_persistent_send(&mut self, outcome: PersistentSendOutcome) -> CnxResult<()> { self.pending_send_requests = self.pending_send_requests.saturating_sub(1); + if let Some(size) = self.pending_send_sizes.pop_front() { + self.pending_send_bytes = self.pending_send_bytes.saturating_sub(size); + } self.maybe_resume_read_after_send(); match outcome.result { Ok(message_id) => { + // Feed only: the caller batches completions and flushes once per + // batch (see the conn_append_rx arm in the connection loop). self.framed - .send(ServerCommand::SendReceipt { + .feed(ServerCommand::SendReceipt { producer_id: outcome.producer_id, sequence_id: outcome.sequence_id, ledger_id: message_id.ledger, @@ -734,9 +833,7 @@ where .await .map_err(to_cnx_error)?; if let Some(producer) = self.producers.get(&outcome.producer_id) { - let topic = producer.get_topic(); - let mut topic_guard = topic.write().await; - topic_guard.dispatch_to_subscriptions().await; + Topic::spawn_dispatcher(producer.get_topic()); } } Err(message) => { @@ -773,6 +870,9 @@ where .unwrap_or(0) + frame.payload.len(); if message_size > self.max_message_size { + crate::broker::stats::get() + .error_counter("message_too_large") + .inc(); self.framed .send(ServerCommand::SendError { producer_id, @@ -804,19 +904,22 @@ where } self.pending_send_requests += 1; - if self.pending_send_requests >= self.max_pending_publish_requests { - self.read_paused = true; - } + self.pending_send_bytes += message_size; + self.maybe_pause_read(); let result = handler::handle_send(&mut self.framed, cmd, frame, &self.producers).await; self.pending_send_requests = self.pending_send_requests.saturating_sub(1); + self.pending_send_bytes = self.pending_send_bytes.saturating_sub(message_size); self.maybe_resume_read_after_send(); return match result { Ok(()) => Ok(()), Err(error) => { if let Some(rate_error) = error.downcast_ref::() { + crate::broker::stats::get() + .error_counter("non_persistent_rate_exceeded") + .inc(); self.framed .send(ServerCommand::SendError { producer_id, @@ -848,16 +951,25 @@ where }; self.complete_persistent_send(outcome).await?; } + // complete_persistent_send only feeds the write buffer; flush the + // receipts drained above before enqueueing more work. + futures::sink::SinkExt::::flush(&mut self.framed) + .await + .map_err(to_cnx_error)?; let meta_len = frame.metadata.as_ref().map(|m| m.len()).unwrap_or(0); let payload_len = frame.payload.len(); - let (topic_name, partition, storage) = { - let mut topic_guard = topic.write().await; + let (topic_name, partition, topic_metrics) = { + let topic_guard = topic.read().await; if let Err(error) = topic_guard.validate_publish_rate_public(frame.metadata.as_ref(), payload_len) { if let Some(rate_error) = error.downcast_ref::() { + topic_guard.metrics.record_rate_limit_reject(); + crate::broker::stats::get() + .error_counter("publish_rate_exceeded") + .inc(); self.framed .send(ServerCommand::SendError { producer_id, @@ -874,17 +986,13 @@ where ( topic_guard.name.clone(), topic_guard.partition, - topic_guard.shared_storage(), + topic_guard.metrics.clone(), ) }; - let appender = { - let guard = storage.lock().await; - guard - .concurrent_appender() - .map_err(|e| to_cnx_error(e.to_string()))? - }; - if let Some(appender) = appender { + // Write-queue handle was cloned once at connection setup; the hot + // path never locks Mutex. + if let Some(appender) = self.persistent_appender.clone() { let meta_slice = frame.metadata.as_ref().map(|b| b.as_ref()).unwrap_or(&[]); appender .enqueue_for_connection( @@ -894,6 +1002,7 @@ where frame.payload.as_ref(), producer_id, sequence_id, + Some(topic_metrics), self.conn_append_tx.clone(), ) .map_err(to_cnx_error)?; @@ -901,18 +1010,19 @@ where producer.record_message_sent(payload_len + meta_len).await; self.pending_send_requests += 1; - if self.pending_send_requests >= self.max_pending_publish_requests { - self.read_paused = true; - } + let send_size = payload_len + meta_len; + self.pending_send_bytes += send_size; + self.pending_send_sizes.push_back(send_size); + self.maybe_pause_read(); return Ok(()); } } // Memory backend fallback (unit tests): await publish on this task. self.pending_send_requests += 1; - if self.pending_send_requests >= self.max_pending_publish_requests { - self.read_paused = true; - } + self.pending_send_bytes += message_size; + self.pending_send_sizes.push_back(message_size); + self.maybe_pause_read(); let metadata = frame.metadata.clone(); let payload = frame.payload.clone(); match handler::publish_persistent_send( @@ -934,8 +1044,14 @@ where } Err(error) => { self.pending_send_requests = self.pending_send_requests.saturating_sub(1); + if let Some(size) = self.pending_send_sizes.pop_front() { + self.pending_send_bytes = self.pending_send_bytes.saturating_sub(size); + } self.maybe_resume_read_after_send(); if let Some(rate_error) = error.downcast_ref::() { + crate::broker::stats::get() + .error_counter("memory_backend_rate_exceeded") + .inc(); self.framed .send(ServerCommand::SendError { producer_id, @@ -1052,6 +1168,7 @@ pub async fn handle_connection( connection_liveness_check_timeout: Duration, max_non_persistent_pending_messages: usize, max_pending_publish_requests: usize, + max_pending_publish_bytes: usize, max_message_size: usize, broker_service_url: String, channel_write_buffer_high_water_mark_bytes: usize, @@ -1064,6 +1181,13 @@ pub async fn handle_connection( "conn-{}", CONNECTION_COUNTER.fetch_add(1, Ordering::Relaxed) ); + // Clone the write-queue handle once per connection (before storage is + // moved into ServerCnx) so the per-Send path never locks Mutex. + #[cfg(feature = "rocksdb-storage")] + let persistent_appender = { + let guard = storage.lock().await; + guard.concurrent_appender().ok().flatten() + }; let mut server_cnx = ServerCnx::new( socket, storage, @@ -1074,10 +1198,13 @@ pub async fn handle_connection( connection_liveness_check_timeout, max_non_persistent_pending_messages, max_pending_publish_requests, + max_pending_publish_bytes, max_message_size, broker_service_url, channel_write_buffer_high_water_mark_bytes, channel_write_buffer_low_water_mark_bytes, + #[cfg(feature = "rocksdb-storage")] + persistent_appender, ); server_cnx.run().await } @@ -1087,11 +1214,11 @@ mod tests { use super::*; use crate::broker::broker_service::{BrokerService, TopicRef}; use crate::broker::service::{topic::TopicRuntimeMode, Producer}; - use crate::protocol::codec::proto::pulsar::{ + use bytes::Bytes; + use pulsar_lite_proto::codec::proto::pulsar::{ base_command, BaseCommand, CommandConnect, CommandPing, CommandSend, }; - use crate::storage::Storage; - use bytes::Bytes; + use pulsar_lite_storage::Storage; use std::sync::Arc; use tempfile::TempDir; use tokio::io::{duplex, DuplexStream}; @@ -1117,10 +1244,13 @@ mod tests { Duration::from_secs(10), 1000, 1000, + 256 * 1024 * 1024, 5 * 1024 * 1024, "pulsar://127.0.0.1:6650".to_string(), 64 * 1024, 32 * 1024, + #[cfg(feature = "rocksdb-storage")] + None, ); let client_framed = Framed::new(client, PulsarFrameCodec::new()); (server_cnx, client_framed, test_dir) @@ -1230,6 +1360,11 @@ mod tests { break; } } + // complete_persistent_send only feeds the write buffer; flush so + // test clients can observe the receipts. + futures::sink::SinkExt::::flush(&mut server_cnx.framed) + .await + .expect("flush persistent receipts"); } let _ = server_cnx; } @@ -1377,7 +1512,7 @@ mod tests { assert_eq!(send_error.sequence_id, 11); assert_eq!( send_error.error, - crate::protocol::codec::proto::pulsar::ServerError::NotAllowedError as i32 + pulsar_lite_proto::codec::proto::pulsar::ServerError::NotAllowedError as i32 ); assert!(send_error.message.contains("Exceed maximum message size")); } diff --git a/rust/src/broker/service/topic/partitioned_topic.rs b/rust/src/broker/service/topic/partitioned_topic.rs index def5a32..510dbe4 100644 --- a/rust/src/broker/service/topic/partitioned_topic.rs +++ b/rust/src/broker/service/topic/partitioned_topic.rs @@ -229,7 +229,10 @@ impl PartitionedTopic { partition_index: usize, metadata: Option, payload: Bytes, - ) -> Result> { + ) -> Result< + pulsar_lite_storage_managed_ledger::MessageId, + Box, + > { if partition_index >= self.partition_count { return Err(format!("Partition index {} out of bounds", partition_index).into()); } diff --git a/rust/src/broker/service/topic/subscription.rs b/rust/src/broker/service/topic/subscription.rs index 6df6166..22c6003 100644 --- a/rust/src/broker/service/topic/subscription.rs +++ b/rust/src/broker/service/topic/subscription.rs @@ -12,7 +12,7 @@ use super::super::{Consumer, SharedStorage}; use crate::broker::dispatcher::redelivery_controller::RedeliveryEntry; use crate::broker::non_persistent::NonPersistentSubscriptionRuntime; use crate::broker::service::persistent::PersistentSubscriptionRuntime; -use crate::storage::{ManagedLedgerPosition, MessageId, NonPersistentEntry, StorageSeekExt}; +use pulsar_lite_storage_managed_ledger::{ManagedLedgerPosition, MessageId, NonPersistentEntry}; /// Subscription type (matches Pulsar protocol) #[derive(Debug, Clone, Copy, PartialEq, Default)] @@ -91,6 +91,8 @@ pub struct Subscription { persistent_runtime: Option, /// Storage backend for reading messages storage: SharedStorage, + /// Pre-resolved Prometheus handles (label lookup happens once here). + pub(crate) metrics: Arc, } impl std::fmt::Debug for Subscription { @@ -173,6 +175,9 @@ impl Subscription { None }; + let metrics = Arc::new(crate::broker::stats::SubscriptionMetrics::new( + &topic, &name, + )); Self { name, topic, @@ -183,6 +188,7 @@ impl Subscription { non_persistent_runtime: None, persistent_runtime, storage, + metrics, } } @@ -595,6 +601,7 @@ impl Subscription { } if let Some(runtime) = self.persistent_runtime.as_mut() { + self.metrics.record_redelivered(dispatchable.len() as u64); runtime.redeliver_messages(dispatchable).await; } @@ -678,18 +685,18 @@ impl Subscription { /// Get total available permits across all consumers pub async fn get_total_permits(&self) -> u32 { - let mut total = 0; + let mut total = 0i32; for consumer in self.get_consumers() { total += consumer.get_available_permits().await; } - total + total.max(0) as u32 } /// Get subscription statistics pub async fn get_stats(&self) -> SubscriptionStats { let consumers = self.get_consumers(); let consumer_count = consumers.len(); - let mut total_permits = 0; + let mut total_permits = 0i32; for consumer in consumers { total_permits += consumer.get_available_permits().await; } @@ -699,7 +706,7 @@ impl Subscription { topic: self.topic.clone(), sub_type: self.sub_type, consumer_count, - total_permits, + total_permits: total_permits.max(0) as u32, received_messages: match self.runtime_mode { SubscriptionRuntimeMode::Persistent => 0, SubscriptionRuntimeMode::NonPersistent => self @@ -769,7 +776,7 @@ impl Subscription { let dispatched = runtime.dispatched_messages(); let dropped = runtime.dropped_messages(); if recv > 0 && recv % 100_000 < 50 { - log::info!( + log::debug!( "[dispatch-metrics] sub='{}' received={} dispatched={} dropped={} drop_rate={:.1}%", self.name, recv, dispatched, dropped, if recv > 0 { dropped as f64 / recv as f64 * 100.0 } else { 0.0 } @@ -795,6 +802,36 @@ impl Subscription { if let Some(runtime) = self.non_persistent_runtime.as_ref() { runtime.record_drop(count); } + self.metrics.record_dropped_n(count); + } + } + + /// Apply a Flow's permits to both accounting layers (consumer-local via + /// the handler, dispatcher aggregate here) without triggering dispatch. + /// Runs synchronously under the subscription read lock so a concurrent + /// consumer removal (write lock) never observes a half-applied Flow. + pub fn apply_flow_permits(&self, consumer_id: u64, additional_permits: u32) { + match self.runtime_mode { + SubscriptionRuntimeMode::Persistent => { + if let Some(runtime) = self.persistent_runtime.as_ref() { + runtime.apply_flow_permits(consumer_id, additional_permits); + } else { + log::warn!( + "No persistent runtime available for subscription '{}'", + self.name + ); + } + } + SubscriptionRuntimeMode::NonPersistent => { + if let Some(ref runtime) = self.non_persistent_runtime { + runtime.consumer_flow(consumer_id, additional_permits); + } else { + log::warn!( + "No non-persistent runtime available for subscription '{}'", + self.name + ); + } + } } } diff --git a/rust/src/broker/service/topic/tests/partitioned_topic.rs b/rust/src/broker/service/topic/tests/partitioned_topic.rs index e80a66b..0774676 100644 --- a/rust/src/broker/service/topic/tests/partitioned_topic.rs +++ b/rust/src/broker/service/topic/tests/partitioned_topic.rs @@ -1,6 +1,6 @@ use crate::broker::service::topic::{PartitionedTopic, Topic}; use crate::broker::service::{Producer, SharedStorage}; -use crate::storage::Storage; +use pulsar_lite_storage::Storage; use std::path::Path; use std::sync::Arc; use tokio::sync::{Mutex, RwLock}; diff --git a/rust/src/broker/service/topic/tests/subscription_non_persistence.rs b/rust/src/broker/service/topic/tests/subscription_non_persistence.rs index 309f34b..27b15ab 100644 --- a/rust/src/broker/service/topic/tests/subscription_non_persistence.rs +++ b/rust/src/broker/service/topic/tests/subscription_non_persistence.rs @@ -2,9 +2,9 @@ use crate::broker::service::topic::{ KeySharedMode, KeySharedPolicy, Subscription, SubscriptionRuntimeMode, SubscriptionType, }; use crate::broker::service::{Consumer, PendingMessage, SharedStorage}; -use crate::storage::MessageId; -use crate::storage::NonPersistentEntry; -use crate::storage::Storage; +use pulsar_lite_storage::Storage; +use pulsar_lite_storage_managed_ledger::MessageId; +use pulsar_lite_storage_managed_ledger::NonPersistentEntry; use std::collections::HashMap; use std::path::Path; use std::sync::Arc; diff --git a/rust/src/broker/service/topic/tests/subscription_persistence.rs b/rust/src/broker/service/topic/tests/subscription_persistence.rs index f65a428..5db499c 100644 --- a/rust/src/broker/service/topic/tests/subscription_persistence.rs +++ b/rust/src/broker/service/topic/tests/subscription_persistence.rs @@ -1,6 +1,7 @@ use crate::broker::service::topic::{Subscription, SubscriptionType}; use crate::broker::service::{Consumer, PendingMessage, SharedStorage}; -use crate::storage::{CursorInitOptions, InitialPosition, Storage}; +use pulsar_lite_storage::Storage; +use pulsar_lite_storage_managed_ledger::{CursorInitOptions, InitialPosition}; use std::path::Path; use std::sync::Arc; use tempfile::tempdir; @@ -371,8 +372,8 @@ async fn persistent_seek_rewinds_dispatch_to_target_message() { } fn metadata_with_publish_time(publish_time: u64) -> Vec { - use crate::protocol::codec::proto::pulsar::MessageMetadata; use prost::Message; + use pulsar_lite_proto::codec::proto::pulsar::MessageMetadata; MessageMetadata { publish_time, ..Default::default() diff --git a/rust/src/broker/service/topic/tests/topic.rs b/rust/src/broker/service/topic/tests/topic.rs index 9dee6f1..9586674 100644 --- a/rust/src/broker/service/topic/tests/topic.rs +++ b/rust/src/broker/service/topic/tests/topic.rs @@ -3,12 +3,12 @@ use crate::broker::service::topic::{ TopicRuntimeMode, }; use crate::broker::service::{Consumer, Producer, SharedStorage}; -use crate::protocol::codec::proto::pulsar::MessageMetadata; -use crate::storage::Storage; -#[cfg(feature = "rocksdb-storage")] -use crate::storage::{CursorInitOptions, InitialPosition}; use bytes::Bytes; use prost::Message; +use pulsar_lite_proto::codec::proto::pulsar::MessageMetadata; +use pulsar_lite_storage::Storage; +#[cfg(feature = "rocksdb-storage")] +use pulsar_lite_storage_managed_ledger::{CursorInitOptions, InitialPosition}; use std::path::Path; use std::sync::{Arc, Arc as StdArc}; use std::time::Instant; @@ -383,6 +383,65 @@ async fn test_non_persistent_dispatches_entries_per_subscription_in_order() { assert_eq!(sub2_second.1.payload, b"second".to_vec()); } +#[tokio::test] +async fn non_persistent_fire_and_forget_enqueue_preserves_order() { + let storage = create_test_storage(); + let topic = Arc::new(RwLock::new(Topic::new( + "non-persistent://public/default/fanout-order".to_string(), + storage, + ))); + + // Publishes with no subscription enqueue and drain without blocking. + for i in 0..5 { + Topic::publish_non_persistent(&topic, None, Bytes::from(format!("nosub-{i}"))) + .await + .unwrap(); + } + + let subscription = topic + .write() + .await + .get_or_create_subscription("sub1", SubscriptionType::Exclusive) + .await + .unwrap(); + let (consumer, mut rx) = create_test_consumer_with_rx(1, subscription.clone()); + { + let mut sub_guard = subscription.write().await; + sub_guard.add_consumer(consumer).unwrap(); + } + { + let sub_guard = subscription.read().await; + sub_guard.get_consumer(1).unwrap().add_permits(400).await; + } + { + let sub_guard = subscription.read().await; + sub_guard.consumer_flow(1, 400).await; + } + + // Hot-path semantics: enqueue and return without awaiting fan-out. + for i in 0..300 { + Topic::publish_non_persistent(&topic, None, Bytes::from(format!("m{i}"))) + .await + .unwrap(); + } + // Barrier: publish_message queues behind all fire-and-forget jobs and + // awaits its own completion, so every earlier job has been dispatched. + topic + .write() + .await + .publish_message(None, Bytes::from_static(b"barrier")) + .await + .unwrap(); + + let mut received = Vec::with_capacity(301); + while let Ok((_, pending)) = rx.try_recv() { + received.push(pending.payload); + } + let mut expected: Vec> = (0..300).map(|i| format!("m{i}").into_bytes()).collect(); + expected.push(b"barrier".to_vec()); + assert_eq!(received, expected); +} + #[tokio::test] async fn test_non_persistent_topic_immediately_drops_for_blocked_subscription() { let storage = create_test_storage(); diff --git a/rust/src/broker/service/topic/topic.rs b/rust/src/broker/service/topic/topic.rs index 81b83d5..9dc2322 100644 --- a/rust/src/broker/service/topic/topic.rs +++ b/rust/src/broker/service/topic/topic.rs @@ -9,13 +9,15 @@ use super::{ use crate::broker::service::persistent::PersistentTopicRuntime; use crate::broker::service::{Consumer, Producer, SharedStorage}; -use crate::storage::{parse_topic_name, CursorInitOptions, MessageId, NonPersistentEntry}; +use pulsar_lite_storage_managed_ledger::{CursorInitOptions, MessageId, NonPersistentEntry}; +use pulsar_lite_storage_metadata::parse_topic_name; + use bytes::Bytes; use std::collections::HashMap; use std::fmt; use std::sync::Arc; use std::time::{Duration, Instant}; -use tokio::sync::RwLock; +use tokio::sync::{mpsc, oneshot, RwLock}; /// Type alias for shared subscription pub type SharedSubscription = Arc>; @@ -51,6 +53,70 @@ impl NonPersistentPublish { } } +/// One queued job for the per-topic non-persistent fan-out worker. +struct NonPersistentFanoutJob { + publish: NonPersistentPublish, + /// When set, the worker replies after dispatching this job (used by + /// `publish_message` for deterministic in-process completion). + done: Option>, +} + +/// Bounded depth of the per-topic non-persistent fan-out queue. The worker +/// applies dispatch-or-drop semantics and never blocks on consumers, so this +/// bounds only the producer-side burst between enqueue and fan-out. +const NON_PERSISTENT_FANOUT_QUEUE_CAPACITY: usize = 4096; + +/// Jobs drained from the fan-out queue per worker pass. Batching amortizes +/// the worker wakeup and (per subscription) the lock acquisition plus the +/// vectorized dispatcher call across many messages. +const NON_PERSISTENT_FANOUT_BATCH_MAX: usize = 128; + +/// Dispatch a drained batch of fan-out jobs. +/// +/// Entries are grouped per subscription across the whole batch (jobs on one +/// topic normally carry identical subscription snapshots), so each +/// subscription pays one lock acquisition and one vectorized +/// `send_non_persistent_entries` call per batch instead of per message. +/// Entry order within a subscription follows job order (FIFO), preserving +/// per-subscription delivery order. `done` replies fire after the whole +/// batch is dispatched, which keeps `publish_message`'s barrier semantics +/// conservative and deterministic. +async fn dispatch_fanout_jobs(jobs: Vec) { + let mut groups: Vec<(SharedSubscription, Vec)> = Vec::new(); + let mut dones: Vec> = Vec::new(); + for job in jobs { + let NonPersistentFanoutJob { publish, done } = job; + if let Some(done) = done { + dones.push(done); + } + let entry = publish.entry; + for subscription in publish.subscriptions { + match groups + .iter_mut() + .find(|(s, _)| Arc::ptr_eq(s, &subscription)) + { + Some((_, entries)) => entries.push(entry.retained_duplicate()), + None => groups.push((subscription, vec![entry.retained_duplicate()])), + } + } + entry.release(); + } + for (subscription, entries) in groups { + let result = { + let sub_guard = subscription.read().await; + sub_guard.send_non_persistent_entries(entries).await + }; + if let Err(e) = result { + log::error!( + "Failed to dispatch non-persistent entries to subscription: {}", + e + ); + } + } + for done in dones { + let _ = done.send(()); + } +} #[derive(Debug, Clone, Copy, Default)] pub struct TopicPublishRate { pub messages_per_sec: u64, @@ -59,6 +125,15 @@ pub struct TopicPublishRate { #[derive(Debug)] struct TopicPublishRateLimiter { + /// Interior mutability: checked from read-locked Topic context on the hot + /// send path, so the window state lives behind a short std Mutex instead of + /// requiring `&mut Topic`. The default unlimited configuration takes the + /// fast path before any window arithmetic. + inner: std::sync::Mutex, +} + +#[derive(Debug)] +struct TopicPublishRateLimiterState { limits: TopicPublishRate, window_started_at: Instant, messages_in_window: u64, @@ -68,40 +143,45 @@ struct TopicPublishRateLimiter { impl TopicPublishRateLimiter { fn new() -> Self { Self { - limits: TopicPublishRate::default(), - window_started_at: Instant::now(), - messages_in_window: 0, - bytes_in_window: 0, + inner: std::sync::Mutex::new(TopicPublishRateLimiterState { + limits: TopicPublishRate::default(), + window_started_at: Instant::now(), + messages_in_window: 0, + bytes_in_window: 0, + }), } } - fn set_limits(&mut self, limits: TopicPublishRate) { - self.limits = limits; - self.window_started_at = Instant::now(); - self.messages_in_window = 0; - self.bytes_in_window = 0; + fn set_limits(&self, limits: TopicPublishRate) { + let mut state = self.inner.lock().unwrap_or_else(|e| e.into_inner()); + state.limits = limits; + state.window_started_at = Instant::now(); + state.messages_in_window = 0; + state.bytes_in_window = 0; } - fn allow_publish(&mut self, message_count: u64, bytes: u64) -> bool { - if self.limits.messages_per_sec == 0 && self.limits.bytes_per_sec == 0 { + fn allow_publish(&self, message_count: u64, bytes: u64) -> bool { + let mut state = self.inner.lock().unwrap_or_else(|e| e.into_inner()); + + if state.limits.messages_per_sec == 0 && state.limits.bytes_per_sec == 0 { return true; } - if self.window_started_at.elapsed() >= Duration::from_secs(1) { - self.window_started_at = Instant::now(); - self.messages_in_window = 0; - self.bytes_in_window = 0; + if state.window_started_at.elapsed() >= Duration::from_secs(1) { + state.window_started_at = Instant::now(); + state.messages_in_window = 0; + state.bytes_in_window = 0; } - let next_messages = self.messages_in_window.saturating_add(message_count); - let next_bytes = self.bytes_in_window.saturating_add(bytes); + let next_messages = state.messages_in_window.saturating_add(message_count); + let next_bytes = state.bytes_in_window.saturating_add(bytes); let messages_ok = - self.limits.messages_per_sec == 0 || next_messages <= self.limits.messages_per_sec; - let bytes_ok = self.limits.bytes_per_sec == 0 || next_bytes <= self.limits.bytes_per_sec; + state.limits.messages_per_sec == 0 || next_messages <= state.limits.messages_per_sec; + let bytes_ok = state.limits.bytes_per_sec == 0 || next_bytes <= state.limits.bytes_per_sec; if messages_ok && bytes_ok { - self.messages_in_window = next_messages; - self.bytes_in_window = next_bytes; + state.messages_in_window = next_messages; + state.bytes_in_window = next_bytes; true } else { false @@ -147,6 +227,14 @@ pub struct Topic { /// Storage backend storage: SharedStorage, publish_rate_limiter: TopicPublishRateLimiter, + /// Pre-resolved Prometheus handles for this topic (label lookup happens + /// once here; publish paths only touch atomic counters). + pub(crate) metrics: Arc, + /// Ordered fan-out queue for non-persistent publishes (lock-free hot path). + /// Mirrors the persistent write-queue shape: connection tasks enqueue under + /// a short read lock and return immediately; a single worker drains FIFO, + /// preserving per-subscription delivery order. + non_persistent_fanout_tx: Option>, } impl Topic { @@ -175,7 +263,8 @@ impl Topic { runtime_mode ); let persistent_runtime = PersistentTopicRuntime::new(storage.clone()); - Self { + let metrics = Arc::new(crate::broker::stats::TopicMetrics::new(&name)); + let mut topic = Self { name, partition, producers: HashMap::new(), @@ -184,7 +273,13 @@ impl Topic { persistent_runtime, storage, publish_rate_limiter: TopicPublishRateLimiter::new(), + metrics, + non_persistent_fanout_tx: None, + }; + if topic.runtime_mode == TopicRuntimeMode::NonPersistent { + topic.ensure_non_persistent_fanout_worker(); } + topic } pub fn runtime_mode(&self) -> TopicRuntimeMode { @@ -193,6 +288,9 @@ impl Topic { pub fn set_runtime_mode(&mut self, mode: TopicRuntimeMode) { self.runtime_mode = mode; + if mode == TopicRuntimeMode::NonPersistent { + self.ensure_non_persistent_fanout_worker(); + } } pub fn set_publish_rate(&mut self, limits: TopicPublishRate) { @@ -204,15 +302,10 @@ impl Topic { self.persistent_runtime.clone() } - #[cfg(feature = "rocksdb-storage")] - pub(crate) fn shared_storage(&self) -> SharedStorage { - self.storage.clone() - } - /// Rate-limit check used by producers that release the topic write lock /// before waiting on storage IO. pub(crate) fn validate_publish_rate_public( - &mut self, + &self, metadata: Option<&Bytes>, payload_len: usize, ) -> Result<(), Box> { @@ -248,7 +341,7 @@ impl Topic { } fn validate_publish_rate( - &mut self, + &self, metadata: Option<&Bytes>, payload_len: usize, ) -> Result<(), Box> { @@ -258,6 +351,7 @@ impl Topic { .saturating_add(payload_len as u64); if !self.publish_rate_limiter.allow_publish(1, total_bytes) { + self.metrics.record_rate_limit_reject(); return Err(Box::new(TopicPublishRateExceeded { topic_name: self.name.clone(), })); @@ -266,7 +360,7 @@ impl Topic { } pub fn prepare_non_persistent_publish( - &mut self, + &self, metadata: Option, payload: Bytes, ) -> Result> { @@ -290,6 +384,81 @@ impl Topic { Ok(self.build_non_persistent_publish(metadata, payload)) } + /// Spawn the per-topic ordered fan-out worker (idempotent). + /// + /// Topics constructed outside a tokio runtime (sync unit-test contexts) + /// skip this and keep the inline `dispatch_sequential` fallback. + fn ensure_non_persistent_fanout_worker(&mut self) { + if self.non_persistent_fanout_tx.is_some() { + return; + } + if tokio::runtime::Handle::try_current().is_err() { + return; + } + let (tx, mut rx) = + mpsc::channel::(NON_PERSISTENT_FANOUT_QUEUE_CAPACITY); + tokio::spawn(async move { + while let Some(first) = rx.recv().await { + let mut jobs = Vec::with_capacity(NON_PERSISTENT_FANOUT_BATCH_MAX); + jobs.push(first); + while jobs.len() < NON_PERSISTENT_FANOUT_BATCH_MAX { + match rx.try_recv() { + Ok(job) => jobs.push(job), + Err(_) => break, + } + } + dispatch_fanout_jobs(jobs).await; + } + }); + self.non_persistent_fanout_tx = Some(tx); + } + + /// Non-persistent publish hot path — the lock-free mirror of the persistent + /// write-queue enqueue: one short read lock for the rate check and the + /// subscription snapshot, then enqueue to the ordered fan-out worker and + /// return. The SendReceipt is an accept-ack; fan-out completes + /// asynchronously (same contract as the drop gate's fake receipts). + pub(crate) async fn publish_non_persistent( + topic: &Arc>, + metadata: Option, + payload: Bytes, + ) -> Result> { + let accepted_bytes = + (metadata.as_ref().map(|m| m.len()).unwrap_or(0) + payload.len()) as u64; + let (publish, fanout_tx, metrics) = { + let topic_guard = topic.read().await; + let publish = topic_guard.prepare_non_persistent_publish(metadata, payload)?; + ( + publish, + topic_guard.non_persistent_fanout_tx.clone(), + topic_guard.metrics.clone(), + ) + }; + let message_id = publish.message_id(); + Self::fan_out_non_persistent(publish, fanout_tx).await?; + metrics.record_publish(1, accepted_bytes); + Ok(message_id) + } + + async fn fan_out_non_persistent( + publish: NonPersistentPublish, + fanout_tx: Option>, + ) -> Result<(), Box> { + match fanout_tx { + Some(tx) => tx + .send(NonPersistentFanoutJob { + publish, + done: None, + }) + .await + .map_err(|e| -> Box { + format!("non-persistent fan-out worker gone: {e}").into() + })?, + None => publish.dispatch_sequential().await, + } + Ok(()) + } + /// Extract partition ID from topic name /// Returns -1 for non-partitioned topics fn extract_partition_from_name(name: &str) -> i32 { @@ -452,6 +621,24 @@ impl Topic { self.subscriptions.len() } + /// Lock-free consumer count for the metrics aggregation loop. + /// + /// Uses `try_read` on each subscription: under churn a busy + /// subscription is skipped, which is acceptable for a gauge sampled + /// every few seconds. + pub fn total_consumer_count_snapshot(&self) -> i64 { + self.subscriptions + .values() + .filter_map(|subscription| subscription.try_read().ok()) + .map(|guard| guard.get_consumer_count() as i64) + .sum() + } + + /// Snapshot of subscriptions for the metrics aggregation loop. + pub fn get_all_subscriptions(&self) -> Vec { + self.subscriptions.values().cloned().collect() + } + /// Check if subscription exists pub fn has_subscription(&self, subscription_name: &str) -> bool { self.subscriptions.contains_key(subscription_name) @@ -502,7 +689,10 @@ impl Topic { &mut self, metadata: Option, payload: Bytes, - ) -> Result> { + ) -> Result< + pulsar_lite_storage_managed_ledger::MessageId, + Box, + > { log::debug!( "Publishing message to topic '{}' partition '{}' (metadata={} bytes, payload={} bytes)", self.name, @@ -522,7 +712,22 @@ impl Topic { TopicRuntimeMode::NonPersistent => { let publish = self.build_non_persistent_publish(metadata, payload); let message_id = publish.message_id(); - publish.dispatch_sequential().await; + match self.non_persistent_fanout_tx.clone() { + Some(tx) => { + // Same ordered queue as the network path; wait for this + // job so in-process callers (and tests) observe delivery + // deterministically. The worker never locks the topic, + // so awaiting under the caller's topic write lock is safe. + let (done_tx, done_rx) = oneshot::channel(); + tx.send(NonPersistentFanoutJob { + publish, + done: Some(done_tx), + }) + .await?; + let _ = done_rx.await; + } + None => publish.dispatch_sequential().await, + } message_id } }; @@ -538,11 +743,22 @@ impl Topic { Ok(message_id) } + /// Fire-and-forget dispatch: never blocks the caller's connection task. + /// The dispatcher's merge-trigger (dispatch_in_progress + should_reschedule + /// in dispatcher/shared.rs) coalesces concurrent triggers, so spawning one + /// task per completion is safe: no lost wakeups, no duplicate dispatch. + pub fn spawn_dispatcher(topic: Arc>) { + tokio::spawn(async move { + let topic_guard = topic.read().await; + topic_guard.dispatch_to_subscriptions().await; + }); + } + /// Dispatch messages to all subscriptions (Push mode - Apache Pulsar style) /// /// This should be called after publish_message() to push messages to consumers. /// It triggers the dispatcher for each subscription to deliver pending messages. - pub async fn dispatch_to_subscriptions(&mut self) { + pub async fn dispatch_to_subscriptions(&self) { let subscription_count = self.subscriptions.len(); match self.runtime_mode { TopicRuntimeMode::Persistent => { @@ -613,7 +829,9 @@ impl Topic { self.producers.is_empty() && self.get_total_consumer_count().await == 0 } - pub async fn get_last_message_id(&self) -> Result, String> { + pub async fn get_last_message_id( + &self, + ) -> Result, String> { match self.runtime_mode { TopicRuntimeMode::Persistent => { self.persistent_runtime diff --git a/rust/src/broker/stats/metrics.rs b/rust/src/broker/stats/metrics.rs deleted file mode 100644 index 0de6858..0000000 --- a/rust/src/broker/stats/metrics.rs +++ /dev/null @@ -1,253 +0,0 @@ -/* - * Broker Metrics - * Provides metrics collection and reporting for Pulsar Lite broker - */ - -use std::sync::atomic::{AtomicU64, Ordering}; -use std::sync::Arc; -use std::time::Instant; - -/// Broker metrics container -#[derive(Debug)] -pub struct BrokerMetrics { - // Connection metrics - pub total_connections: AtomicU64, - pub active_connections: AtomicU64, - - // Producer metrics - pub total_producers: AtomicU64, - pub messages_published: AtomicU64, - pub bytes_published: AtomicU64, - - // Consumer metrics - pub total_consumers: AtomicU64, - pub messages_delivered: AtomicU64, - pub bytes_delivered: AtomicU64, - pub messages_acked: AtomicU64, - - // Topic metrics - pub total_topics: AtomicU64, - pub total_subscriptions: AtomicU64, - - // Error metrics - pub errors: AtomicU64, - - // Performance metrics - pub start_time: Instant, -} - -impl Default for BrokerMetrics { - fn default() -> Self { - Self { - total_connections: AtomicU64::new(0), - active_connections: AtomicU64::new(0), - total_producers: AtomicU64::new(0), - messages_published: AtomicU64::new(0), - bytes_published: AtomicU64::new(0), - total_consumers: AtomicU64::new(0), - messages_delivered: AtomicU64::new(0), - bytes_delivered: AtomicU64::new(0), - messages_acked: AtomicU64::new(0), - total_topics: AtomicU64::new(0), - total_subscriptions: AtomicU64::new(0), - errors: AtomicU64::new(0), - start_time: Instant::now(), - } - } -} - -impl BrokerMetrics { - /// Create a new metrics instance - pub fn new() -> Self { - Self::default() - } - - /// Increment connection count - pub fn inc_connections(&self) { - self.total_connections.fetch_add(1, Ordering::Relaxed); - self.active_connections.fetch_add(1, Ordering::Relaxed); - } - - /// Decrement active connection count - pub fn dec_active_connections(&self) { - self.active_connections.fetch_sub(1, Ordering::Relaxed); - } - - /// Increment producer count - pub fn inc_producers(&self) { - self.total_producers.fetch_add(1, Ordering::Relaxed); - } - - /// Decrement producer count - pub fn dec_producers(&self) { - self.total_producers.fetch_sub(1, Ordering::Relaxed); - } - - /// Record a published message - pub fn record_message_published(&self, size: usize) { - self.messages_published.fetch_add(1, Ordering::Relaxed); - self.bytes_published - .fetch_add(size as u64, Ordering::Relaxed); - } - - /// Increment consumer count - pub fn inc_consumers(&self) { - self.total_consumers.fetch_add(1, Ordering::Relaxed); - } - - /// Decrement consumer count - pub fn dec_consumers(&self) { - self.total_consumers.fetch_sub(1, Ordering::Relaxed); - } - - /// Record a delivered message - pub fn record_message_delivered(&self, size: usize) { - self.messages_delivered.fetch_add(1, Ordering::Relaxed); - self.bytes_delivered - .fetch_add(size as u64, Ordering::Relaxed); - } - - /// Record an acknowledged message - pub fn record_message_acked(&self) { - self.messages_acked.fetch_add(1, Ordering::Relaxed); - } - - /// Increment topic count - pub fn inc_topics(&self) { - self.total_topics.fetch_add(1, Ordering::Relaxed); - } - - /// Increment subscription count - pub fn inc_subscriptions(&self) { - self.total_subscriptions.fetch_add(1, Ordering::Relaxed); - } - - /// Record an error - pub fn record_error(&self) { - self.errors.fetch_add(1, Ordering::Relaxed); - } - - /// Get uptime in seconds - pub fn uptime_secs(&self) -> u64 { - self.start_time.elapsed().as_secs() - } - - /// Get messages published per second (since start) - pub fn messages_published_rate(&self) -> f64 { - let uptime = self.uptime_secs(); - if uptime == 0 { - return 0.0; - } - let total = self.messages_published.load(Ordering::Relaxed); - total as f64 / uptime as f64 - } - - /// Get messages delivered per second (since start) - pub fn messages_delivered_rate(&self) -> f64 { - let uptime = self.uptime_secs(); - if uptime == 0 { - return 0.0; - } - let total = self.messages_delivered.load(Ordering::Relaxed); - total as f64 / uptime as f64 - } - - /// Format metrics as a human-readable string - pub fn to_string(&self) -> String { - format!( - "Broker Metrics:\n\ - ===============\n\ - Uptime: {}s\n\ - \n\ - Connections:\n\ - - Total: {}\n\ - - Active: {}\n\ - \n\ - Producers:\n\ - - Total: {}\n\ - - Messages Published: {} ({:.2}/s)\n\ - - Bytes Published: {} ({:.2} MB)\n\ - \n\ - Consumers:\n\ - - Total: {}\n\ - - Messages Delivered: {} ({:.2}/s)\n\ - - Bytes Delivered: {} ({:.2} MB)\n\ - - Messages Acked: {}\n\ - \n\ - Topics:\n\ - - Total: {}\n\ - - Subscriptions: {}\n\ - \n\ - Errors: {}", - self.uptime_secs(), - self.total_connections.load(Ordering::Relaxed), - self.active_connections.load(Ordering::Relaxed), - self.total_producers.load(Ordering::Relaxed), - self.messages_published.load(Ordering::Relaxed), - self.messages_published_rate(), - self.bytes_published.load(Ordering::Relaxed), - self.bytes_published.load(Ordering::Relaxed) as f64 / 1_048_576.0, - self.total_consumers.load(Ordering::Relaxed), - self.messages_delivered.load(Ordering::Relaxed), - self.messages_delivered_rate(), - self.bytes_delivered.load(Ordering::Relaxed), - self.bytes_delivered.load(Ordering::Relaxed) as f64 / 1_048_576.0, - self.messages_acked.load(Ordering::Relaxed), - self.total_topics.load(Ordering::Relaxed), - self.total_subscriptions.load(Ordering::Relaxed), - self.errors.load(Ordering::Relaxed), - ) - } -} - -/// Shared metrics instance -pub type SharedMetrics = Arc; - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn test_metrics_basic() { - let metrics = BrokerMetrics::new(); - - metrics.inc_connections(); - metrics.inc_producers(); - metrics.record_message_published(100); - metrics.inc_consumers(); - metrics.record_message_delivered(100); - metrics.record_message_acked(); - - assert_eq!(metrics.total_connections.load(Ordering::Relaxed), 1); - assert_eq!(metrics.active_connections.load(Ordering::Relaxed), 1); - assert_eq!(metrics.total_producers.load(Ordering::Relaxed), 1); - assert_eq!(metrics.messages_published.load(Ordering::Relaxed), 1); - assert_eq!(metrics.bytes_published.load(Ordering::Relaxed), 100); - assert_eq!(metrics.total_consumers.load(Ordering::Relaxed), 1); - assert_eq!(metrics.messages_delivered.load(Ordering::Relaxed), 1); - assert_eq!(metrics.bytes_delivered.load(Ordering::Relaxed), 100); - assert_eq!(metrics.messages_acked.load(Ordering::Relaxed), 1); - - // Test decrement - metrics.dec_active_connections(); - assert_eq!(metrics.active_connections.load(Ordering::Relaxed), 0); - } - - #[test] - fn test_metrics_rates() { - let metrics = BrokerMetrics::new(); - - // Initially zero - assert_eq!(metrics.messages_published_rate(), 0.0); - assert_eq!(metrics.messages_delivered_rate(), 0.0); - - // Record some messages - for _ in 0..100 { - metrics.record_message_published(50); - } - - // Rate should be calculated based on uptime - let rate = metrics.messages_published_rate(); - assert!(rate >= 0.0); - } -} diff --git a/rust/src/broker/stats/mod.rs b/rust/src/broker/stats/mod.rs index 4e219c0..e379032 100644 --- a/rust/src/broker/stats/mod.rs +++ b/rust/src/broker/stats/mod.rs @@ -1,8 +1,11 @@ /* * Broker Stats Module - * Provides metrics collection and monitoring + * Scrape-time aggregation plus re-exports of the metric handles defined in + * the `pulsar-lite-metrics` crate. */ -mod metrics; +pub mod scrape; -pub use metrics::{BrokerMetrics, SharedMetrics}; +pub use pulsar_lite_metrics::{ + get, init, parse_topic_labels, BrokerMetrics, SubscriptionMetrics, TopicLabels, TopicMetrics, +}; diff --git a/rust/src/broker/stats/scrape.rs b/rust/src/broker/stats/scrape.rs new file mode 100644 index 0000000..05a3105 --- /dev/null +++ b/rust/src/broker/stats/scrape.rs @@ -0,0 +1,324 @@ +/* + * Scrape-time aggregation task + * + * A 5s background task that walks broker state and sets the gauge half of + * the exported metrics: entity counts, backlog, unacked state, and + * scrape-derived rates. Monotonic counters are maintained on hot paths + * and never touched here. + * + * Lock discipline: broker/topic/subscription locks are acquired with + * `try_read`; a busy lock skips that entity for this round (gauges keep + * their previous value) so aggregation can never block the + * publish/dispatch paths. The storage lock is `try_lock` for the same + * reason. + */ + +use std::sync::Arc; +use std::time::Duration; + +use tokio::sync::{Mutex, RwLock}; +use tokio::time::{interval, MissedTickBehavior}; + +use crate::broker::connection_limiter::ConnectionLimiter; +use crate::broker::dispatcher::DEFAULT_MAX_UNACKED_MESSAGES_PER_CONSUMER; +use crate::broker::service::topic::{SubscriptionRuntimeMode, Topic}; +use crate::broker::service::Consumer; +use crate::broker::stats; +use crate::broker::BrokerService; + +use pulsar_lite_storage::Storage; + +/// Aggregation cadence; rate windows are computed from samples, not from +/// this interval, so changing one does not change the other. +const AGGREGATION_INTERVAL: Duration = Duration::from_secs(5); + +type SharedBrokerService = Arc>; +type SharedStorage = Arc>; + +/// Spawns the aggregation loop. Owns clones of broker-wide shared handles; +/// aborts with the process. +pub fn spawn( + broker_service: SharedBrokerService, + storage: SharedStorage, + connection_limiter: ConnectionLimiter, + rate_window_secs: u64, +) { + tokio::spawn(async move { + let mut ticker = interval(AGGREGATION_INTERVAL); + ticker.set_missed_tick_behavior(MissedTickBehavior::Skip); + // First tick fires immediately, which publishes an initial zero + // snapshot before any traffic arrives. + loop { + ticker.tick().await; + aggregate( + &broker_service, + &storage, + &connection_limiter, + rate_window_secs, + ) + .await; + } + }); +} + +async fn aggregate( + broker_service: &SharedBrokerService, + storage: &SharedStorage, + connection_limiter: &ConnectionLimiter, + rate_window_secs: u64, +) { + let metrics = stats::get(); + metrics + .active_connections + .set(connection_limiter.active_connections() as i64); + + let broker = match broker_service.try_read() { + Ok(guard) => guard, + Err(_) => return, + }; + + let mut totals = WalkTotals::default(); + + for topic in broker.get_all_topics().values() { + walk_topic(topic, storage, rate_window_secs, &mut totals).await; + } + for partitioned in broker.get_all_partitioned_topics().values() { + let guard = match partitioned.try_read() { + Ok(guard) => guard, + Err(_) => continue, + }; + for partition in guard.get_all_partitions() { + walk_topic(partition, storage, rate_window_secs, &mut totals).await; + } + } + + metrics.broker_topics_count.set(totals.topics); + metrics.broker_subscriptions_count.set(totals.subscriptions); + metrics.broker_producers_count.set(totals.producers); + metrics.broker_consumers_count.set(totals.consumers); + // A skipped entity means "unknown", not zero: hold the previous broker + // total instead of reporting a spurious dip under storage contention. + if !totals.backlog_skipped { + metrics.broker_msg_backlog.set(totals.backlog); + } + if !totals.storage_skipped { + metrics.broker_storage_size.set(totals.stored_bytes as i64); + } + let window_secs = rate_window_secs.max(1) as f64; + metrics + .broker_rate_in + .set(totals.window_messages as f64 / window_secs); + metrics + .broker_throughput_in + .set(totals.window_bytes as f64 / window_secs); + metrics + .broker_rate_out + .set(totals.window_out_messages as f64 / window_secs); + metrics + .broker_throughput_out + .set(totals.window_out_bytes as f64 / window_secs); +} + +#[derive(Default)] +struct WalkTotals { + topics: i64, + subscriptions: i64, + producers: i64, + consumers: i64, + backlog: i64, + /// True when any subscription's backlog read hit a busy lock this round. + backlog_skipped: bool, + stored_bytes: u64, + /// True when any topic's stored-bytes read hit a busy lock this round. + storage_skipped: bool, + window_messages: u64, + window_bytes: u64, + window_out_messages: u64, + window_out_bytes: u64, +} + +async fn walk_topic( + topic: &Arc>, + storage: &SharedStorage, + rate_window_secs: u64, + totals: &mut WalkTotals, +) { + // Phase 1 (holding short-lived try_read guards, fully synchronous): + // snapshot everything needed, then drop the guards before any await so + // aggregation never extends broker lock hold times. + let snapshot = match topic.try_read() { + Ok(guard) => { + let producers = guard.get_producer_count() as i64; + let consumers = guard.total_consumer_count_snapshot(); + let subscriptions = guard.get_subscription_count() as i64; + let (d_msgs, d_bytes) = guard.metrics.update_rates(rate_window_secs); + guard + .metrics + .set_entity_counts(subscriptions, producers, consumers); + let topic_metrics = Arc::clone(&guard.metrics); + let topic_name = guard.name.clone(); + let subscriptions: Vec = guard + .get_all_subscriptions() + .into_iter() + .filter_map(|subscription| { + let sub_guard = subscription.try_read().ok()?; + Some(SubscriptionSnapshot { + metrics: Arc::clone(&sub_guard.metrics), + name: sub_guard.name.clone(), + persistent: sub_guard.runtime_mode() == SubscriptionRuntimeMode::Persistent, + consumers: sub_guard.get_consumers(), + }) + }) + .collect(); + let subscription_count = subscriptions.len() as i64; + ( + topic_metrics, + topic_name, + subscriptions, + producers, + consumers, + subscription_count, + d_msgs, + d_bytes, + ) + } + Err(_) => return, + }; + let ( + topic_metrics, + topic_name, + subscriptions, + producers, + consumers, + subscription_count, + d_msgs, + d_bytes, + ) = snapshot; + + totals.topics += 1; + totals.subscriptions += subscription_count; + totals.producers += producers; + totals.consumers += consumers; + totals.window_messages += d_msgs; + totals.window_bytes += d_bytes; + + // Phase 2 (no broker locks held): storage queries and per-consumer + // awaits. Storage uses try_lock per query; a busy lock means "unknown + // this round", so gauges keep their previous value instead of dipping + // to zero under publish-path contention. + match storage.try_lock() { + Ok(storage_guard) => { + let stored_bytes = storage_guard.stored_bytes(&topic_name); + topic_metrics.set_storage_size(stored_bytes as i64); + totals.stored_bytes += stored_bytes; + } + Err(_) => totals.storage_skipped = true, + } + + for subscription in subscriptions { + let mut unacked: i64 = 0; + let mut blocked = false; + for consumer in &subscription.consumers { + let pending = consumer.pending_ack_count().await as i64; + unacked += pending; + if pending >= DEFAULT_MAX_UNACKED_MESSAGES_PER_CONSUMER as i64 { + blocked = true; + } + } + + let backlog = if subscription.persistent { + storage.try_lock().ok().map(|storage_guard| { + storage_guard + .backlog_entries(&topic_name, &subscription.name) + .unwrap_or(0) as i64 + }) + } else { + Some(0) + }; + + let (d_out, d_out_bytes) = subscription.metrics.update_rates(rate_window_secs); + subscription.metrics.set_state( + backlog, + unacked, + blocked, + subscription.consumers.len() as i64, + ); + + match backlog { + Some(backlog) => totals.backlog += backlog, + None => totals.backlog_skipped = true, + } + totals.window_out_messages += d_out; + totals.window_out_bytes += d_out_bytes; + } +} + +/// Synchronous per-subscription snapshot taken under the subscription's +/// try_read guard. +struct SubscriptionSnapshot { + metrics: Arc, + name: String, + persistent: bool, + consumers: Vec>, +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::broker::service::topic::{Subscription, SubscriptionType, Topic}; + use pulsar_lite_storage::Storage; + + #[cfg(feature = "rocksdb-storage")] + #[tokio::test] + async fn backlog_reflects_partial_acks_through_broker_subscription() { + let dir = tempfile::tempdir().unwrap(); + let storage: SharedStorage = Arc::new(Mutex::new( + Storage::new_rocksdb(dir.path()).expect("open rocksdb"), + )); + let topic: Arc> = Arc::new(RwLock::new(Topic::new( + "persistent://public/default/scrape-backlog".to_string(), + storage.clone(), + ))); + + { + let mut guard = topic.write().await; + for i in 0..10u64 { + guard + .publish_message(None, bytes::Bytes::from(format!("m{i}"))) + .await + .unwrap(); + } + } + + let subscription: Arc> = Arc::new(RwLock::new(Subscription::new( + "sub".to_string(), + "persistent://public/default/scrape-backlog".to_string(), + SubscriptionType::Exclusive, + storage.clone(), + ))); + + // Ack the first four entries through the broker ack path. + { + let mut sub = subscription.write().await; + let ids: Vec<_> = (0..4u64) + .map(|entry| pulsar_lite_storage_managed_ledger::MessageId { + ledger: 0, + entry, + partition: -1, + }) + .collect(); + sub.acknowledge_message( + &ids, + crate::broker::service::topic::AckCommandType::Individual, + ) + .await + .unwrap(); + } + + let direct = { + let guard = storage.lock().await; + guard.backlog_entries("persistent://public/default/scrape-backlog", "sub") + }; + assert_eq!(direct, Some(6), "storage-level backlog must be 6"); + } +} diff --git a/rust/src/config.rs b/rust/src/config.rs index 244535b..33b0f7d 100644 --- a/rust/src/config.rs +++ b/rust/src/config.rs @@ -59,6 +59,14 @@ pub struct Config { #[serde(default = "default_max_pending_publish_requests_per_connection")] pub max_pending_publish_requests_per_connection: usize, + /// Byte-level TCP throttle high watermark per connection. + /// In-flight publish bytes reaching this limit also pause reads (same + /// hysteresis as the request-count limit: resume at 50%). This caps memory + /// for large-message workloads where 1000 requests could hold GBs. + /// Default: 256 MiB + #[serde(default = "default_max_pending_publish_bytes_per_connection")] + pub max_pending_publish_bytes_per_connection: usize, + /// Maximum allowed message size in bytes. #[serde(default = "default_max_message_size_bytes")] pub max_message_size_bytes: usize, @@ -80,6 +88,60 @@ pub struct Config { /// Mirrors Netty's WRITE_BUFFER_LOW_WATER_MARK hysteresis semantics. #[serde(default = "default_pulsar_channel_write_buffer_low_water_mark_bytes")] pub pulsar_channel_write_buffer_low_water_mark_bytes: usize, + + /// Prometheus metrics endpoint configuration. + #[serde(default)] + pub metrics: MetricsConfig, +} + +/// Prometheus `/metrics` endpoint settings. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct MetricsConfig { + /// Serve GET /metrics when true. When false the broker neither binds + /// the metrics port nor runs the scrape aggregation task. + #[serde(default = "default_metrics_enabled")] + pub enabled: bool, + + /// Listen address for the Prometheus scrape endpoint (same 8080 model + /// as the native Pulsar web service port). + #[serde(default = "default_metrics_addr")] + pub addr: String, + + /// Value of the `cluster` label attached to every exported family. + #[serde(default = "default_metrics_cluster")] + pub cluster: String, + + /// Window (seconds) for scrape-derived rate gauges such as + /// pulsar_rate_in, mirroring the native stats period. + #[serde(default = "default_metrics_rate_window_secs")] + pub rate_window_secs: u64, +} + +impl Default for MetricsConfig { + fn default() -> Self { + Self { + enabled: default_metrics_enabled(), + addr: default_metrics_addr(), + cluster: default_metrics_cluster(), + rate_window_secs: default_metrics_rate_window_secs(), + } + } +} + +fn default_metrics_enabled() -> bool { + true +} + +fn default_metrics_addr() -> String { + "0.0.0.0:8080".to_string() +} + +fn default_metrics_cluster() -> String { + "pulsar-lite".to_string() +} + +fn default_metrics_rate_window_secs() -> u64 { + 60 } fn default_addr() -> String { @@ -114,6 +176,10 @@ fn default_max_pending_publish_requests_per_connection() -> usize { 1000 } +fn default_max_pending_publish_bytes_per_connection() -> usize { + 256 * 1024 * 1024 +} + fn default_max_message_size_bytes() -> usize { 5 * 1024 * 1024 } @@ -143,6 +209,8 @@ impl Default for Config { default_max_concurrent_non_persistent_messages_per_connection(), max_pending_publish_requests_per_connection: default_max_pending_publish_requests_per_connection(), + max_pending_publish_bytes_per_connection: + default_max_pending_publish_bytes_per_connection(), max_message_size_bytes: default_max_message_size_bytes(), publish_rate_messages_per_sec: 0, publish_rate_bytes_per_sec: 0, @@ -150,6 +218,7 @@ impl Default for Config { default_pulsar_channel_write_buffer_high_water_mark_bytes(), pulsar_channel_write_buffer_low_water_mark_bytes: default_pulsar_channel_write_buffer_low_water_mark_bytes(), + metrics: MetricsConfig::default(), } } } @@ -229,6 +298,29 @@ mod tests { config.pulsar_channel_write_buffer_low_water_mark_bytes, 32 * 1024 ); + assert!(config.metrics.enabled); + assert_eq!(config.metrics.addr, "0.0.0.0:8080"); + assert_eq!(config.metrics.cluster, "pulsar-lite"); + assert_eq!(config.metrics.rate_window_secs, 60); + } + + #[test] + fn test_metrics_section_overrides_defaults() { + let parsed: Config = toml::from_str( + r#" +[metrics] +enabled = false +addr = "127.0.0.1:9109" +cluster = "bench" +rate_window_secs = 15 +"#, + ) + .unwrap(); + + assert!(!parsed.metrics.enabled); + assert_eq!(parsed.metrics.addr, "127.0.0.1:9109"); + assert_eq!(parsed.metrics.cluster, "bench"); + assert_eq!(parsed.metrics.rate_window_secs, 15); } #[test] @@ -271,11 +363,13 @@ managed_ledger_store = "rocksdb" max_connections_per_ip: 8, max_concurrent_non_persistent_messages_per_connection: 10000, max_pending_publish_requests_per_connection: 2000, + max_pending_publish_bytes_per_connection: 512 * 1024 * 1024, max_message_size_bytes: 2048, publish_rate_messages_per_sec: 123, publish_rate_bytes_per_sec: 456, pulsar_channel_write_buffer_high_water_mark_bytes: 96 * 1024, pulsar_channel_write_buffer_low_water_mark_bytes: 48 * 1024, + metrics: MetricsConfig::default(), }; let toml_str = toml::to_string(&config).unwrap(); @@ -338,11 +432,13 @@ managed_ledger_store = "rocksdb" max_connections_per_ip: 8, max_concurrent_non_persistent_messages_per_connection: 10000, max_pending_publish_requests_per_connection: 2000, + max_pending_publish_bytes_per_connection: 512 * 1024 * 1024, max_message_size_bytes: 2048, publish_rate_messages_per_sec: 123, publish_rate_bytes_per_sec: 456, pulsar_channel_write_buffer_high_water_mark_bytes: 96 * 1024, pulsar_channel_write_buffer_low_water_mark_bytes: 48 * 1024, + metrics: MetricsConfig::default(), }; let overridden = config.with_cli_overrides( diff --git a/rust/src/lib.rs b/rust/src/lib.rs index 0c3ac21..bfee99a 100644 --- a/rust/src/lib.rs +++ b/rust/src/lib.rs @@ -1,11 +1,8 @@ pub mod broker; pub mod config; pub mod error; -pub mod protocol; -pub mod storage; // Re-export commonly used types pub use broker::BrokerService; pub use config::Config; pub use error::{Error, Result}; -pub use storage::Storage; diff --git a/rust/src/main.rs b/rust/src/main.rs index 3a62f2d..b0ac8fb 100644 --- a/rust/src/main.rs +++ b/rust/src/main.rs @@ -7,19 +7,24 @@ use clap::Parser; use futures::SinkExt; use pulsar_lite::broker::handle_connection; use pulsar_lite::broker::service::topic::TopicPublishRate; +use pulsar_lite::broker::stats; use pulsar_lite::broker::{BrokerService, ConnectionLimiter}; use pulsar_lite::config::Config; -use pulsar_lite::protocol::codec::PulsarFrameCodec; -use pulsar_lite::protocol::ServerCommand; -use pulsar_lite::storage::Storage; +use pulsar_lite_proto::codec::PulsarFrameCodec; +use pulsar_lite_proto::ServerCommand; +use pulsar_lite_storage::Storage; use std::net::{IpAddr, Ipv4Addr, SocketAddr}; use std::path::PathBuf; use std::sync::Arc; +use tikv_jemallocator::Jemalloc; use tokio::net::TcpListener; use tokio::sync::{Mutex, RwLock}; use tokio::time::Duration; use tokio_util::codec::Framed; +#[global_allocator] +static GLOBAL: Jemalloc = Jemalloc; + #[derive(Debug, Parser)] #[command(name = "pulsar-lite")] #[command(about = "Embedded lightweight message queue compatible with Apache Pulsar")] @@ -75,9 +80,10 @@ async fn main() -> Result<(), Box> { config.max_connections_per_ip ); log::info!( - "Non-persistent limits: max_concurrent_per_connection={}, max_pending_publish={}, max_message_size={}B", + "Non-persistent limits: max_concurrent_per_connection={}, max_pending_publish={}, max_pending_publish_bytes={}MiB, max_message_size={}B", config.max_concurrent_non_persistent_messages_per_connection, config.max_pending_publish_requests_per_connection, + config.max_pending_publish_bytes_per_connection / (1024 * 1024), config.max_message_size_bytes ); log::info!( @@ -86,6 +92,41 @@ async fn main() -> Result<(), Box> { config.publish_rate_bytes_per_sec ); + // Prometheus metrics: families always exist (pre-resolved handles keep + // hot paths branch-free); `enabled` only controls serving and scraping. + stats::init(&config.metrics.cluster); + if config.metrics.enabled { + match config.metrics.addr.parse::() { + Ok(metrics_addr) => { + let registry = pulsar_lite_metrics::global_registry(); + tokio::spawn(async move { + // Runs until process exit; serves GET /metrics from the + // shared registry via prometheus-hyper. + if let Err(error) = prometheus_hyper::Server::run( + registry, + metrics_addr, + std::future::pending::<()>(), + ) + .await + { + log::error!("Metrics server terminated: {}", error); + } + }); + log::info!( + "Metrics endpoint listening on http://{}/metrics", + metrics_addr + ); + } + Err(error) => { + log::warn!( + "Invalid metrics address '{}': {} — metrics disabled", + config.metrics.addr, + error + ); + } + } + } + // Initialize storage let storage = Arc::new(Mutex::new(Storage::new(&config.db_path)?)); let restored_partition_metadata = { @@ -114,6 +155,15 @@ async fn main() -> Result<(), Box> { let connection_limiter = ConnectionLimiter::new(config.max_connections, config.max_connections_per_ip); + // Scrape-time aggregation (entity counts, derived rates, active gauge). + if config.metrics.enabled { + stats::scrape::spawn( + Arc::clone(&broker_service), + Arc::clone(&storage), + connection_limiter.clone(), + config.metrics.rate_window_secs, + ); + } loop { let ret = listener.accept().await; let (socket, peer_addr) = match ret { @@ -124,12 +174,20 @@ async fn main() -> Result<(), Box> { } }; + // Request-response protocol (Send -> SendReceipt, Ping -> Pong): Nagle + // delays small replies until the previous segment is ACKed, adding + // millisecond-scale RTT. Disable it like Netty (tcpNoDelay=true default). + if let Err(e) = socket.set_nodelay(true) { + log::warn!("Failed to set TCP_NODELAY for {}: {}", peer_addr, e); + } + log::info!("New connection from {}", peer_addr); let permit = match connection_limiter.try_acquire(peer_addr.ip()) { Ok(permit) => permit, Err(error) => { log::warn!("Rejecting connection from {}: {}", peer_addr, error); + stats::get().error_counter("connection_limit").inc(); let mut framed = Framed::new(socket, PulsarFrameCodec::new()); let _ = framed .send(ServerCommand::Error { @@ -140,6 +198,7 @@ async fn main() -> Result<(), Box> { continue; } }; + stats::get().connection_created.inc(); let storage = Arc::clone(&storage); let broker_service = Arc::clone(&broker_service); @@ -155,6 +214,7 @@ async fn main() -> Result<(), Box> { connection_liveness_check_timeout, config.max_concurrent_non_persistent_messages_per_connection, config.max_pending_publish_requests_per_connection, + config.max_pending_publish_bytes_per_connection, config.max_message_size_bytes, advertised_broker_url, config.pulsar_channel_write_buffer_high_water_mark_bytes, @@ -165,6 +225,7 @@ async fn main() -> Result<(), Box> { log::error!("Connection error from {}: {}", peer_addr, e); } log::info!("Connection closed from {}", peer_addr); + stats::get().connection_closed.inc(); }); } } diff --git a/rust/src/storage/managed_ledger/mod.rs b/rust/src/storage/managed_ledger/mod.rs deleted file mode 100644 index fe4517f..0000000 --- a/rust/src/storage/managed_ledger/mod.rs +++ /dev/null @@ -1,8 +0,0 @@ -//! Managed-ledger module: re-exports the managed-ledger crate types. -pub use pulsar_lite_storage_managed_ledger::{ - CursorInitOptions, CursorOpenResult, InMemoryManagedCursor, InMemoryManagedLedger, - InMemoryManagedLedgerFactory, InMemoryManagedLedgerStorage, InitialPosition, ManagedCursor, - ManagedCursorState, ManagedLedger, ManagedLedgerConfig, ManagedLedgerFactory, - ManagedLedgerPosition, ManagedLedgerStorage, MessageId, NonPersistentEntry, StoredMessage, - SubscriptionCursor, -}; diff --git a/rust/src/storage/mod.rs b/rust/src/storage/mod.rs deleted file mode 100644 index 96ed42d..0000000 --- a/rust/src/storage/mod.rs +++ /dev/null @@ -1,66 +0,0 @@ -mod managed_ledger; - -use anyhow::Result; -pub use pulsar_lite_storage::{ManagedLedgerStore, Storage}; - -pub use managed_ledger::{ - CursorInitOptions, CursorOpenResult, InMemoryManagedCursor, InMemoryManagedLedger, - InMemoryManagedLedgerFactory, InMemoryManagedLedgerStorage, InitialPosition, ManagedCursor, - ManagedCursorState, ManagedLedger, ManagedLedgerConfig, ManagedLedgerFactory, - ManagedLedgerPosition, ManagedLedgerStorage, MessageId, NonPersistentEntry, StoredMessage, - SubscriptionCursor, -}; -pub use pulsar_lite_storage_metadata::{ - parse_topic_name, DomainNode, FileMetadataStore, MetadataDocument, MetadataFileNode, - MetadataStore, NamespaceMetadata, NamespaceNode, ParsedTopicName, PartitionedTopicNode, - SubscriptionMetadata, SubscriptionNode, TenantMetadata, TenantNode, TopicMetadata, TopicNode, -}; -pub use pulsar_lite_storage_resources::{ - NamespaceResources, PulsarResources, TenantResources, TopicResources, -}; - -pub(crate) fn decode_publish_time(metadata: &[u8]) -> Option { - use crate::protocol::codec::proto::pulsar::MessageMetadata; - use prost::Message; - if metadata.is_empty() { - return None; - } - MessageMetadata::decode(metadata) - .ok() - .map(|m| m.publish_time) -} - -/// Seek helpers that depend on broker protocol decoding (kept in the main crate). -pub trait StorageSeekExt { - fn find_message_id_by_publish_time( - &self, - topic: &str, - publish_time: u64, - ) -> Result>; -} - -impl StorageSeekExt for Storage { - fn find_message_id_by_publish_time( - &self, - topic: &str, - publish_time: u64, - ) -> Result> { - let entries = self.get_message_entries(topic); - if entries.is_empty() { - return Ok(None); - } - let mut last_earlier: Option = None; - for (i, entry) in entries.iter().enumerate() { - match decode_publish_time(&entry.metadata) { - Some(pt) if pt < publish_time => last_earlier = Some(i), - Some(_) => break, - None => {} - } - } - let target = match last_earlier { - None => Some(entries[0].message_id.clone()), - Some(i) => entries.get(i + 1).map(|e| e.message_id.clone()), - }; - Ok(target) - } -} diff --git a/rust/storage/core/Cargo.toml b/rust/storage/core/Cargo.toml index 98441f0..a52ab76 100644 --- a/rust/storage/core/Cargo.toml +++ b/rust/storage/core/Cargo.toml @@ -13,8 +13,11 @@ pulsar-lite-storage-metadata = { path = "../metadata" } pulsar-lite-storage-resources = { path = "../resources" } pulsar-lite-storage-managed-ledger = { path = "../managed-ledger" } pulsar-lite-storage-managed-ledger-rocksdb = { path = "../managed-ledger-rocksdb", optional = true } +pulsar-lite-proto = { path = "../../proto" } anyhow.workspace = true log.workspace = true +prost = "0.13" +prometheus.workspace = true [dev-dependencies] tempfile = "3.9" diff --git a/rust/storage/core/src/backend.rs b/rust/storage/core/src/backend.rs index b56f361..8887d1c 100644 --- a/rust/storage/core/src/backend.rs +++ b/rust/storage/core/src/backend.rs @@ -4,9 +4,7 @@ use pulsar_lite_storage_managed_ledger::{ ManagedLedgerStorage, MessageId, StoredMessage, }; #[cfg(feature = "rocksdb-storage")] -use pulsar_lite_storage_managed_ledger_rocksdb::{ - ConcurrentAppender, RocksDbManagedLedgerStorage, -}; +use pulsar_lite_storage_managed_ledger_rocksdb::{ConcurrentAppender, RocksDbManagedLedgerStorage}; #[cfg(feature = "rocksdb-storage")] use std::path::Path; @@ -279,4 +277,20 @@ impl ManagedLedgerStorage for ManagedLedgerStore { Self::RocksDb(inner) => inner.get_mark_delete_position(topic, subscription), } } + + fn backlog_entries(&self, topic: &str, subscription: &str) -> Option { + match self { + Self::Memory(inner) => inner.backlog_entries(topic, subscription), + #[cfg(feature = "rocksdb-storage")] + Self::RocksDb(inner) => inner.backlog_entries(topic, subscription), + } + } + + fn stored_bytes(&self, topic: &str) -> u64 { + match self { + Self::Memory(inner) => inner.stored_bytes(topic), + #[cfg(feature = "rocksdb-storage")] + Self::RocksDb(inner) => inner.stored_bytes(topic), + } + } } diff --git a/rust/storage/core/src/lib.rs b/rust/storage/core/src/lib.rs index 4b4a78d..3739f02 100644 --- a/rust/storage/core/src/lib.rs +++ b/rust/storage/core/src/lib.rs @@ -3,8 +3,7 @@ mod backend; mod config; mod error; -mod service; - +pub mod service; pub use backend::ManagedLedgerStore; pub use config::{ManagedLedgerBackendConfig, StorageConfig}; pub use error::StorageResult; diff --git a/rust/storage/core/src/service.rs b/rust/storage/core/src/service.rs index f6a942d..731b43e 100644 --- a/rust/storage/core/src/service.rs +++ b/rust/storage/core/src/service.rs @@ -2,6 +2,8 @@ use crate::backend::ManagedLedgerStore; use crate::config::{ManagedLedgerBackendConfig, StorageConfig}; use crate::error::StorageResult; use log::{debug, info}; +use prost::Message; +use pulsar_lite_proto::codec::proto::pulsar::MessageMetadata; use pulsar_lite_storage_managed_ledger::{ CursorInitOptions, CursorOpenResult, ManagedLedgerPosition, ManagedLedgerStorage, MessageId, StoredMessage, @@ -159,6 +161,39 @@ impl Storage { .seek_cursor(topic, subscription, message_id, shared) } + pub fn decode_publish_time(metadata: &[u8]) -> Option { + if metadata.is_empty() { + return None; + } + MessageMetadata::decode(metadata) + .ok() + .map(|m| m.publish_time) + } + + pub fn find_message_id_by_publish_time( + &self, + topic: &str, + publish_time: u64, + ) -> StorageResult> { + let entries = self.get_message_entries(topic); + if entries.is_empty() { + return Ok(None); + } + let mut last_earlier: Option = None; + for (i, entry) in entries.iter().enumerate() { + match Self::decode_publish_time(&entry.metadata) { + Some(pt) if pt < publish_time => last_earlier = Some(i), + Some(_) => break, + None => {} + } + } + let target = match last_earlier { + None => Some(entries[0].message_id.clone()), + Some(i) => entries.get(i + 1).map(|e| e.message_id.clone()), + }; + Ok(target) + } + pub fn first_unacked_position( &self, topic: &str, @@ -277,4 +312,14 @@ impl Storage { self.managed_ledger .get_mark_delete_position(topic, subscription) } + + /// Ledger-aware unacked entry count for a subscription cursor. + pub fn backlog_entries(&self, topic: &str, subscription: &str) -> Option { + self.managed_ledger.backlog_entries(topic, subscription) + } + + /// Bytes durably stored for `topic`. + pub fn stored_bytes(&self, topic: &str) -> u64 { + self.managed_ledger.stored_bytes(topic) + } } diff --git a/rust/storage/managed-ledger-rocksdb/Cargo.toml b/rust/storage/managed-ledger-rocksdb/Cargo.toml index b66a022..2397b32 100644 --- a/rust/storage/managed-ledger-rocksdb/Cargo.toml +++ b/rust/storage/managed-ledger-rocksdb/Cargo.toml @@ -5,6 +5,8 @@ edition.workspace = true license.workspace = true [dependencies] +pulsar-lite-metrics = { path = "../../metrics" } +pulsar-lite-proto = { path = "../../proto" } pulsar-lite-storage-managed-ledger = { path = "../managed-ledger" } anyhow.workspace = true rocksdb = "0.24" @@ -13,6 +15,7 @@ serde.workspace = true bincode = "1.3" log.workspace = true tokio = { workspace = true } +arc-swap = "1.7" [build-dependencies] prost-build = "0.13" diff --git a/rust/storage/managed-ledger-rocksdb/src/cursor.rs b/rust/storage/managed-ledger-rocksdb/src/cursor.rs index 0f1c8cc..28c84a4 100644 --- a/rust/storage/managed-ledger-rocksdb/src/cursor.rs +++ b/rust/storage/managed-ledger-rocksdb/src/cursor.rs @@ -158,14 +158,19 @@ pub fn ack_managed_cursor_shared( Some(_) => {} } + // Advance the mark-delete frontier across contiguous acknowledged ranges. + // `take_covering` consumes a whole coalesced range in one step. while let Some(mark_delete) = cursor.state().mark_delete.clone() { let Some(next) = next_position(&mark_delete, info) else { break; }; - if cursor.state.individually_deleted_entries.remove(&next) { - cursor.mark_delete(next)?; - } else { - break; + match cursor + .state + .individually_deleted_entries + .take_covering(&next) + { + Some(end) => cursor.mark_delete(end)?, + None => break, } } diff --git a/rust/storage/managed-ledger-rocksdb/src/entrylog.rs b/rust/storage/managed-ledger-rocksdb/src/entrylog.rs index dc8d41c..071f40c 100644 --- a/rust/storage/managed-ledger-rocksdb/src/entrylog.rs +++ b/rust/storage/managed-ledger-rocksdb/src/entrylog.rs @@ -164,9 +164,8 @@ impl WriteState { let checksum = EntryLogStore::checksum(&[&entry.metadata, &entry.payload]); let metadata_len = entry.metadata.len() as u32; let payload_len = entry.payload.len() as u32; - let len = ENTRY_HEADER_LEN as u64 - + entry.metadata.len() as u64 - + entry.payload.len() as u64; + let len = + ENTRY_HEADER_LEN as u64 + entry.metadata.len() as u64 + entry.payload.len() as u64; pending.push(EntryIndex { ledger_id: entry.ledger_id, @@ -386,10 +385,7 @@ impl EntryLogStore { .as_ref() .ok_or_else(|| anyhow!("entrylog writer is closed"))?; sender - .send(WriterMsg::Batch { - entries, - reply: tx, - }) + .send(WriterMsg::Batch { entries, reply: tx }) .map_err(|_| anyhow!("entrylog writer disconnected"))?; rx.recv() .map_err(|_| anyhow!("entrylog writer disconnected"))? diff --git a/rust/storage/managed-ledger-rocksdb/src/factory.rs b/rust/storage/managed-ledger-rocksdb/src/factory.rs index b12f371..1d3aae2 100644 --- a/rust/storage/managed-ledger-rocksdb/src/factory.rs +++ b/rust/storage/managed-ledger-rocksdb/src/factory.rs @@ -9,13 +9,16 @@ use std::sync::Mutex; use crate::entrylog::EntryLogStore; use pulsar_lite_storage_managed_ledger::{ManagedLedgerConfig, ManagedLedgerFactory}; -pub(crate) type SharedLedger = Arc>; +/// Shared handle for one managed ledger (write-queue worker and store readers). +/// Contents are not wrapped in a mutex; published LAC/meta use interior atomics. +pub(crate) type SharedLedger = Arc; type LedgerCache = HashMap; #[derive(Debug, Clone)] pub struct RocksDBManagedLedgerFactory { db: Arc, entry_log: Arc, + /// Serializes cache get/insert only (one Arc per name). ledgers: Arc>, } @@ -55,9 +58,8 @@ impl RocksDBManagedLedgerFactory { return Ok(Arc::clone(ledger)); } - let ledger = Arc::new(Mutex::new(self.load_ledger(name, config)?)); + let ledger = Arc::new(self.load_ledger(name, config)?); ledgers.insert(name.to_string(), Arc::clone(&ledger)); - Ok(ledger) } @@ -77,25 +79,13 @@ impl RocksDBManagedLedgerFactory { .delete(keys::managed_cursor_key(ledger_name, cursor_name))?; Ok(()) } - - fn open_ledger_with_config( - &self, - name: &str, - config: &ManagedLedgerConfig, - ) -> Result { - RocksDBManagedLedger::open_with_config( - name, - Arc::clone(&self.db), - Arc::clone(&self.entry_log), - config, - ) - } } impl ManagedLedgerFactory for RocksDBManagedLedgerFactory { type Ledger = RocksDBManagedLedger; + /// Returns an uncached instance. Production appends must use `open_ledger` + write queue. fn open(&mut self, name: &str, config: &ManagedLedgerConfig) -> Result { - self.open_ledger_with_config(name, config) + self.load_ledger(name, config) } } diff --git a/rust/storage/managed-ledger-rocksdb/src/ledger.rs b/rust/storage/managed-ledger-rocksdb/src/ledger.rs index 303ab5b..4f80921 100644 --- a/rust/storage/managed-ledger-rocksdb/src/ledger.rs +++ b/rust/storage/managed-ledger-rocksdb/src/ledger.rs @@ -2,8 +2,10 @@ use super::cursor::{next_position, RocksDBManagedCursor}; use super::entrylog::{EntryIndex, EntryLogStore, EntryRecord, EntryToAppend}; use super::keys; use super::metadata::{StoredEntryLocation, StoredManagedLedgerInfo}; -use anyhow::{Ok, Result}; +use anyhow::Result; +use arc_swap::ArcSwap; use rocksdb::{WriteBatch, DB}; +use std::sync::atomic::{AtomicU64, Ordering}; use std::sync::Arc; use pulsar_lite_storage_managed_ledger::{ @@ -12,30 +14,42 @@ use pulsar_lite_storage_managed_ledger::{ const DEFAULT_MAX_ENTRIES_PER_LEDGER: u64 = 50_000; -/// Status of the ManagedLedger that currently working on -#[derive(Debug, Clone)] +/// Assignment cursor for the active ledger segment. +/// Only the managed-ledger write-queue worker updates these fields. +#[derive(Debug)] struct ManagedLedgerRuntimeState { - /// The current ledger ID that the ManagedLedger is working on. - current_ledger_id: u64, - - /// The number of entries in the current ledger. - current_ledger_entries: u64, - - /// The size of the current ledger in bytes. - current_ledger_size: u64, + current_ledger_id: AtomicU64, + current_ledger_entries: AtomicU64, + current_ledger_size: AtomicU64, +} - /// The last confirmed position in the entire managed ledger. - last_confirmed_position: Option, +impl ManagedLedgerRuntimeState { + fn from_info(info: &StoredManagedLedgerInfo) -> Self { + let current = info + .ledgers + .last() + .expect("managed ledger info is initialized"); + Self { + current_ledger_id: AtomicU64::new(current.ledger_id), + current_ledger_entries: AtomicU64::new(current.entries), + current_ledger_size: AtomicU64::new(current.size), + } + } } -#[derive(Debug, Clone)] +#[derive(Debug)] pub struct RocksDBManagedLedger { name: String, db: Arc, - pub info: StoredManagedLedgerInfo, - runtime: ManagedLedgerRuntimeState, - max_entries_per_ledger: u64, entry_log: Arc, + max_entries_per_ledger: u64, + + /// Published ledger metadata; readers load this snapshot. + info: ArcSwap, + /// Last durable position readers may observe. + lac: ArcSwap>, + /// Write assignment cursor (single-writer: write-queue worker). + runtime: ManagedLedgerRuntimeState, } impl RocksDBManagedLedger { @@ -70,61 +84,43 @@ impl RocksDBManagedLedger { db.put(&key, info.encode_to_vec())?; - let runtime = Self::runtime_from_info(&info, &db)?; + let lac = Self::lac_from_info_and_db(&info, &db)?; + let runtime = ManagedLedgerRuntimeState::from_info(&info); Ok(Self { name: name.to_string(), db, entry_log, - info, - runtime, max_entries_per_ledger, + info: ArcSwap::from_pointee(info), + lac: ArcSwap::from_pointee(lac), + runtime, }) } - fn runtime_from_info( + fn lac_from_info_and_db( info: &StoredManagedLedgerInfo, db: &DB, - ) -> Result { - let current_ledger = info - .ledgers - .last() - .expect("managed ledger info is initialized"); - - let last_confirmed_position = - match info.ledgers.iter().rev().find(|ledger| ledger.entries > 0) { - Some(last_non_empty_ledger) => { - let entry_id = last_non_empty_ledger.entries - 1; - - let Some(value) = db.get(keys::managed_entry_key( - last_non_empty_ledger.ledger_id, - entry_id, - ))? - else { - return Ok(ManagedLedgerRuntimeState { - current_ledger_id: current_ledger.ledger_id, - current_ledger_entries: current_ledger.entries, - current_ledger_size: current_ledger.size, - last_confirmed_position: None, - }); - }; - - let location: StoredEntryLocation = bincode::deserialize(&value)?; - Some(ManagedLedgerPosition { - ledger_id: last_non_empty_ledger.ledger_id, - entry_id, - partition: location.partition, - }) - } - None => None, - }; - - Ok(ManagedLedgerRuntimeState { - current_ledger_id: current_ledger.ledger_id, - current_ledger_entries: current_ledger.entries, - current_ledger_size: current_ledger.size, - last_confirmed_position, - }) + ) -> Result> { + match info.ledgers.iter().rev().find(|ledger| ledger.entries > 0) { + Some(last_non_empty_ledger) => { + let entry_id = last_non_empty_ledger.entries - 1; + let Some(value) = db.get(keys::managed_entry_key( + last_non_empty_ledger.ledger_id, + entry_id, + ))? + else { + return Ok(None); + }; + let location: StoredEntryLocation = bincode::deserialize(&value)?; + Ok(Some(ManagedLedgerPosition { + ledger_id: last_non_empty_ledger.ledger_id, + entry_id, + partition: location.partition, + })) + } + None => Ok(None), + } } fn allocate_ledger_id(db: &DB) -> Result { @@ -138,52 +134,28 @@ impl RocksDBManagedLedger { Ok(next_ledger_id) } - fn load_entry_index(&self, ledger_id: u64, entry_id: u64) -> Result> { - let Some(value) = self.db.get(keys::managed_entry_key(ledger_id, entry_id))? else { - return Ok(None); - }; - let location: StoredEntryLocation = bincode::deserialize(&value)?; - Ok(Some(EntryIndex { - ledger_id, - entry_id, - file_id: location.file_id, - offset: location.offset, - len: location.len, - checksum: location.checksum, - partition: location.partition, - })) - } - - fn read_entry_record( - &self, - ledger_id: u64, - entry_id: u64, - ) -> Result> { - let Some(index) = self.load_entry_index(ledger_id, entry_id)? else { - return Ok(None); - }; - let position = ManagedLedgerPosition { - ledger_id, - entry_id, - partition: index.partition, - }; - let record = self.entry_log.read(&index)?; - Ok(Some((position, record))) + pub fn info_snapshot(&self) -> arc_swap::Guard> { + self.info.load() } pub fn last_position(&self) -> Result> { - Ok(self.runtime.last_confirmed_position.clone()) + Ok((**self.lac.load()).clone()) } - /// Returns true if the given position is visible in the ledger. - fn is_visible(&self, pos: &ManagedLedgerPosition) -> bool { - self.info - .ledgers + fn is_visible( + &self, + pos: &ManagedLedgerPosition, + info: &StoredManagedLedgerInfo, + lac: &ManagedLedgerPosition, + ) -> bool { + if pos > lac { + return false; + } + info.ledgers .iter() .any(|l| l.ledger_id == pos.ledger_id && l.entries > pos.entry_id) } - /// Reads a batch of entries starting from the given position, up to the specified limit. pub fn read_entries_from( &self, from: &ManagedLedgerPosition, @@ -193,15 +165,23 @@ impl RocksDBManagedLedger { return Ok(Vec::new()); } + let lac_guard = self.lac.load(); + let Some(lac) = lac_guard.as_ref() else { + return Ok(Vec::new()); + }; + if from > lac { + return Ok(Vec::new()); + } + + let info = self.info.load(); let mut out = Vec::with_capacity(limit.min(64)); let mut current = Some(from.clone()); while let Some(pos) = current { - if out.len() >= limit { + if out.len() >= limit || pos > *lac { break; } - - if self.is_visible(&pos) { + if self.is_visible(&pos, info.as_ref(), lac) { if let Some((stored, record)) = self.read_entry_record(pos.ledger_id, pos.entry_id)? { @@ -212,62 +192,75 @@ impl RocksDBManagedLedger { }); } } - current = next_position(&pos, &self.info); + current = next_position(&pos, info.as_ref()); } - Ok(out) } - pub fn add_entry_with_partition( - &mut self, - partition: i32, - payload: &[u8], - ) -> Result { - self.add_entry_with_partition_and_metadata(partition, &[], payload) + fn load_entry_index(&self, ledger_id: u64, entry_id: u64) -> Result> { + let Some(value) = self.db.get(keys::managed_entry_key(ledger_id, entry_id))? else { + return Ok(None); + }; + let location: StoredEntryLocation = bincode::deserialize(&value)?; + Ok(Some(EntryIndex { + ledger_id, + entry_id, + file_id: location.file_id, + offset: location.offset, + len: location.len, + checksum: location.checksum, + partition: location.partition, + })) } - pub fn add_entry_with_partition_and_metadata( - &mut self, - partition: i32, - metadata: &[u8], - payload: &[u8], - ) -> Result { - let mut positions = self.add_entries_with_partition_and_metadata(&[(partition,metadata,payload)])?; - positions.pop().ok_or_else(|| anyhow::anyhow!("add_entries returned empty positions")) + fn read_entry_record( + &self, + ledger_id: u64, + entry_id: u64, + ) -> Result> { + let Some(index) = self.load_entry_index(ledger_id, entry_id)? else { + return Ok(None); + }; + let position = ManagedLedgerPosition { + ledger_id, + entry_id, + partition: index.partition, + }; + let record = self.entry_log.read(&index)?; + Ok(Some((position, record))) } - /// Append many entries with one entrylog flush and one RocksDB WriteBatch. + /// Durable batch append. Intended for the write-queue worker only (single writer). /// - /// - entry_id assignment stays serial (same as single append) - /// - entrylog is written once via `append_batch` (one write_all + flush) - /// - entry locations + final managed-ledger info are written once - /// - runtime state is published only after both durable steps succeed - pub fn add_entries_with_partition_and_metadata( - &mut self, + /// Order: allocate → entrylog + rocks → publish meta → publish LAC. + pub(crate) fn add_entries_with_partition_and_metadata( + &self, items: &[(i32, &[u8], &[u8])], ) -> Result> { if items.is_empty() { return Ok(Vec::new()); } - // Pass 1: allocate positions and next ledger/runtime state in memory. - let mut next_info = self.info.clone(); - let mut next_runtime = self.runtime.clone(); + let mut next_info = (**self.info.load()).clone(); + let mut cur_id = self.runtime.current_ledger_id.load(Ordering::Relaxed); + let mut cur_entries = self.runtime.current_ledger_entries.load(Ordering::Relaxed); + let mut cur_size = self.runtime.current_ledger_size.load(Ordering::Relaxed); + let mut positions = Vec::with_capacity(items.len()); let mut to_append = Vec::with_capacity(items.len()); for (partition, metadata, payload) in items { - if next_runtime.current_ledger_entries >= self.max_entries_per_ledger { + if cur_entries >= self.max_entries_per_ledger { let next_ledger_id = Self::allocate_ledger_id(&self.db)?; next_info.roll_over_current_ledger(next_ledger_id); - next_runtime.current_ledger_id = next_ledger_id; - next_runtime.current_ledger_entries = 0; - next_runtime.current_ledger_size = 0; + cur_id = next_ledger_id; + cur_entries = 0; + cur_size = 0; } let position = ManagedLedgerPosition { - ledger_id: next_runtime.current_ledger_id, - entry_id: next_runtime.current_ledger_entries, + ledger_id: cur_id, + entry_id: cur_entries, partition: *partition, }; @@ -275,16 +268,15 @@ impl RocksDBManagedLedger { let message_size = metadata.len() as u64 + payload.len() as u64; current_ledger.entries += 1; current_ledger.size += message_size; - next_runtime.current_ledger_entries += 1; - next_runtime.current_ledger_size += message_size; - next_runtime.last_confirmed_position = Some(position.clone()); + cur_entries += 1; + cur_size += message_size; - if next_runtime.current_ledger_entries >= self.max_entries_per_ledger { + if cur_entries >= self.max_entries_per_ledger { let next_ledger_id = Self::allocate_ledger_id(&self.db)?; next_info.roll_over_current_ledger(next_ledger_id); - next_runtime.current_ledger_entries = 0; - next_runtime.current_ledger_size = 0; - next_runtime.current_ledger_id = next_ledger_id; + cur_id = next_ledger_id; + cur_entries = 0; + cur_size = 0; } to_append.push(EntryToAppend { @@ -297,7 +289,8 @@ impl RocksDBManagedLedger { positions.push(position); } - // Pass 2: one entrylog IO for the whole batch. + let last_pos = positions.last().cloned(); + let indices = self.entry_log.append_batch(to_append)?; if indices.len() != positions.len() { anyhow::bail!( @@ -307,7 +300,6 @@ impl RocksDBManagedLedger { ); } - // Pass 3: one RocksDB WriteBatch for locations + ledger info. let mut batch = WriteBatch::default(); for (position, entry_index) in positions.iter().zip(indices) { let stored_entry_location = StoredEntryLocation::from(entry_index); @@ -322,21 +314,21 @@ impl RocksDBManagedLedger { ); self.db.write(batch)?; - // Pass 4: publish in-memory state only after durable writes succeed. - self.info = next_info; - self.runtime = next_runtime; + self.info.store(Arc::new(next_info)); + self.runtime + .current_ledger_id + .store(cur_id, Ordering::Relaxed); + self.runtime + .current_ledger_entries + .store(cur_entries, Ordering::Relaxed); + self.runtime + .current_ledger_size + .store(cur_size, Ordering::Relaxed); + self.lac.store(Arc::new(last_pos)); + Ok(positions) } - - #[allow(dead_code)] - pub fn ledger_info(&self) -> &StoredManagedLedgerInfo { - &self.info - } - /// Position immediately before `position` in ledger/entry order. - /// - entry_id > 0 -> same ledger, entry_id - 1 - /// - entry_id == 0 -> last entry of the previous non-empty ledger - /// - no previous -> None ("before first entry", i.e. seek to earliest) pub fn previous_position( &self, position: &ManagedLedgerPosition, @@ -348,8 +340,8 @@ impl RocksDBManagedLedger { partition: position.partition, }); } - let prev = self - .info + let info = self.info.load(); + let prev = info .ledgers .iter() .filter(|l| l.ledger_id < position.ledger_id && l.entries > 0) @@ -361,17 +353,31 @@ impl RocksDBManagedLedger { }) } + pub fn open_cursor(&self, name: &str) -> Result { + RocksDBManagedCursor::open(&self.name, name, Arc::clone(&self.db)) + } + pub fn get_message_by_id(&self, message_id: &MessageId) -> Option<(MessageId, Vec)> { self.get_message_entry_by_id(message_id) .map(|entry| (entry.message_id, entry.payload)) } pub fn get_message_entry_by_id(&self, message_id: &MessageId) -> Option { - let (position, record) = self + let pos = ManagedLedgerPosition::from(message_id); + let lac_guard = self.lac.load(); + let lac = lac_guard.as_ref().as_ref()?; + if &pos > lac { + return None; + } + let info = self.info.load(); + if !self.is_visible(&pos, info.as_ref(), lac) { + return None; + } + let (stored, record) = self .read_entry_record(message_id.ledger, message_id.entry) .ok() .flatten()?; - if position.partition != message_id.partition { + if stored.partition != message_id.partition { return None; } Some(StoredMessage::new( @@ -389,11 +395,10 @@ impl RocksDBManagedLedger { .collect()) } - /// TODO: A temporary global scanning interface. Subsequently, all related global scanning codes will be gradually migrated. pub fn message_entries(&self) -> Result> { + let info = self.info.load(); let Some(from) = - self.info - .ledgers + info.ledgers .iter() .find(|l| l.entries > 0) .map(|l| ManagedLedgerPosition { @@ -402,16 +407,9 @@ impl RocksDBManagedLedger { partition: -1, }) else { - return Ok(vec![]); + return Ok(Vec::new()); }; - - let total = self - .info - .ledgers - .iter() - .map(|l| l.entries as usize) - .sum::(); - + let total: usize = info.ledgers.iter().map(|l| l.entries as usize).sum(); self.read_entries_from(&from, total) } } @@ -423,8 +421,10 @@ impl ManagedLedger for RocksDBManagedLedger { &self.name } - fn add_entry(&mut self, payload: &[u8]) -> Result { - self.add_entry_with_partition(-1, payload) + fn add_entry(&mut self, _payload: &[u8]) -> Result { + anyhow::bail!( + "RocksDB managed-ledger appends must go through WriteQueue; direct add_entry is disabled" + ) } fn open_cursor(&mut self, name: &str) -> Result { @@ -432,6 +432,15 @@ impl ManagedLedger for RocksDBManagedLedger { } fn read_entry(&self, position: &ManagedLedgerPosition) -> Option> { + let lac_guard = self.lac.load(); + let lac = lac_guard.as_ref().as_ref()?; + if position > lac { + return None; + } + let info = self.info.load(); + if !self.is_visible(position, info.as_ref(), lac) { + return None; + } let (stored, record) = self .read_entry_record(position.ledger_id, position.entry_id) .ok() diff --git a/rust/storage/managed-ledger-rocksdb/src/lib.rs b/rust/storage/managed-ledger-rocksdb/src/lib.rs index d390526..78266f2 100644 --- a/rust/storage/managed-ledger-rocksdb/src/lib.rs +++ b/rust/storage/managed-ledger-rocksdb/src/lib.rs @@ -10,6 +10,7 @@ mod write_queue; pub use store::{ConcurrentAppender, RocksDbManagedLedgerStorage}; pub use write_queue::ConnAppendResult; +pub use pulsar_lite_metrics::PublishCommitObserver; /// Internal types exposed for integration tests in `tests/`. #[doc(hidden)] @@ -28,4 +29,31 @@ pub mod test_support { managed_ledger_name, }; } + + use anyhow::Result; + use pulsar_lite_storage_managed_ledger::ManagedLedgerPosition; + + /// Test-only durable append (crate-private API). Production must use WriteQueue. + pub fn append_payload( + ledger: &RocksDBManagedLedger, + payload: &[u8], + ) -> Result { + append_with_partition(ledger, -1, payload) + } + + /// Test-only durable append with partition. + pub fn append_with_partition( + ledger: &RocksDBManagedLedger, + partition: i32, + payload: &[u8], + ) -> Result { + let mut positions = ledger.add_entries_with_partition_and_metadata(&[( + partition, + &[] as &[u8], + payload, + )])?; + positions + .pop() + .ok_or_else(|| anyhow::anyhow!("add_entries returned empty positions")) + } } diff --git a/rust/storage/managed-ledger-rocksdb/src/metadata.rs b/rust/storage/managed-ledger-rocksdb/src/metadata.rs index 08a4e45..e078711 100644 --- a/rust/storage/managed-ledger-rocksdb/src/metadata.rs +++ b/rust/storage/managed-ledger-rocksdb/src/metadata.rs @@ -1,9 +1,8 @@ use super::entrylog::EntryIndex; use anyhow::{anyhow, Result}; use prost::Message; -use pulsar_lite_storage_managed_ledger::{ManagedCursorState, ManagedLedgerPosition}; +use pulsar_lite_storage_managed_ledger::{ManagedCursorState, ManagedLedgerPosition, RangeSet}; use serde::{Deserialize, Serialize}; -use std::collections::BTreeSet; use std::time::{SystemTime, UNIX_EPOCH}; pub mod proto { @@ -36,7 +35,7 @@ impl From for StoredEntryLocation { #[derive(Debug, Clone, Serialize, Deserialize)] pub struct StoredManagedCursorState { pub mark_delete: Option, - pub individually_deleted_entries: BTreeSet, + pub individually_deleted_entries: RangeSet, } impl From for StoredManagedCursorState { @@ -68,7 +67,7 @@ impl StoredManagedCursorState { individual_deleted_messages: self .individually_deleted_entries .iter() - .map(position_range) + .map(|(start, end)| position_range(start, end)) .collect(), properties: Vec::new(), last_active: None, @@ -88,21 +87,28 @@ impl StoredManagedCursorState { }), _ => None, }; - let mut individually_deleted_entries = BTreeSet::new(); + let mut individually_deleted_entries = RangeSet::new(); for range in info.individual_deleted_messages { let lower = range.lower_endpoint; let upper = range.upper_endpoint; - if lower.ledger_id != upper.ledger_id || lower.entry_id != upper.entry_id { + if lower.ledger_id != upper.ledger_id { return Err(anyhow!( - "cursor delete range spans multiple positions, which pulsar-lite does not support yet" + "cursor delete range spans multiple ledgers, which is unsupported" )); } - individually_deleted_entries.insert(ManagedLedgerPosition { - ledger_id: lower.ledger_id.try_into()?, - entry_id: lower.entry_id.try_into()?, - partition: -1, - }); + individually_deleted_entries.insert_range( + ManagedLedgerPosition { + ledger_id: lower.ledger_id.try_into()?, + entry_id: lower.entry_id.try_into()?, + partition: -1, + }, + ManagedLedgerPosition { + ledger_id: upper.ledger_id.try_into()?, + entry_id: upper.entry_id.try_into()?, + partition: -1, + }, + ); } Ok(Self { @@ -214,13 +220,17 @@ fn current_time_millis() -> u64 { .unwrap_or_default() } -fn position_range(position: &ManagedLedgerPosition) -> proto::MessageRange { - let endpoint = proto::NestedPositionInfo { - ledger_id: position.ledger_id as i64, - entry_id: position.entry_id as i64, +fn position_range(start: &ManagedLedgerPosition, end: &ManagedLedgerPosition) -> proto::MessageRange { + let lower_endpoint = proto::NestedPositionInfo { + ledger_id: start.ledger_id as i64, + entry_id: start.entry_id as i64, + }; + let upper_endpoint = proto::NestedPositionInfo { + ledger_id: end.ledger_id as i64, + entry_id: end.entry_id as i64, }; proto::MessageRange { - lower_endpoint: endpoint, - upper_endpoint: endpoint, + lower_endpoint, + upper_endpoint, } } diff --git a/rust/storage/managed-ledger-rocksdb/src/store.rs b/rust/storage/managed-ledger-rocksdb/src/store.rs index 1bdb73a..e0e3a25 100644 --- a/rust/storage/managed-ledger-rocksdb/src/store.rs +++ b/rust/storage/managed-ledger-rocksdb/src/store.rs @@ -2,18 +2,17 @@ use super::cursor::{ack_managed_cursor_shared, is_managed_position_acknowledged, use super::entrylog::EntryLogStore; use super::factory::{RocksDBManagedLedgerFactory, SharedLedger}; use super::keys; -use super::ledger::RocksDBManagedLedger; use super::write_queue::WriteQueue; use crate::cursor::first_position; use anyhow::{anyhow, Result}; -use rocksdb::{Options, DB}; +use rocksdb::{BlockBasedOptions, Cache, Options, DB}; use std::fmt; use std::path::Path; -use std::sync::{Arc, MutexGuard}; +use std::sync::{Arc, Mutex}; use pulsar_lite_storage_managed_ledger::{ - CursorInitOptions, CursorOpenResult, InitialPosition, ManagedCursor, ManagedLedger, - ManagedLedgerPosition, ManagedLedgerStorage, MessageId, StoredMessage, + CursorInitOptions, CursorOpenResult, InitialPosition, ManagedCursor, ManagedLedgerPosition, + ManagedLedgerStorage, MessageId, StoredMessage, }; /// Cloned handle that can append without holding `Mutex`. @@ -58,8 +57,9 @@ impl ConcurrentAppender { .map_err(|e| anyhow!(e)) } - /// Enqueue without waiting (BookKeeper logAddEntry style for connections). - /// Completion is delivered on `completion_tx` from the write-queue worker thread. + /// Enqueue without waiting. Completion is delivered on `completion_tx` + /// from the write-queue worker thread; `observer` (when present) is + /// invoked once per committed batch group with message/byte totals. pub fn enqueue_for_connection( &self, topic: &str, @@ -68,6 +68,7 @@ impl ConcurrentAppender { payload: &[u8], producer_id: u64, sequence_id: u64, + observer: Option>, completion_tx: tokio::sync::mpsc::Sender, ) -> Result<()> { WriteQueue::enqueue_for_connection( @@ -78,6 +79,7 @@ impl ConcurrentAppender { payload, producer_id, sequence_id, + observer, completion_tx, ) .map_err(|e| anyhow!(e)) @@ -94,6 +96,8 @@ impl fmt::Debug for ConcurrentAppender { pub struct RocksDbManagedLedgerStorage { factory: RocksDBManagedLedgerFactory, write_queue: WriteQueue, + /// Serializes cursor RMW (ack/seek/init). Must not cover entrylog IO. + cursor_mu: Mutex<()>, } impl fmt::Debug for RocksDbManagedLedgerStorage { @@ -109,6 +113,14 @@ impl RocksDbManagedLedgerStorage { pub fn open(path: &Path) -> Result { let mut options = Options::default(); options.create_if_missing(true); + options.set_max_open_files(512); + let mut block_opts = BlockBasedOptions::default(); + block_opts.set_block_cache(&Cache::new_lru_cache(256 * 1024 * 1024)); + block_opts.set_cache_index_and_filter_blocks(true); + options.set_block_based_table_factory(&block_opts); + options.set_write_buffer_size(64 * 1024 * 1024); + options.set_max_write_buffer_number(2); + options.set_max_background_jobs(4); let db = Arc::new(DB::open(&options, path)?); let entry_log = Arc::new(EntryLogStore::open(path)?); let factory = RocksDBManagedLedgerFactory::new(db, entry_log); @@ -116,6 +128,7 @@ impl RocksDbManagedLedgerStorage { Ok(Self { write_queue: WriteQueue::new(factory.clone()), factory, + cursor_mu: Mutex::new(()), }) } @@ -131,10 +144,12 @@ impl RocksDbManagedLedgerStorage { self.factory.open_ledger(&ledger_name) } - fn lock_ledger(ledger: &SharedLedger) -> Result> { - ledger + fn with_cursor_lock(&self, f: impl FnOnce() -> Result) -> Result { + let _guard = self + .cursor_mu .lock() - .map_err(|_| anyhow!("managed ledger lock poisoned")) + .map_err(|_| anyhow!("cursor lock poisoned"))?; + f() } fn cursor_exists(&self, topic: &str, subscription: &str) -> Result { @@ -144,25 +159,27 @@ impl RocksDbManagedLedgerStorage { } fn persist_empty_cursor(&self, topic: &str, subscription: &str) -> Result<()> { - let cursor_name = keys::encode_cursor_name(subscription); - let shared = self.topic_ledger(topic)?; - let mut ledger = Self::lock_ledger(&shared)?; - let cursor = ledger.open_cursor(&cursor_name)?; - cursor.persist_state() + self.with_cursor_lock(|| { + let cursor_name = keys::encode_cursor_name(subscription); + let ledger = self.topic_ledger(topic)?; + let cursor = ledger.open_cursor(&cursor_name)?; + cursor.persist_state() + }) } fn apply_latest_cursor(&self, topic: &str, subscription: &str) -> Result<()> { - let cursor_name = keys::encode_cursor_name(subscription); - let shared = self.topic_ledger(topic)?; - let mut ledger = Self::lock_ledger(&shared)?; - let last = ledger.last_position()?; - let mut cursor = ledger.open_cursor(&cursor_name)?; - - if let Some(last) = last { - cursor.mark_delete(last) - } else { - cursor.persist_state() - } + self.with_cursor_lock(|| { + let cursor_name = keys::encode_cursor_name(subscription); + let ledger = self.topic_ledger(topic)?; + let last = ledger.last_position()?; + let mut cursor = ledger.open_cursor(&cursor_name)?; + + if let Some(last) = last { + cursor.mark_delete(last) + } else { + cursor.persist_state() + } + }) } fn apply_start_message_id_cursor( @@ -171,19 +188,19 @@ impl RocksDbManagedLedgerStorage { subscription: &str, start: &MessageId, ) -> Result<()> { - let cursor_name = keys::encode_cursor_name(subscription); - let target = ManagedLedgerPosition::from(start); - let shared = self.topic_ledger(topic)?; - let mut ledger = Self::lock_ledger(&shared)?; - - let previous = ledger.previous_position(&target); - let mut cursor = ledger.open_cursor(&cursor_name)?; - - if let Some(previous) = previous { - cursor.mark_delete(previous) - } else { - cursor.persist_state() - } + self.with_cursor_lock(|| { + let cursor_name = keys::encode_cursor_name(subscription); + let target = ManagedLedgerPosition::from(start); + let ledger = self.topic_ledger(topic)?; + let previous = ledger.previous_position(&target); + let mut cursor = ledger.open_cursor(&cursor_name)?; + + if let Some(previous) = previous { + cursor.mark_delete(previous) + } else { + cursor.persist_state() + } + }) } } @@ -252,18 +269,14 @@ impl ManagedLedgerStorage for RocksDbManagedLedgerStorage { message_id: &MessageId, _shared: bool, ) -> Result<()> { - let cursor_name = keys::encode_cursor_name(subscription); - let position = ManagedLedgerPosition::from(message_id); - - let (mut cursor, marker_delete_posistion) = { - let shared = self.topic_ledger(topic)?; - let mut ledger = Self::lock_ledger(&shared)?; - + self.with_cursor_lock(|| { + let cursor_name = keys::encode_cursor_name(subscription); + let position = ManagedLedgerPosition::from(message_id); + let ledger = self.topic_ledger(topic)?; let mark_delete_position = ledger.previous_position(&position); - let cursor = ledger.open_cursor(&cursor_name)?; - (cursor, mark_delete_position) - }; - cursor.reset_cursor(marker_delete_posistion) + let mut cursor = ledger.open_cursor(&cursor_name)?; + cursor.reset_cursor(mark_delete_position) + }) } fn first_unacked_position( @@ -271,25 +284,33 @@ impl ManagedLedgerStorage for RocksDbManagedLedgerStorage { topic: &str, subscription: &str, ) -> Result> { - let cursor_name = keys::encode_cursor_name(subscription); - let shared = self.topic_ledger(topic)?; - let mut ledger = Self::lock_ledger(&shared)?; - - let cursor = ledger.open_cursor(&cursor_name)?; - let state = cursor.state(); + self.with_cursor_lock(|| { + let cursor_name = keys::encode_cursor_name(subscription); + let ledger = self.topic_ledger(topic)?; + let Some(lac) = ledger.last_position()? else { + return Ok(None); + }; - let mut candidate = match state.mark_delete.as_ref() { - Some(mark_delete) => next_position(mark_delete, &ledger.info), - None => first_position(&ledger.info, -1), - }; - - while let Some(position) = candidate { - if !is_managed_position_acknowledged(state, &position) { - return Ok(Some(position)); + let cursor = ledger.open_cursor(&cursor_name)?; + let state = cursor.state(); + let info = ledger.info_snapshot(); + + let mut candidate = match state.mark_delete.as_ref() { + Some(mark_delete) => next_position(mark_delete, info.as_ref()), + None => first_position(info.as_ref(), -1), + }; + + while let Some(position) = candidate { + if position > lac { + return Ok(None); + } + if !is_managed_position_acknowledged(state, &position) { + return Ok(Some(position)); + } + candidate = next_position(&position, info.as_ref()); } - candidate = next_position(&position, &ledger.info); - } - Ok(None) + Ok(None) + }) } fn read_from( @@ -298,9 +319,7 @@ impl ManagedLedgerStorage for RocksDbManagedLedgerStorage { from: &ManagedLedgerPosition, limit: usize, ) -> Result)>> { - let shared = self.topic_ledger(topic)?; - let ledger = Self::lock_ledger(&shared)?; - + let ledger = self.topic_ledger(topic)?; Ok(ledger .read_entries_from(from, limit)? .into_iter() @@ -314,15 +333,11 @@ impl ManagedLedgerStorage for RocksDbManagedLedgerStorage { from: &ManagedLedgerPosition, limit: usize, ) -> Result> { - let shared = self.topic_ledger(topic)?; - let ledger = Self::lock_ledger(&shared)?; - ledger.read_entries_from(from, limit) + self.topic_ledger(topic)?.read_entries_from(from, limit) } fn get_last_position(&self, topic: &str) -> Result> { - let shared = self.topic_ledger(topic)?; - let ledger = Self::lock_ledger(&shared)?; - ledger.last_position() + self.topic_ledger(topic)?.last_position() } fn get_next_position( @@ -330,9 +345,9 @@ impl ManagedLedgerStorage for RocksDbManagedLedgerStorage { topic: &str, current: &ManagedLedgerPosition, ) -> Result> { - let shared = self.topic_ledger(topic)?; - let ledger = Self::lock_ledger(&shared)?; - Ok(next_position(current, &ledger.info)) + let ledger = self.topic_ledger(topic)?; + let info = ledger.info_snapshot(); + Ok(next_position(current, info.as_ref())) } fn is_acknowledged( @@ -350,11 +365,12 @@ impl ManagedLedgerStorage for RocksDbManagedLedgerStorage { subscription: &str, message_id: MessageId, ) -> Result<()> { - let cursor_name = keys::encode_cursor_name(subscription); - let shared = self.topic_ledger(topic)?; - let mut ledger = Self::lock_ledger(&shared)?; - let mut cursor = ledger.open_cursor(&cursor_name)?; - cursor.mark_delete(ManagedLedgerPosition::from(message_id)) + self.with_cursor_lock(|| { + let cursor_name = keys::encode_cursor_name(subscription); + let ledger = self.topic_ledger(topic)?; + let mut cursor = ledger.open_cursor(&cursor_name)?; + cursor.mark_delete(ManagedLedgerPosition::from(message_id)) + }) } fn ack_message_shared( @@ -363,15 +379,17 @@ impl ManagedLedgerStorage for RocksDbManagedLedgerStorage { subscription: &str, message_id: MessageId, ) -> Result<()> { - let cursor_name = keys::encode_cursor_name(subscription); - let shared = self.topic_ledger(topic)?; - let mut ledger = Self::lock_ledger(&shared)?; - let mut cursor = ledger.open_cursor(&cursor_name)?; - ack_managed_cursor_shared( - &mut cursor, - ManagedLedgerPosition::from(message_id), - &ledger.info, - ) + self.with_cursor_lock(|| { + let cursor_name = keys::encode_cursor_name(subscription); + let ledger = self.topic_ledger(topic)?; + let mut cursor = ledger.open_cursor(&cursor_name)?; + let info = ledger.info_snapshot(); + ack_managed_cursor_shared( + &mut cursor, + ManagedLedgerPosition::from(message_id), + info.as_ref(), + ) + }) } fn get_message_by_id( @@ -379,9 +397,7 @@ impl ManagedLedgerStorage for RocksDbManagedLedgerStorage { topic: &str, message_id: &MessageId, ) -> Option<(MessageId, Vec)> { - let shared = self.topic_ledger(topic).ok()?; - let ledger = Self::lock_ledger(&shared).ok()?; - ledger.get_message_by_id(message_id) + self.topic_ledger(topic).ok()?.get_message_by_id(message_id) } fn get_message_entry_by_id( @@ -389,35 +405,13 @@ impl ManagedLedgerStorage for RocksDbManagedLedgerStorage { topic: &str, message_id: &MessageId, ) -> Option { - let shared = self.topic_ledger(topic).ok()?; - let ledger = Self::lock_ledger(&shared).ok()?; - ledger.get_message_entry_by_id(message_id) + self.topic_ledger(topic) + .ok()? + .get_message_entry_by_id(message_id) } fn get_messages(&self, topic: &str) -> Vec<(MessageId, Vec)> { - let shared = match self.topic_ledger(topic) { - Ok(shared) => shared, - Err(error) => { - log::error!( - "Failed to open managed ledger for topic '{}': {}", - topic, - error - ); - return Vec::new(); - } - }; - let ledger = match Self::lock_ledger(&shared) { - Ok(ledger) => ledger, - Err(error) => { - log::error!( - "Failed to lock managed ledger for topic '{}': {}", - topic, - error - ); - return Vec::new(); - } - }; - match ledger.messages() { + match self.topic_ledger(topic).and_then(|l| l.messages()) { Ok(messages) => messages, Err(error) => { log::error!( @@ -431,29 +425,7 @@ impl ManagedLedgerStorage for RocksDbManagedLedgerStorage { } fn get_message_entries(&self, topic: &str) -> Vec { - let shared = match self.topic_ledger(topic) { - Ok(shared) => shared, - Err(error) => { - log::error!( - "Failed to open managed ledger for topic '{}': {}", - topic, - error - ); - return Vec::new(); - } - }; - let ledger = match Self::lock_ledger(&shared) { - Ok(ledger) => ledger, - Err(error) => { - log::error!( - "Failed to lock managed ledger for topic '{}': {}", - topic, - error - ); - return Vec::new(); - } - }; - match ledger.message_entries() { + match self.topic_ledger(topic).and_then(|l| l.message_entries()) { Ok(entries) => entries, Err(error) => { log::error!( @@ -472,13 +444,11 @@ impl ManagedLedgerStorage for RocksDbManagedLedgerStorage { subscription: &str, message_id: &MessageId, ) -> bool { - let shared = match self.topic_ledger(topic) { - Ok(shared) => shared, - Err(_) => return false, + let Ok(_guard) = self.cursor_mu.lock() else { + return false; }; - let mut ledger = match Self::lock_ledger(&shared) { - Ok(ledger) => ledger, - Err(_) => return false, + let Ok(ledger) = self.topic_ledger(topic) else { + return false; }; let cursor_name = keys::encode_cursor_name(subscription); ledger @@ -493,17 +463,54 @@ impl ManagedLedgerStorage for RocksDbManagedLedgerStorage { } fn get_mark_delete_position(&self, topic: &str, subscription: &str) -> Option { - let cursor_name = keys::encode_cursor_name(subscription); - let shared = self.topic_ledger(topic).ok()?; - let mut ledger = match Self::lock_ledger(&shared) { - Ok(ledger) => ledger, - Err(_) => return None, - }; - let cursor = ledger.open_cursor(&cursor_name).ok()?; + let _guard = self.cursor_mu.lock().ok()?; + let ledger = self.topic_ledger(topic).ok()?; + let cursor = ledger + .open_cursor(&keys::encode_cursor_name(subscription)) + .ok()?; cursor .state() .mark_delete .as_ref() .map(|position| position.entry_id) } + + fn backlog_entries(&self, topic: &str, subscription: &str) -> Option { + let _guard = self.cursor_mu.lock().ok()?; + let ledger = self.topic_ledger(topic).ok()?; + let info = ledger.info_snapshot(); + let total: u64 = info.ledgers.iter().map(|l| l.entries).sum(); + + let cursor = ledger + .open_cursor(&keys::encode_cursor_name(subscription)) + .ok()?; + let state = cursor.state(); + let Some(mark) = state.mark_delete.as_ref() else { + // Cursor never advanced: nothing acknowledged yet. + return Some(total); + }; + // Acked = everything in fully-retired ledgers below the mark's + // ledger, the mark itself, and individually-deleted holes above it. + let retired: u64 = info + .ledgers + .iter() + .filter(|l| l.ledger_id < mark.ledger_id) + .map(|l| l.entries) + .sum(); + let holes: u64 = state + .individually_deleted_entries + .iter() + .map(|(start, end)| end.entry_id.saturating_sub(start.entry_id) + 1) + .sum(); + let acked = retired + mark.entry_id + 1 + holes; + Some(total.saturating_sub(acked)) + } + + fn stored_bytes(&self, topic: &str) -> u64 { + let Ok(ledger) = self.topic_ledger(topic) else { + return 0; + }; + let info = ledger.info_snapshot(); + info.ledgers.iter().map(|l| l.size).sum() + } } diff --git a/rust/storage/managed-ledger-rocksdb/src/write_queue.rs b/rust/storage/managed-ledger-rocksdb/src/write_queue.rs index bbdb04c..24021a1 100644 --- a/rust/storage/managed-ledger-rocksdb/src/write_queue.rs +++ b/rust/storage/managed-ledger-rocksdb/src/write_queue.rs @@ -1,7 +1,9 @@ -use crate::factory::RocksDBManagedLedgerFactory; +use crate::factory::{RocksDBManagedLedgerFactory, SharedLedger}; use crate::keys; +use pulsar_lite_metrics::PublishCommitObserver; use pulsar_lite_storage_managed_ledger::MessageId; use std::collections::HashMap; +use std::sync::Arc; use std::sync::mpsc; use std::thread::{self, JoinHandle}; use std::time::{Duration, Instant}; @@ -59,14 +61,18 @@ pub(crate) struct WriteReq { partition: i32, metadata: Vec, payload: Vec, + observer: Option>, + /// Set on the connection path; anchors the end-to-end publish latency + /// histogram (enqueue → committed batch). + enqueued_at: Option, reply: WriteReply, } /// Single-writer queue for managed-ledger appends. /// -/// Worker owns ledgers in a thread-local HashMap and writes via `&mut` -/// (no SharedLedger lock on the append path). Reply uses tokio oneshot so -/// async callers can await without blocking the Tokio worker thread. +/// Worker caches the same `Arc` handles as store reads so +/// durable success updates one published LAC/meta view. Entry-id assignment stays +/// serial on this thread; no outer content mutex around append IO. pub(crate) struct WriteQueue { tx: Option>, worker: Option>, @@ -90,16 +96,11 @@ impl WriteQueue { } fn worker_loop(factory: RocksDBManagedLedgerFactory, rx: mpsc::Receiver) { - // Batch-size metrics (aggregated to avoid log spam under stress). - let mut metric_batches: u64 = 0; - let mut metric_msgs: u64 = 0; - let mut metric_max_batch: usize = 0; - let mut metric_batch_eq1: u64 = 0; - let mut metric_group_ops: u64 = 0; - let mut metric_group_msgs: u64 = 0; - let mut metric_max_group: usize = 0; - let mut metric_group_eq1: u64 = 0; - let mut metric_window_started = Instant::now(); + // Same Arc instances as store reads (singleton per ledger name). + let mut ledgers: HashMap = HashMap::new(); + + // Prometheus families (no-op before metrics::init). + let metrics = pulsar_lite_metrics::storage_metrics(); loop { let first = match rx.recv() { @@ -146,12 +147,7 @@ impl WriteQueue { } let queue_batch_len = batch.len(); - metric_batches += 1; - metric_msgs += queue_batch_len as u64; - metric_max_batch = metric_max_batch.max(queue_batch_len); - if queue_batch_len == 1 { - metric_batch_eq1 += 1; - } + metrics.observe_batch(queue_batch_len as u64); // Group by ledger so one rocksdb write covers many entries of the same topic. let mut order: Vec = Vec::new(); @@ -170,34 +166,28 @@ impl WriteQueue { _ => continue, }; - let group_len = reqs.len(); - metric_group_ops += 1; - metric_group_msgs += group_len as u64; - metric_max_group = metric_max_group.max(group_len); - if group_len == 1 { - metric_group_eq1 += 1; - } - // Route the batch through the factory's SHARED ledger cache: - // the worker-owned ledger keeps its runtime state (LAC, - // entry counters) private in memory, so appends committed - // through it were invisible to readers using the shared - // ledger until the LAC-ArcSwap rework. Locking the shared - // ledger unifies the in-memory state; batching (one entrylog - // flush + one RocksDB write per group) is preserved. - let shared = match factory.open_ledger(&ledger_name) { - Ok(shared) => shared, - Err(e) => { - let msg = e.to_string(); - for req in reqs { - req.reply.complete(Err(msg.clone())); + if !ledgers.contains_key(&ledger_name) { + match factory.open_ledger(&ledger_name) { + Ok(ledger) => { + ledgers.insert(ledger_name.clone(), ledger); + } + Err(e) => { + let msg = e.to_string(); + for req in reqs { + req.reply.complete(Err(msg.clone())); + } + continue; } - continue; } - }; - let mut ledger = match shared.lock() { - Ok(guard) => guard, - Err(poisoned) => poisoned.into_inner(), + } + + let Some(ledger) = ledgers.get(&ledger_name) else { + for req in reqs { + req.reply + .complete(Err("ledger missing from worker cache".to_string())); + } + continue; }; // Borrow metadata/payload only while reqs is still alive. @@ -206,8 +196,43 @@ impl WriteQueue { .map(|r| (r.partition, r.metadata.as_slice(), r.payload.as_slice())) .collect(); - match ledger.add_entries_with_partition_and_metadata(&inputs) { + // Append publishes meta then LAC only after durable OK; + // complete only after that returns Ok. + let append_started = Instant::now(); + let append_result = + ledger.add_entries_with_partition_and_metadata(&inputs); + metrics.observe_ledger_write_latency(append_started.elapsed().as_secs_f64()); + match append_result { Ok(positions) => { + // One observer call per committed group: all reqs in + // a group share the topic, so the first req's observer + // already targets the right counters. + let mut committed_messages: u64 = 0; + let mut committed_bytes: u64 = 0; + let mut observer: Option> = None; + let committed_at = Instant::now(); + for req in &reqs { + // Batched producers pack N client-visible messages + // into one request; counters must fold N, not 1 + // (bytes already fold the full payload). + committed_messages += + pulsar_lite_proto::codec::messages_in_batch(&req.metadata) + as u64; + let req_bytes = (req.metadata.len() + req.payload.len()) as u64; + committed_bytes += req_bytes; + metrics.observe_entry_size(req_bytes as f64); + if let Some(enqueued_at) = req.enqueued_at { + metrics.observe_write_latency( + committed_at.duration_since(enqueued_at).as_secs_f64(), + ); + } + if observer.is_none() { + observer = req.observer.clone(); + } + } + if let Some(observer) = observer { + observer.on_commit(committed_messages, committed_bytes); + } for (req, position) in reqs.into_iter().zip(positions) { req.reply.complete(Ok(MessageId::from(position))); } @@ -221,53 +246,6 @@ impl WriteQueue { } } - // Emit ~1Hz summary so stress runs stay readable. - if metric_window_started.elapsed() >= Duration::from_secs(1) { - let avg_queue = if metric_batches == 0 { - 0.0 - } else { - metric_msgs as f64 / metric_batches as f64 - }; - let avg_group = if metric_group_ops == 0 { - 0.0 - } else { - metric_group_msgs as f64 / metric_group_ops as f64 - }; - let pct_queue_eq1 = if metric_batches == 0 { - 0.0 - } else { - 100.0 * metric_batch_eq1 as f64 / metric_batches as f64 - }; - let pct_group_eq1 = if metric_group_ops == 0 { - 0.0 - } else { - 100.0 * metric_group_eq1 as f64 / metric_group_ops as f64 - }; - - log::info!( - "write_queue metrics: queue_batches={} queue_msgs={} queue_batch_avg={:.2} queue_batch_max={} queue_batch_eq1={:.1}% group_ops={} group_msgs={} group_avg={:.2} group_max={} group_eq1={:.1}%", - metric_batches, - metric_msgs, - avg_queue, - metric_max_batch, - pct_queue_eq1, - metric_group_ops, - metric_group_msgs, - avg_group, - metric_max_group, - pct_group_eq1, - ); - - metric_batches = 0; - metric_msgs = 0; - metric_max_batch = 0; - metric_batch_eq1 = 0; - metric_group_ops = 0; - metric_group_msgs = 0; - metric_max_group = 0; - metric_group_eq1 = 0; - metric_window_started = Instant::now(); - } } } @@ -292,13 +270,14 @@ impl WriteQueue { partition, metadata: metadata.to_vec(), payload: payload.to_vec(), + observer: None, + enqueued_at: None, reply: WriteReply::Oneshot(reply_tx), }) .map_err(|_| "write queue worker disconnected".to_string())?; Ok(reply_rx) } - /// Enqueue for a broker connection: returns immediately; completion is pushed to `completion_tx`. pub(crate) fn enqueue_for_connection( tx: &mpsc::Sender, topic: &str, @@ -307,6 +286,7 @@ impl WriteQueue { payload: &[u8], producer_id: u64, sequence_id: u64, + observer: Option>, completion_tx: tokio::sync::mpsc::Sender, ) -> Result<(), String> { tx.send(WriteReq { @@ -314,6 +294,8 @@ impl WriteQueue { partition, metadata: metadata.to_vec(), payload: payload.to_vec(), + observer, + enqueued_at: Some(Instant::now()), reply: WriteReply::Conn { producer_id, sequence_id, diff --git a/rust/storage/managed-ledger-rocksdb/tests/backlog_entries.rs b/rust/storage/managed-ledger-rocksdb/tests/backlog_entries.rs new file mode 100644 index 0000000..98fb9c3 --- /dev/null +++ b/rust/storage/managed-ledger-rocksdb/tests/backlog_entries.rs @@ -0,0 +1,63 @@ +use pulsar_lite_storage_managed_ledger::{ + CursorInitOptions, InitialPosition, ManagedLedgerStorage, +}; +use pulsar_lite_storage_managed_ledger_rocksdb::RocksDbManagedLedgerStorage; +use tempfile::tempdir; + +fn open(path: &std::path::Path) -> RocksDbManagedLedgerStorage { + RocksDbManagedLedgerStorage::open(path).expect("open rocksdb store") +} + +#[test] +fn backlog_tracks_shared_acks_and_full_drain() { + let dir = tempdir().expect("tempdir"); + let topic = "persistent://public/default/backlog-probe"; + + { + let mut store = open(dir.path()); + store.create_topic(topic).expect("create topic"); + store + .initialize_or_open_cursor( + topic, + "sub", + CursorInitOptions { + initial_position: InitialPosition::Earliest, + ..Default::default() + }, + ) + .expect("open cursor"); + for i in 0..10u64 { + store + .append_message(topic, -1, format!("m{i}").as_bytes()) + .expect("append"); + } + assert_eq!(store.backlog_entries(topic, "sub"), Some(10)); + + // Ack entries 0..3 under Shared semantics. + for entry in 0..4u64 { + let id = pulsar_lite_storage_managed_ledger::MessageId { + ledger: store + .get_last_position(topic) + .expect("last position") + .map(|p| p.ledger_id) + .unwrap_or_default(), + entry, + partition: -1, + }; + store + .ack_message_shared(topic, "sub", id) + .expect("ack shared"); + } + assert_eq!( + store.backlog_entries(topic, "sub"), + Some(6), + "4 acked of 10 must leave backlog 6" + ); + } + + // Reopen: backlog must survive restart. + { + let store = open(dir.path()); + assert_eq!(store.backlog_entries(topic, "sub"), Some(6)); + } +} diff --git a/rust/storage/managed-ledger-rocksdb/tests/rocksdb_cursor.rs b/rust/storage/managed-ledger-rocksdb/tests/rocksdb_cursor.rs index ab9c912..e5e66ed 100644 --- a/rust/storage/managed-ledger-rocksdb/tests/rocksdb_cursor.rs +++ b/rust/storage/managed-ledger-rocksdb/tests/rocksdb_cursor.rs @@ -3,13 +3,14 @@ mod common; use common::*; -use pulsar_lite_storage_managed_ledger::{ManagedCursor, ManagedLedger}; +use pulsar_lite_storage_managed_ledger::ManagedCursor; use pulsar_lite_storage_managed_ledger::{ ManagedLedgerConfig, ManagedLedgerFactory, ManagedLedgerPosition, }; use pulsar_lite_storage_managed_ledger_rocksdb::test_support::{ - ack_managed_cursor_shared, is_managed_position_acknowledged, RocksDBManagedCursor, - RocksDBManagedLedger, RocksDBManagedLedgerFactory, + ack_managed_cursor_shared, append_payload, append_with_partition, + is_managed_position_acknowledged, RocksDBManagedCursor, RocksDBManagedLedger, + RocksDBManagedLedgerFactory, }; use std::sync::Arc; use tempfile::tempdir; @@ -91,18 +92,19 @@ fn shared_ack_advances_contiguously_across_rolled_ledgers() { }; let entry_log = open_test_entry_log(&db_path); let mut factory = RocksDBManagedLedgerFactory::new(Arc::clone(&db), entry_log); - let mut ledger = factory.open("ledger-a", &config).unwrap(); - let first = ledger.add_entry(b"first").unwrap(); - let second = ledger.add_entry(b"second").unwrap(); - let third = ledger.add_entry(b"third").unwrap(); + let ledger = factory.open("ledger-a", &config).unwrap(); + let first = append_payload(&ledger, b"first").unwrap(); + let second = append_payload(&ledger, b"second").unwrap(); + let third = append_payload(&ledger, b"third").unwrap(); let mut cursor = ledger.open_cursor("sub-a").unwrap(); - ack_managed_cursor_shared(&mut cursor, third.clone(), &ledger.ledger_info()).unwrap(); + ack_managed_cursor_shared(&mut cursor, third.clone(), &ledger.info_snapshot().as_ref()) + .unwrap(); assert_eq!(cursor.state().mark_delete, None); assert!(cursor.state().individually_deleted_entries.contains(&third)); - ack_managed_cursor_shared(&mut cursor, first, &ledger.ledger_info()).unwrap(); - ack_managed_cursor_shared(&mut cursor, second, &ledger.ledger_info()).unwrap(); + ack_managed_cursor_shared(&mut cursor, first, &ledger.info_snapshot().as_ref()).unwrap(); + ack_managed_cursor_shared(&mut cursor, second, &ledger.info_snapshot().as_ref()).unwrap(); assert_eq!(cursor.state().mark_delete, Some(third)); assert!(cursor.state().individually_deleted_entries.is_empty()); @@ -116,14 +118,14 @@ fn managed_ledger_open_cursor_recovers_cursor_state() { { let db = open_test_db(&db_path); let entry_log = open_test_entry_log(&db_path); - let mut ledger = RocksDBManagedLedger::open("ledger-a", db, entry_log).unwrap(); + let ledger = RocksDBManagedLedger::open("ledger-a", db, entry_log).unwrap(); let mut cursor = ledger.open_cursor("sub-a").unwrap(); cursor.mark_delete(mark_delete.clone()).unwrap(); } let db = open_test_db(&db_path); let entry_log = open_test_entry_log(&db_path); - let mut ledger = RocksDBManagedLedger::open("ledger-a", db, entry_log).unwrap(); + let ledger = RocksDBManagedLedger::open("ledger-a", db, entry_log).unwrap(); let cursor = ledger.open_cursor("sub-a").unwrap(); assert_eq!(cursor.state().mark_delete, Some(mark_delete)); @@ -259,16 +261,17 @@ fn shared_ack_normalizes_partition_when_advancing_mark_delete() { let db = open_test_db(&db_path); let entry_log = open_test_entry_log(&db_path); - let mut ledger = RocksDBManagedLedger::open("ledger-a", Arc::clone(&db), entry_log).unwrap(); + let ledger = RocksDBManagedLedger::open("ledger-a", Arc::clone(&db), entry_log).unwrap(); - let first = ledger.add_entry_with_partition(7, b"first").unwrap(); + let first = append_with_partition(&ledger, 7, b"first").unwrap(); - let second = ledger.add_entry_with_partition(7, b"second").unwrap(); + let second = append_with_partition(&ledger, 7, b"second").unwrap(); let mut cursor = ledger.open_cursor("sub-a").unwrap(); // First, reorder ACK item 'second'. It should enter the individual collection - ack_managed_cursor_shared(&mut cursor, second.clone(), ledger.ledger_info()).unwrap(); + ack_managed_cursor_shared(&mut cursor, second.clone(), ledger.info_snapshot().as_ref()) + .unwrap(); assert_eq!(cursor.state().mark_delete, None); assert!(cursor @@ -281,7 +284,7 @@ fn shared_ack_normalizes_partition_when_advancing_mark_delete() { })); // Then move on to the first item, 'mark_delete'proceed forward - ack_managed_cursor_shared(&mut cursor, first.clone(), ledger.ledger_info()).unwrap(); + ack_managed_cursor_shared(&mut cursor, first.clone(), ledger.info_snapshot().as_ref()).unwrap(); assert_eq!( cursor.state().mark_delete, Some(ManagedLedgerPosition { diff --git a/rust/storage/managed-ledger-rocksdb/tests/rocksdb_entrylog.rs b/rust/storage/managed-ledger-rocksdb/tests/rocksdb_entrylog.rs index 86726d4..f43d3e9 100644 --- a/rust/storage/managed-ledger-rocksdb/tests/rocksdb_entrylog.rs +++ b/rust/storage/managed-ledger-rocksdb/tests/rocksdb_entrylog.rs @@ -4,7 +4,6 @@ use pulsar_lite_storage_managed_ledger_rocksdb::test_support::{EntryLogStore, En use std::fs; use tempfile::tempdir; - #[test] fn entrylog_appends_and_reads_entry_payload() { let dir = tempdir().unwrap(); @@ -296,4 +295,3 @@ fn entrylog_append_batch_then_single_append_continues_offsets() { assert_eq!(single.offset, batch[1].offset + batch[1].len); assert_eq!(store.read(&single).unwrap().payload, b"c"); } - diff --git a/rust/storage/managed-ledger-rocksdb/tests/rocksdb_keys.rs b/rust/storage/managed-ledger-rocksdb/tests/rocksdb_keys.rs index 5839d12..fe747c0 100644 --- a/rust/storage/managed-ledger-rocksdb/tests/rocksdb_keys.rs +++ b/rust/storage/managed-ledger-rocksdb/tests/rocksdb_keys.rs @@ -4,9 +4,9 @@ mod common; use common::*; use prost::Message; -use pulsar_lite_storage_managed_ledger::{ManagedCursor, ManagedLedger}; +use pulsar_lite_storage_managed_ledger::ManagedCursor; use pulsar_lite_storage_managed_ledger_rocksdb::test_support::{ - keys, proto, RocksDBManagedCursor, RocksDBManagedLedger, + append_payload, keys, proto, RocksDBManagedCursor, RocksDBManagedLedger, }; use std::sync::Arc; use tempfile::tempdir; @@ -36,9 +36,8 @@ fn managed_ledger_info_value_is_protobuf_encoded() { { let entry_log = open_test_entry_log(&db_path); - let mut ledger = - RocksDBManagedLedger::open("ledger-a", Arc::clone(&db), entry_log).unwrap(); - ledger.add_entry(b"first").unwrap(); + let ledger = RocksDBManagedLedger::open("ledger-a", Arc::clone(&db), entry_log).unwrap(); + append_payload(&ledger, b"first").unwrap(); } let bytes = read_raw_value(&db, keys::managed_ledger_key("ledger-a")); diff --git a/rust/storage/managed-ledger-rocksdb/tests/rocksdb_ledger.rs b/rust/storage/managed-ledger-rocksdb/tests/rocksdb_ledger.rs index 6835248..9350eab 100644 --- a/rust/storage/managed-ledger-rocksdb/tests/rocksdb_ledger.rs +++ b/rust/storage/managed-ledger-rocksdb/tests/rocksdb_ledger.rs @@ -7,8 +7,8 @@ use pulsar_lite_storage_managed_ledger::{ ManagedLedger, ManagedLedgerConfig, ManagedLedgerFactory, }; use pulsar_lite_storage_managed_ledger_rocksdb::test_support::{ - keys, RocksDBManagedLedger, RocksDBManagedLedgerFactory, StoredEntryLocation, - StoredManagedLedgerInfo, + append_payload, append_with_partition, keys, RocksDBManagedLedger, RocksDBManagedLedgerFactory, + StoredEntryLocation, StoredManagedLedgerInfo, }; use std::sync::Arc; use tempfile::tempdir; @@ -21,8 +21,8 @@ fn managed_ledger_entry_recovers_after_reopen() { let first_position = { let db = open_test_db(&db_path); let entry_log = open_test_entry_log(&db_path); - let mut ledger = RocksDBManagedLedger::open("ledger-a", db, entry_log).unwrap(); - ledger.add_entry(b"first").unwrap() + let ledger = RocksDBManagedLedger::open("ledger-a", db, entry_log).unwrap(); + append_payload(&ledger, b"first").unwrap() }; let db = open_test_db(&db_path); @@ -72,8 +72,8 @@ fn managed_ledger_entry_value_stores_location_not_payload() { let entry_log = open_test_entry_log(&db_path); let payload = b"payload-in-entrylog"; - let mut ledger = RocksDBManagedLedger::open("ledger-a", Arc::clone(&db), entry_log).unwrap(); - let position = ledger.add_entry(payload).unwrap(); + let ledger = RocksDBManagedLedger::open("ledger-a", Arc::clone(&db), entry_log).unwrap(); + let position = append_payload(&ledger, payload).unwrap(); let raw_value = read_raw_value( &db, @@ -97,9 +97,8 @@ fn managed_ledger_returns_none_for_bad_entry_location() { let entry_log = open_test_entry_log(&db_path); let position = { - let mut ledger = - RocksDBManagedLedger::open("ledger-a", Arc::clone(&db), entry_log).unwrap(); - ledger.add_entry(b"payload").unwrap() + let ledger = RocksDBManagedLedger::open("ledger-a", Arc::clone(&db), entry_log).unwrap(); + append_payload(&ledger, b"payload").unwrap() }; let mut location: StoredEntryLocation = bincode::deserialize(&read_raw_value( @@ -129,15 +128,15 @@ fn managed_ledger_next_entry_id_is_derived_from_last_ledger_entries() { { let db = open_test_db(&db_path); let entry_log = open_test_entry_log(&db_path); - let mut ledger = RocksDBManagedLedger::open("ledger-a", db, entry_log).unwrap(); - assert_eq!(ledger.add_entry(b"first").unwrap().entry_id, 0); - assert_eq!(ledger.add_entry(b"second").unwrap().entry_id, 1); + let ledger = RocksDBManagedLedger::open("ledger-a", db, entry_log).unwrap(); + assert_eq!(append_payload(&ledger, b"first").unwrap().entry_id, 0); + assert_eq!(append_payload(&ledger, b"second").unwrap().entry_id, 1); } let db = open_test_db(&db_path); let entry_log = open_test_entry_log(&db_path); - let mut ledger = RocksDBManagedLedger::open("ledger-a", db, entry_log).unwrap(); - let third_position = ledger.add_entry(b"third").unwrap(); + let ledger = RocksDBManagedLedger::open("ledger-a", db, entry_log).unwrap(); + let third_position = append_payload(&ledger, b"third").unwrap(); assert_eq!(third_position.ledger_id, 0); assert_eq!(third_position.entry_id, 2); @@ -160,19 +159,19 @@ fn managed_ledger_rolls_over_after_max_entries_like_pulsar() { let mut factory = RocksDBManagedLedgerFactory::new(Arc::clone(&db), entry_log); { - let mut ledger = factory.open("ledger-a", &config).unwrap(); - assert_eq!(ledger.add_entry(b"first").unwrap(), position(0, 0)); - assert_eq!(ledger.add_entry(b"second").unwrap(), position(0, 1)); - assert_eq!(ledger.add_entry(b"third").unwrap(), position(1, 0)); + let ledger = factory.open("ledger-a", &config).unwrap(); + assert_eq!(append_payload(&ledger, b"first").unwrap(), position(0, 0)); + assert_eq!(append_payload(&ledger, b"second").unwrap(), position(0, 1)); + assert_eq!(append_payload(&ledger, b"third").unwrap(), position(1, 0)); } let ledger = factory.open("ledger-a", &config).unwrap(); - assert_eq!(ledger.ledger_info().ledgers.len(), 2); - assert_eq!(ledger.ledger_info().ledgers[0].ledger_id, 0); - assert_eq!(ledger.ledger_info().ledgers[0].entries, 2); - assert_eq!(ledger.ledger_info().ledgers[1].ledger_id, 1); - assert_eq!(ledger.ledger_info().ledgers[1].entries, 1); + assert_eq!(ledger.info_snapshot().ledgers.len(), 2); + assert_eq!(ledger.info_snapshot().ledgers[0].ledger_id, 0); + assert_eq!(ledger.info_snapshot().ledgers[0].entries, 2); + assert_eq!(ledger.info_snapshot().ledgers[1].ledger_id, 1); + assert_eq!(ledger.info_snapshot().ledgers[1].entries, 1); assert_eq!( ledger.read_entry(&position(0, 0)).as_deref(), Some(b"first".as_slice()) @@ -200,10 +199,10 @@ fn managed_ledger_rollover_metadata_is_persisted_in_rocksdb() { let mut factory = RocksDBManagedLedgerFactory::new(Arc::clone(&db), entry_log); { - let mut ledger = factory.open("ledger-a", &config).unwrap(); - ledger.add_entry(b"first").unwrap(); - ledger.add_entry(b"second").unwrap(); - ledger.add_entry(b"third").unwrap(); + let ledger = factory.open("ledger-a", &config).unwrap(); + append_payload(&ledger, b"first").unwrap(); + append_payload(&ledger, b"second").unwrap(); + append_payload(&ledger, b"third").unwrap(); } let info = read_managed_ledger_info(&db, "ledger-a"); @@ -235,19 +234,19 @@ fn managed_ledger_reopen_continues_from_persisted_rollover_metadata() { let db = open_test_db(&db_path); let entry_log = open_test_entry_log(&db_path); let mut factory = RocksDBManagedLedgerFactory::new(db, entry_log); - let mut ledger = factory.open("ledger-a", &config).unwrap(); - assert_eq!(ledger.add_entry(b"first").unwrap(), position(0, 0)); - assert_eq!(ledger.add_entry(b"second").unwrap(), position(0, 1)); + let ledger = factory.open("ledger-a", &config).unwrap(); + assert_eq!(append_payload(&ledger, b"first").unwrap(), position(0, 0)); + assert_eq!(append_payload(&ledger, b"second").unwrap(), position(0, 1)); } { let db = open_test_db(&db_path); let entry_log = open_test_entry_log(&db_path); let mut factory = RocksDBManagedLedgerFactory::new(Arc::clone(&db), entry_log); - let mut ledger = factory.open("ledger-a", &config).unwrap(); - assert_eq!(ledger.add_entry(b"third").unwrap(), position(1, 0)); - assert_eq!(ledger.add_entry(b"fourth").unwrap(), position(1, 1)); - assert_eq!(ledger.add_entry(b"fifth").unwrap(), position(2, 0)); + let ledger = factory.open("ledger-a", &config).unwrap(); + assert_eq!(append_payload(&ledger, b"third").unwrap(), position(1, 0)); + assert_eq!(append_payload(&ledger, b"fourth").unwrap(), position(1, 1)); + assert_eq!(append_payload(&ledger, b"fifth").unwrap(), position(2, 0)); let info = read_managed_ledger_info(&db, "ledger-a"); assert_eq!(info.ledgers.len(), 3); @@ -281,21 +280,21 @@ fn managed_ledger_ids_are_global_across_topics() { let db = open_test_db(&db_path); let entry_log = open_test_entry_log(&db_path); - let mut orders = RocksDBManagedLedger::open( + let orders = RocksDBManagedLedger::open( "public/default/persistent/orders", Arc::clone(&db), Arc::clone(&entry_log), ) .unwrap(); - let mut payments = RocksDBManagedLedger::open( + let payments = RocksDBManagedLedger::open( "public/default/persistent/payments", Arc::clone(&db), Arc::clone(&entry_log), ) .unwrap(); - let orders_position = orders.add_entry(b"order-1").unwrap(); - let payments_position = payments.add_entry(b"payment-1").unwrap(); + let orders_position = append_payload(&orders, b"order-1").unwrap(); + let payments_position = append_payload(&payments, b"payment-1").unwrap(); assert_ne!(orders_position.ledger_id, payments_position.ledger_id); assert_eq!(orders_position.entry_id, 0); @@ -335,12 +334,12 @@ fn rolled_ledgers_allocate_global_ledger_ids() { let entry_log = open_test_entry_log(&db_path); let mut factory = RocksDBManagedLedgerFactory::new(Arc::clone(&db), entry_log); - let mut orders = factory.open("orders", &config).unwrap(); - let mut payments = factory.open("payments", &config).unwrap(); + let orders = factory.open("orders", &config).unwrap(); + let payments = factory.open("payments", &config).unwrap(); - let orders_first = orders.add_entry(b"order-1").unwrap(); - let payments_first = payments.add_entry(b"payment-1").unwrap(); - let orders_second = orders.add_entry(b"order-2").unwrap(); + let orders_first = append_payload(&orders, b"order-1").unwrap(); + let payments_first = append_payload(&payments, b"payment-1").unwrap(); + let orders_second = append_payload(&orders, b"order-2").unwrap(); assert_ne!(orders_first.ledger_id, payments_first.ledger_id); assert_ne!(orders_second.ledger_id, orders_first.ledger_id); @@ -366,10 +365,10 @@ fn previous_position_handles_same_ledger_cross_ledger_and_before_first() { let entry_log = open_test_entry_log(&db_path); let mut factory = RocksDBManagedLedgerFactory::new(Arc::clone(&db), entry_log); - let mut ledger = factory.open("ledger-a", &config).unwrap(); - ledger.add_entry(b"first").unwrap(); - ledger.add_entry(b"second").unwrap(); - ledger.add_entry(b"third").unwrap(); + let ledger = factory.open("ledger-a", &config).unwrap(); + append_payload(&ledger, b"first").unwrap(); + append_payload(&ledger, b"second").unwrap(); + append_payload(&ledger, b"third").unwrap(); // phase 1: entry_id > 0 -> same ledger assert_eq!( @@ -390,10 +389,10 @@ fn read_entries_from_respects_limit() { let dir = tempdir().unwrap(); let db = open_test_db(dir.path()); let entry_log = open_test_entry_log(dir.path()); - let mut ledger = RocksDBManagedLedger::open("ledger-1", db, entry_log).unwrap(); - ledger.add_entry(b"first").unwrap(); - ledger.add_entry(b"second").unwrap(); - ledger.add_entry(b"third").unwrap(); + let ledger = RocksDBManagedLedger::open("ledger-1", db, entry_log).unwrap(); + append_payload(&ledger, b"first").unwrap(); + append_payload(&ledger, b"second").unwrap(); + append_payload(&ledger, b"third").unwrap(); let entries = ledger.read_entries_from(&position(0, 0), 2).unwrap(); assert_eq!(entries.len(), 2); assert_eq!(entries[0].payload, "first".as_bytes()); @@ -408,8 +407,8 @@ fn managed_ledger_open_initializes_last_position_runtime_state() { let expected_position = { let db = open_test_db(&db_path); let entry_log = open_test_entry_log(&db_path); - let mut ledger = RocksDBManagedLedger::open("ledger-a", db, entry_log).unwrap(); - ledger.add_entry_with_partition(7, b"first").unwrap() + let ledger = RocksDBManagedLedger::open("ledger-a", db, entry_log).unwrap(); + append_with_partition(&ledger, 7, b"first").unwrap() }; let db = open_test_db(&db_path); @@ -435,10 +434,10 @@ fn message_ledger_append_updates_runtime_last_position() { let db = open_test_db(&db_path); let entry_log = open_test_entry_log(&db_path); - let mut ledger = + let ledger = RocksDBManagedLedger::open("ledger-a", Arc::clone(&db), Arc::clone(&entry_log)).unwrap(); - let first = ledger.add_entry_with_partition(7, b"first").unwrap(); + let first = append_with_partition(&ledger, 7, b"first").unwrap(); // Delete the index and ensure that last_position cannot be obtained through a temporary query in RocksDB. db.delete(keys::managed_entry_key(first.ledger_id, first.entry_id)) @@ -446,11 +445,11 @@ fn message_ledger_append_updates_runtime_last_position() { assert_eq!(ledger.last_position().unwrap(), Some(first)); // Consecutive append - advancing to the last position in the runtime. - let mut ledger = + let ledger = RocksDBManagedLedger::open("ledger-b", Arc::clone(&db), Arc::clone(&entry_log)).unwrap(); - let first = ledger.add_entry_with_partition(7, b"first").unwrap(); - let second = ledger.add_entry_with_partition(7, b"second").unwrap(); + let first = append_with_partition(&ledger, 7, b"first").unwrap(); + let second = append_with_partition(&ledger, 7, b"second").unwrap(); assert_eq!(ledger.last_position().unwrap(), Some(second.clone()),); assert_eq!(second.ledger_id, first.ledger_id); assert_eq!(second.entry_id, first.entry_id + 1); @@ -467,7 +466,7 @@ fn managed_ledger_runtime_last_position_survives_rollover_to_empty_ledger() { ..ManagedLedgerConfig::default() }; - let mut ledger = RocksDBManagedLedger::open_with_config( + let ledger = RocksDBManagedLedger::open_with_config( "ledger-a", Arc::clone(&db), Arc::clone(&entry_log), @@ -475,8 +474,8 @@ fn managed_ledger_runtime_last_position_survives_rollover_to_empty_ledger() { ) .unwrap(); - let first = ledger.add_entry_with_partition(7, b"first").unwrap(); - let second = ledger.add_entry_with_partition(7, b"second").unwrap(); + let first = append_with_partition(&ledger, 7, b"first").unwrap(); + let second = append_with_partition(&ledger, 7, b"second").unwrap(); assert_eq!(first.ledger_id, second.ledger_id); assert_eq!(second.ledger_id, 0); @@ -484,7 +483,7 @@ fn managed_ledger_runtime_last_position_survives_rollover_to_empty_ledger() { assert_eq!(ledger.last_position().unwrap(), Some(second),); // After adding a new entry to the new ledger, the last position should be updated. - let third = ledger.add_entry_with_partition(7, b"third").unwrap(); + let third = append_with_partition(&ledger, 7, b"third").unwrap(); assert_eq!(third.ledger_id, 1); assert_eq!(ledger.last_position().unwrap(), Some(third),) } @@ -498,20 +497,20 @@ fn managed_ledger_runtime_state_recovers_and_continues_after_reopen() { let db = open_test_db(&db_path); let entry_log = open_test_entry_log(&db_path); - let mut ledger = RocksDBManagedLedger::open("ledger-a", db, entry_log).unwrap(); + let ledger = RocksDBManagedLedger::open("ledger-a", db, entry_log).unwrap(); - ledger.add_entry_with_partition(7, b"first").unwrap(); - ledger.add_entry_with_partition(7, b"second").unwrap() + append_with_partition(&ledger, 7, b"first").unwrap(); + append_with_partition(&ledger, 7, b"second").unwrap() }; let db = open_test_db(&db_path); let entry_log = open_test_entry_log(&db_path); - let mut ledger = RocksDBManagedLedger::open("ledger-a", db, entry_log).unwrap(); + let ledger = RocksDBManagedLedger::open("ledger-a", db, entry_log).unwrap(); assert_eq!(ledger.last_position().unwrap(), Some(second.clone())); - let third = ledger.add_entry_with_partition(7, b"third").unwrap(); + let third = append_with_partition(&ledger, 7, b"third").unwrap(); assert_eq!(third.ledger_id, second.ledger_id); assert_eq!(third.entry_id, second.entry_id + 1); diff --git a/rust/storage/managed-ledger-rocksdb/tests/rocksdb_storage.rs b/rust/storage/managed-ledger-rocksdb/tests/rocksdb_storage.rs index aa5e343..3f22302 100644 --- a/rust/storage/managed-ledger-rocksdb/tests/rocksdb_storage.rs +++ b/rust/storage/managed-ledger-rocksdb/tests/rocksdb_storage.rs @@ -3,7 +3,9 @@ mod common; use common::*; -use pulsar_lite_storage_managed_ledger::ManagedLedgerStorage; +use pulsar_lite_storage_managed_ledger::{ + CursorInitOptions, InitialPosition, ManagedLedgerStorage, +}; use pulsar_lite_storage_managed_ledger_rocksdb::{test_support::keys, RocksDbManagedLedgerStorage}; use tempfile::tempdir; @@ -102,3 +104,52 @@ fn storage_normalizes_topic_url_and_encodes_cursor_name() { .unwrap() .is_none()); } + +#[test] +fn append_after_latest_cursor_is_visible_to_reads() { + // Reproduces the consumer-0-message bug: + // 1) subscribe/Latest warms the SharedLedger cache with empty in-memory info + // 2) producer appends through the write-queue worker's owned ledger copy + // 3) dispatch reads via SharedLedger and must still see the new entries + let dir = tempdir().unwrap(); + let db_path = dir.path().join("storage-append-visibility"); + let topic = "persistent://public/default/visibility"; + let subscription = "sub"; + + let mut storage = RocksDbManagedLedgerStorage::open(&db_path).unwrap(); + storage.create_topic(topic).unwrap(); + + // Warm the reader-side SharedLedger cache the same way subscribe does. + storage + .initialize_or_open_cursor( + topic, + subscription, + CursorInitOptions { + initial_position: InitialPosition::Latest, + start_message_id: None, + }, + ) + .unwrap(); + + assert_eq!( + storage.first_unacked_position(topic, subscription).unwrap(), + None, + "empty topic with Latest cursor should have no backlog" + ); + + let message_id = storage + .append_message(topic, -1, b"hello-after-subscribe") + .unwrap(); + + let first = storage + .first_unacked_position(topic, subscription) + .unwrap() + .expect("appended entry must be visible as first unacked"); + assert_eq!(first.ledger_id, message_id.ledger); + assert_eq!(first.entry_id, message_id.entry); + + let entries = storage.read_entries_from(topic, &first, 1).unwrap(); + assert_eq!(entries.len(), 1); + assert_eq!(entries[0].message_id, message_id); + assert_eq!(entries[0].payload, b"hello-after-subscribe"); +} diff --git a/rust/storage/managed-ledger/src/cursor.rs b/rust/storage/managed-ledger/src/cursor.rs index 1aede05..593726c 100644 --- a/rust/storage/managed-ledger/src/cursor.rs +++ b/rust/storage/managed-ledger/src/cursor.rs @@ -1,6 +1,6 @@ use crate::position::ManagedLedgerPosition; +use crate::range_set::RangeSet; use anyhow::Result; -use std::collections::BTreeSet; /// Managed-cursor state skeleton. /// @@ -9,7 +9,10 @@ use std::collections::BTreeSet; #[derive(Debug, Clone, Default, PartialEq, Eq)] pub struct ManagedCursorState { pub mark_delete: Option, - pub individually_deleted_entries: BTreeSet, + /// Out-of-order acknowledgements past the mark-delete frontier, stored as + /// coalesced ranges (see `RangeSet`) so memory stays O(ranges) even when + /// millions of individual acks arrive out of order. + pub individually_deleted_entries: RangeSet, } /// Cursor abstraction for managed-ledger style persistence. @@ -29,7 +32,7 @@ pub trait ManagedCursor: Send + Sync { #[derive(Debug, Clone, Default, PartialEq, Eq)] pub struct SubscriptionCursor { pub mark_delete: Option, - pub acked_holes: BTreeSet, + pub acked_holes: RangeSet, } pub fn is_message_acknowledged(cursor: Option<&SubscriptionCursor>, entry: u64) -> bool { @@ -43,16 +46,24 @@ pub fn is_message_acknowledged(cursor: Option<&SubscriptionCursor>, entry: u64) .unwrap_or(false) } +/// Advances the mark-delete frontier across contiguous acknowledged ranges. +/// +/// `take_covering` removes the whole covering range at once, so a long +/// sequential ack streak is consumed in one step per stored range instead of +/// one position at a time. pub fn advance_mark_delete(cursor: &mut SubscriptionCursor) { - let mut next_expected = cursor.mark_delete.map_or(0, |mark_delete| mark_delete + 1); - while cursor.acked_holes.remove(&next_expected) { - cursor.mark_delete = Some(next_expected); - next_expected += 1; + loop { + let next_expected = cursor.mark_delete.map_or(0, |mark_delete| mark_delete + 1); + match cursor.acked_holes.take_covering(&next_expected) { + Some(end) => cursor.mark_delete = Some(end), + None => break, + } } } pub fn ack_shared(cursor: &mut SubscriptionCursor, entry: u64) -> (Option, usize) { if is_message_acknowledged(Some(cursor), entry) { + // Second value is the number of stored ranges (not individual holes). return (cursor.mark_delete, cursor.acked_holes.len()); } @@ -75,5 +86,6 @@ pub fn ack_shared(cursor: &mut SubscriptionCursor, entry: u64) -> (Option, } } + // Second value is the number of stored ranges (not individual holes). (cursor.mark_delete, cursor.acked_holes.len()) } diff --git a/rust/storage/managed-ledger/src/legacy_storage.rs b/rust/storage/managed-ledger/src/legacy_storage.rs index bf5c6be..522b796 100644 --- a/rust/storage/managed-ledger/src/legacy_storage.rs +++ b/rust/storage/managed-ledger/src/legacy_storage.rs @@ -143,6 +143,19 @@ pub trait ManagedLedgerStorage: Send + Sync { subscription: &str, message_id: &MessageId, ) -> bool; - fn get_mark_delete_position(&self, topic: &str, subscription: &str) -> Option; + + /// Number of stored entries not yet acknowledged by `subscription` + /// (ledger-aware across rollovers). `None` when the topic/cursor is + /// unknown to the backend. + fn backlog_entries(&self, topic: &str, subscription: &str) -> Option { + let _ = (topic, subscription); + None + } + + /// Bytes durably stored for `topic` (sum of ledger sizes). + fn stored_bytes(&self, topic: &str) -> u64 { + let _ = topic; + 0 + } } diff --git a/rust/storage/managed-ledger/src/lib.rs b/rust/storage/managed-ledger/src/lib.rs index f909c76..aa57aaa 100644 --- a/rust/storage/managed-ledger/src/lib.rs +++ b/rust/storage/managed-ledger/src/lib.rs @@ -8,6 +8,7 @@ mod ledger; mod legacy_storage; mod memory; mod position; +mod range_set; pub use config::ManagedLedgerConfig; pub use cursor::{ @@ -27,3 +28,4 @@ pub use memory::{ InMemoryManagedLedgerStorage, }; pub use position::{ManagedLedgerPosition, MessageId, NonPersistentEntry, StoredMessage}; +pub use range_set::{RangeSet, Succ}; diff --git a/rust/storage/managed-ledger/src/memory.rs b/rust/storage/managed-ledger/src/memory.rs index 5c7453d..9758200 100644 --- a/rust/storage/managed-ledger/src/memory.rs +++ b/rust/storage/managed-ledger/src/memory.rs @@ -306,11 +306,41 @@ impl ManagedLedgerStorage for InMemoryManagedLedgerStorage { let cursor_key = format!("{}:{}", topic, subscription); is_message_acknowledged(self.subscription_cursors.get(&cursor_key), message_id.entry) } - fn get_mark_delete_position(&self, topic: &str, subscription: &str) -> Option { let cursor_key = format!("{}:{}", topic, subscription); self.subscription_cursors.get(&cursor_key)?.mark_delete } + + fn backlog_entries(&self, topic: &str, subscription: &str) -> Option { + let cursor_key = format!("{}:{}", topic, subscription); + let total = self.messages(topic).len() as u64; + + // Shared subscription: frontier plus individual holes. + if let Some(cursor) = self.subscription_cursors.get(&cursor_key) { + let acked = cursor.mark_delete.map_or(0, |mark| mark + 1); + return Some(total.saturating_sub(acked)); + } + + // Exclusive/failover: plain u64 frontier; u64::MAX = nothing acked. + match self.cursors.get(&cursor_key).copied() { + Some(mark) if mark != u64::MAX => Some(total.saturating_sub(mark + 1)), + _ => Some(total), + } + } + + fn stored_bytes(&self, topic: &str) -> u64 { + self.messages(topic) + .iter() + .map(|(message_id, payload)| { + let metadata = self + .entry_metadata + .get(message_id) + .map(|value| value.len()) + .unwrap_or(0); + metadata + payload.len() + }) + .sum::() as u64 + } } #[derive(Debug, Default)] diff --git a/rust/storage/managed-ledger/src/range_set.rs b/rust/storage/managed-ledger/src/range_set.rs new file mode 100644 index 0000000..f7ffb6e --- /dev/null +++ b/rust/storage/managed-ledger/src/range_set.rs @@ -0,0 +1,190 @@ +use crate::position::ManagedLedgerPosition; +use serde::Deserialize; +use std::collections::BTreeSet; + +/// A sorted, non-overlapping set of deleted-position ranges (both endpoints +/// inclusive). +/// +/// Consecutive acknowledgements are coalesced into a single range, so memory +/// usage is O(number of ranges) instead of O(number of acknowledged +/// positions). A Shared-subscription cursor acknowledging millions of messages +/// in near-sequential order keeps only a handful of ranges in memory. +/// +/// The representation mirrors Apache Pulsar's `individualDeletedMessages` +/// (a `LongPairRangeSet`): the ack frontier (`mark_delete`) can only express a +/// contiguous prefix, so out-of-order acknowledgements past the frontier are +/// tracked here as compact ranges instead of one entry per position. +#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize)] +pub struct RangeSet { + /// (start, end) pairs; ranges are kept disjoint and non-adjacent + /// (adjacent ranges are merged on insert). + ranges: BTreeSet<(K, K)>, +} + +impl<'de, K: Ord + Deserialize<'de>> serde::Deserialize<'de> for RangeSet { + fn deserialize>(deserializer: D) -> Result { + Ok(Self { + ranges: BTreeSet::deserialize(deserializer)?, + }) + } +} + +impl Default for RangeSet { + fn default() -> Self { + Self { + ranges: BTreeSet::new(), + } + } +} + +/// Successor operation on a position key, used for adjacency checks during +/// range merging. Returns `None` on overflow (e.g. `u64::MAX`), which simply +/// disables adjacency merging at that boundary. +pub trait Succ: Ord + Sized { + fn succ(&self) -> Option; +} + +impl Succ for u64 { + fn succ(&self) -> Option { + self.checked_add(1) + } +} + +impl Succ for ManagedLedgerPosition { + fn succ(&self) -> Option { + self.entry_id.checked_add(1).map(|entry_id| ManagedLedgerPosition { + ledger_id: self.ledger_id, + entry_id, + partition: self.partition, + }) + } +} + +impl RangeSet { + pub fn new() -> Self { + Self::default() + } + + pub fn clear(&mut self) { + self.ranges.clear(); + } + + /// Number of stored ranges (the memory-relevant metric). + pub fn len(&self) -> usize { + self.ranges.len() + } + + pub fn is_empty(&self) -> bool { + self.ranges.is_empty() + } + + pub fn iter(&self) -> impl Iterator { + self.ranges.iter() + } + + /// Returns `true` if `pos` falls inside any stored range. + /// + /// Two lookups are needed because a range starting exactly at `pos` sorts + /// after the `(pos, pos)` probe when its end is greater than `pos`. + pub fn contains(&self, pos: &K) -> bool { + // 1) The largest range whose start is <= pos. + if let Some((start, end)) = self.ranges.range(..=(pos.clone(), pos.clone())).next_back() { + if start <= pos && pos <= end { + return true; + } + } + // 2) A range starting exactly at pos. + if let Some((start, _)) = self.ranges.range((pos.clone(), pos.clone())..).next() { + if start == pos { + return true; + } + } + false + } + + /// Inserts the inclusive range `[start, end]`, merging every stored range + /// that intersects or is adjacent to it. + pub fn insert_range(&mut self, start: K, end: K) { + if start > end { + return; + } + let mut start = start; + let mut end = end; + + // Merge ranges starting at or after the new start that intersect or + // are adjacent to the growing range. Adjacency is `s <= end + 1`. + loop { + let limit = end.succ().unwrap_or_else(|| end.clone()); + let merge = self + .ranges + .range((start.clone(), start.clone())..) + .next() + .cloned() + .filter(|(s, _)| *s <= limit); + let Some((s, e)) = merge else { + break; + }; + self.ranges.remove(&(s.clone(), e.clone())); + if s < start { + start = s; + } + if end < e { + end = e; + } + } + + // Merge the left neighbour when it is adjacent to the new start. + if let Some((s, e)) = self + .ranges + .range(..=(start.clone(), start.clone())) + .next_back() + .cloned() + { + if e.succ() == Some(start.clone()) { + self.ranges.remove(&(s.clone(), e.clone())); + start = s; + } + } + + self.ranges.insert((start, end)); + } + + /// Inserts a single position (a one-point range). + pub fn insert(&mut self, pos: K) { + if self.contains(&pos) { + return; + } + self.insert_range(pos.clone(), pos); + } + + /// If `pos` falls inside a stored range, removes that range and returns + /// its end. Used by mark-delete advancement: the whole contiguous deleted + /// segment can be skipped in one step instead of one position at a time. + pub fn take_covering(&mut self, pos: &K) -> Option { + // 1) The largest range whose start is <= pos. + if let Some((start, end)) = self + .ranges + .range(..=(pos.clone(), pos.clone())) + .next_back() + .cloned() + { + if start <= *pos { + self.ranges.remove(&(start.clone(), end.clone())); + return Some(end); + } + } + // 2) A range starting exactly at pos. + if let Some((start, end)) = self + .ranges + .range((pos.clone(), pos.clone())..) + .next() + .cloned() + { + if start == *pos { + self.ranges.remove(&(start.clone(), end.clone())); + return Some(end); + } + } + None + } +} diff --git a/tests/perf/lib/broker.py b/tests/perf/lib/broker.py index 7f2bb24..3eeacb7 100644 --- a/tests/perf/lib/broker.py +++ b/tests/perf/lib/broker.py @@ -11,6 +11,7 @@ import time from pathlib import Path import shutil +import sys from . import BASE_CONFIG, BROKER_BIN @@ -20,13 +21,23 @@ class BrokerConfig: name: str port: int default_partitions: int + # Prometheus /metrics port; derived so existing call sites keep 3-arg + # construction. Offset 1430 keeps the base mapping 6650 -> 8080 (default + # web service port), giving every perf broker a private metrics port + # (6651 -> 8081, 6662 -> 8092, 6672 -> 8102, ...). + metrics_port: int | None = None + + def __post_init__(self) -> None: + if self.metrics_port is None: + self.metrics_port = self.port + 1430 class BrokerSampler(threading.Thread): - def __init__(self, pid: int, interval: float = 0.5): + def __init__(self, pid: int, interval: float = 0.5, cgroup_dir: str | None = None): super().__init__(daemon=True) self.pid = pid self.interval = interval + self.cgroup_dir = cgroup_dir self.samples: list[dict[str, float]] = [] self._stop_event = threading.Event() self._last_total = None @@ -36,6 +47,13 @@ def __init__(self, pid: int, interval: float = 0.5): def stop(self) -> None: self._stop_event.set() + def reset(self) -> None: + """Drop samples and restart the CPU delta baseline. Call before each + scenario so metrics() reflects only that scenario's window.""" + self.samples.clear() + self._last_total = None + self._last_time = None + def run(self) -> None: while not self._stop_event.is_set(): try: @@ -58,24 +76,55 @@ def run(self) -> None: rss_match = re.search(r"^VmRSS:\s+(\d+)\s+kB$", status_text, re.MULTILINE) rss_mb = (float(rss_match.group(1)) / 1024.0) if rss_match else 0.0 - self.samples.append({"cpu_pct": cpu_pct, "rss_mb": rss_mb}) + sample: dict[str, float] = {"cpu_pct": cpu_pct, "rss_mb": rss_mb} + if self.cgroup_dir: + try: + with open( + f"{self.cgroup_dir}/memory.stat", "r", encoding="utf-8" + ) as fh: + mem_stat = fh.read() + for line in mem_stat.splitlines(): + key, _, value = line.partition(" ") + if key == "anon": + sample["anon_mb"] = int(value) / 1048576.0 + elif key == "file": + sample["file_mb"] = int(value) / 1048576.0 + except (OSError, ValueError): + pass + self.samples.append(sample) time.sleep(self.interval) def write_csv(self, csv_path: Path) -> None: + fieldnames = ["cpu_pct", "rss_mb"] + if any("anon_mb" in sample for sample in self.samples): + fieldnames += ["anon_mb", "file_mb"] with csv_path.open("w", encoding="utf-8", newline="") as fh: - writer = csv.DictWriter(fh, fieldnames=["cpu_pct", "rss_mb"]) + writer = csv.DictWriter(fh, fieldnames=fieldnames) writer.writeheader() writer.writerows(self.samples) def _config_text(config: BrokerConfig, db_path: str) -> str: config_text = BASE_CONFIG.read_text(encoding="utf-8") - config_text = re.sub( + # The base config has two `addr` lines: the protocol listener at the top + # and the metrics endpoint inside the [metrics] section. Rewrite them + # separately so they never collapse onto the same port (both listeners + # racing for one bind), and keep metrics on 0.0.0.0 so the compose + # Prometheus can scrape it through host.docker.internal. + head, sep, metrics_section = config_text.partition("[metrics]") + head = re.sub( r'^addr\s*=\s*".*"$', f'addr = "127.0.0.1:{config.port}"', - config_text, + head, flags=re.MULTILINE, ) + metrics_section = re.sub( + r'^addr\s*=\s*".*"$', + f'addr = "0.0.0.0:{config.metrics_port}"', + metrics_section, + flags=re.MULTILINE, + ) + config_text = head + sep + metrics_section config_text = re.sub( r'^db_path\s*=\s*".*"$', f'db_path = "{db_path}"', @@ -92,14 +141,46 @@ def _config_text(config: BrokerConfig, db_path: str) -> str: class BrokerProcess: - def __init__(self, config: BrokerConfig): + def __init__( + self, + config: BrokerConfig, + cgroup_memory: str | None = None, + cgroup_cpus: str | None = None, + ): + """Local broker process. + + cgroup_memory: MemoryMax for systemd-run --user --scope (e.g. "4294967296" + or "4G"); MemorySwapMax is pinned to 0 to match docker --memory-swap. + Requires user-scope cgroup delegation (memory controller). + cgroup_cpus: CPU affinity via taskset -c (e.g. "0-3"), equivalent to + docker --cpuset-cpus (same sched_setaffinity mechanism). + """ self.config = config + self.cgroup_memory = cgroup_memory + self.cgroup_cpus = cgroup_cpus self.proc: subprocess.Popen[str] | None = None self.broker_pid: int | None = None self.workdir: Path | None = None self.log_path: Path | None = None self.sampler: BrokerSampler | None = None + def _broker_cmd(self) -> list[str]: + cmd: list[str] = [] + if self.cgroup_memory: + cmd += [ + "systemd-run", + "--user", + "--scope", + "-p", + f"MemoryMax={self.cgroup_memory}", + "-p", + "MemorySwapMax=0", + ] + if self.cgroup_cpus: + cmd += ["taskset", "-c", self.cgroup_cpus] + cmd.append(str(BROKER_BIN)) + return cmd + def start(self) -> None: temp_dir = Path( tempfile.mkdtemp(prefix=f"pulsar-lite-{self.config.name}-", dir="/tmp") @@ -111,7 +192,7 @@ def start(self) -> None: self.log_path = temp_dir / "broker.log" log_file = self.log_path.open("w", encoding="utf-8") self.proc = subprocess.Popen( - [str(BROKER_BIN)], + self._broker_cmd(), cwd=temp_dir, stdout=log_file, stderr=subprocess.STDOUT, @@ -178,7 +259,7 @@ def restart(self, preserve_storage: bool = False) -> None: # Reopen log file for appending log_file = self.log_path.open("a", encoding="utf-8") self.proc = subprocess.Popen( - [str(BROKER_BIN)], + self._broker_cmd(), cwd=self.workdir, stdout=log_file, stderr=subprocess.STDOUT, @@ -190,8 +271,8 @@ def restart(self, preserve_storage: bool = False) -> None: self.sampler = BrokerSampler(self.broker_pid) self.sampler.start() else: - # Original behavior: fresh start - self.stop() + # Fresh storage: drop the previous /tmp workdir so disk does not accumulate. + self.stop(cleanup=True) self.start() def metrics(self) -> dict[str, float]: @@ -201,16 +282,101 @@ def metrics(self) -> dict[str, float]: "broker_avg_cpu_pct": 0.0, "broker_peak_cpu_pct": 0.0, "broker_peak_rss_mb": 0.0, + "broker_peak_anon_mb": 0.0, + "broker_peak_file_mb": 0.0, } cpu_values = [sample["cpu_pct"] for sample in samples[1:]] or [0.0] rss_values = [sample["rss_mb"] for sample in samples] + anon_values = [sample.get("anon_mb", 0.0) for sample in samples] + file_values = [sample.get("file_mb", 0.0) for sample in samples] return { "broker_avg_cpu_pct": round(sum(cpu_values) / len(cpu_values), 3), "broker_peak_cpu_pct": round(max(cpu_values), 3), "broker_peak_rss_mb": round(max(rss_values), 3), + "broker_peak_anon_mb": round(max(anon_values), 3), + "broker_peak_file_mb": round(max(file_values), 3), } +class ExternalBrokerProcess(BrokerProcess): + """Adapter for an already-running external broker (e.g. Apache Pulsar + standalone started manually with cgroup limits). + + No lifecycle management: the harness only builds perf commands against + ``broker.config.port``. Scenarios that restart the broker + (restart_replay, redelivery_unacked) are not supported. + + If ``unit`` is given (systemd unit name, e.g. ``pulsar-standalone``), the + broker PID is resolved via ``systemctl show -p MainPID`` and CPU / + RSS / cgroup anon+file metrics are sampled like the local backend. + """ + + def __init__(self, config: BrokerConfig, unit: str | None = None): + super().__init__(config) + self.log_path = None + self.unit = unit + self.sampler = None + + def _resolve_pid(self) -> int | None: + try: + out = subprocess.run( + ["systemctl", "show", self.unit, "-p", "MainPID", "--value"], + capture_output=True, + text=True, + timeout=10, + ).stdout.strip() + except (OSError, subprocess.TimeoutExpired): + return None + if not out.isdigit(): + return None + pid = int(out) + if not os.path.exists(f"/proc/{pid}"): + return None + return pid + + def _resolve_cgroup_dir(self, pid: int) -> str | None: + try: + with open(f"/proc/{pid}/cgroup", "r", encoding="utf-8") as fh: + rel = fh.read().strip().split(":")[-1] + path = f"/sys/fs/cgroup{rel}" + if os.path.isfile(f"{path}/memory.stat"): + return path + except OSError: + pass + return None + + def start(self) -> None: + # External broker must already be listening. + self.broker_pid = None + self._wait_for_port() + if not self.unit: + return + pid = self._resolve_pid() + if pid is None: + print( + f" [warn] systemd unit '{self.unit}' not found; " + "broker CPU/memory metrics disabled", + file=sys.stderr, + ) + return + self.broker_pid = pid + self.sampler = BrokerSampler(pid, cgroup_dir=self._resolve_cgroup_dir(pid)) + self.sampler.start() + + def stop(self, cleanup: bool = False) -> dict[str, float]: + metrics = self.metrics() + if self.sampler: + self.sampler.stop() + return metrics + + def restart(self, preserve_storage: bool = False) -> None: + raise NotImplementedError( + "external broker backend cannot restart the broker; " + "use scenarios that do not restart " + "(produce / consume_e2e / backlog_drain)" + ) + + class DockerBrokerProcess(BrokerProcess): def __init__( self, config: BrokerConfig, image_tag: str, cpuset_cpus: str, memory: str @@ -378,6 +544,6 @@ def restart(self, preserve_storage: bool = False) -> None: self.sampler = BrokerSampler(self.broker_pid) self.sampler.start() else: - # Original behavior: fresh start - self.stop() + # Fresh storage: drop the previous /tmp workdir so disk does not accumulate. + self.stop(cleanup=True) self.start() diff --git a/tests/perf/lib/parsing.py b/tests/perf/lib/parsing.py index fa75f3d..fad07f9 100644 --- a/tests/perf/lib/parsing.py +++ b/tests/perf/lib/parsing.py @@ -1,110 +1,246 @@ from __future__ import annotations import re +import statistics from typing import Any +# pulsar-perf prints one interval line about every 10s (Thread.sleep(10000)). +# First window is often cold (connect / warmup / empty feed); drop it when we +# have 2+ samples and take median of the rest as steady-state thr. -def parse_producer_output(text: str) -> dict[str, Any]: - throughput = re.search( - r"Aggregated throughput stats ---\s+(\d+) records sent ---\s+([\d.]+) msg/s ---\s+([\d.]+) Mbit/s", - text, - ) - latency = re.search( - r"Aggregated latency stats --- Latency: mean:\s+([\d.]+) ms - med:\s+([\d.]+) - 95pct:\s+([\d.]+) - 99pct:\s+([\d.]+) - 99\.9pct:\s+([\d.]+) - 99\.99pct:\s+([\d.]+) - 99\.999pct:\s+([\d.]+) - Max:\s+([\d.]+)", - text, - ) - if not throughput or not latency: - interval_matches = re.findall( - r"Throughput produced:\s+(\d+) msg ---\s+([\d.]+)\s+msg/s ---\s+([\d.]+)\s+Mbit/s.*?Latency: mean:\s+([\d.]+)\s+ms.*?99pct:\s+([\d.]+).*?Max:\s+([\d.]+)", - text, - re.S, +_AGG_PRODUCER_THR = re.compile( + r"Aggregated throughput stats ---\s+(\d+) records sent ---\s+([\d.]+) msg/s ---\s+([\d.]+) Mbit/s" +) +_AGG_CONSUMER_THR = re.compile( + r"Aggregated throughput stats ---\s+(\d+) records received ---\s+([\d.]+) msg/s ---\s+([\d.]+) Mbit/s --- AckRate: ([\d.]+)\s+msg/s --- ack failed (\d+) msg" +) +_AGG_LATENCY = re.compile( + r"Aggregated latency stats --- Latency: mean:\s+([\d.]+) ms - med:\s+([\d.]+) - 95pct:\s+([\d.]+) - 99pct:\s+([\d.]+) - 99\.9pct:\s+([\d.]+) - 99\.99pct:\s+([\d.]+) - 99\.999pct:\s+([\d.]+) - Max:\s+([\d.]+)" +) + +# Interval lines (per ~10s window). cumulative total is first number; rate is window thr. +_INTERVAL_PRODUCER = re.compile( + r"Throughput produced:\s+(\d+) msg ---\s+([\d.]+)\s+msg/s ---\s+([\d.]+)\s+Mbit/s" + r".*?Latency: mean:\s+([\d.]+)\s+ms - med:\s+([\d.]+) - 95pct:\s+([\d.]+) - 99pct:\s+([\d.]+)" + r".*?Max:\s+([\d.]+)", + re.S, +) +_INTERVAL_CONSUMER = re.compile( + r"Throughput received:\s+(\d+) msg ---\s+([\d.]+)\s+msg/s ---\s+([\d.]+)\s+Mbit/s" + r".*?Latency: mean:\s+([\d.]+)\s+ms - med:\s+([\d.]+) - 95pct:\s+([\d.]+) - 99pct:\s+([\d.]+)" + r".*?Max:\s+([\d.]+)", + re.S, +) + + +def _median(values: list[float]) -> float: + return float(statistics.median(values)) + + +def _steady_slices(intervals: list[dict[str, float]]) -> list[dict[str, float]]: + """Drop the first interval when 2+ windows exist (cold start / empty feed).""" + if len(intervals) >= 2: + return intervals[1:] + return intervals + + +def _from_intervals(intervals: list[dict[str, float]]) -> dict[str, Any] | None: + if not intervals: + return None + steady = _steady_slices(intervals) + thr = [row["throughput_msg_s"] for row in steady] + mbit = [row["throughput_mbit_s"] for row in steady] + lat_mean = [row["latency_mean_ms"] for row in steady] + lat_p50 = [row["latency_p50_ms"] for row in steady] + lat_p95 = [row["latency_p95_ms"] for row in steady] + lat_p99 = [row["latency_p99_ms"] for row in steady] + lat_max = [row["latency_max_ms"] for row in steady] + return { + "throughput_msg_s": _median(thr), + "throughput_mbit_s": _median(mbit), + "latency_mean_ms": _median(lat_mean), + "latency_p50_ms": _median(lat_p50), + "latency_p95_ms": _median(lat_p95), + "latency_p99_ms": _median(lat_p99), + "latency_max_ms": max(lat_max), + "interval_count": len(intervals), + "steady_interval_count": len(steady), + "interval_throughput_msg_s_median": _median(thr), + "interval_throughput_msg_s_min": min(thr), + "interval_throughput_msg_s_max": max(thr), + "records_cumulative_last": int(intervals[-1]["records_cumulative"]), + "partial": len(steady) < 2, + } + + +def _parse_producer_intervals(text: str) -> list[dict[str, float]]: + rows: list[dict[str, float]] = [] + for m in _INTERVAL_PRODUCER.finditer(text): + rows.append( + { + "records_cumulative": float(m.group(1)), + "throughput_msg_s": float(m.group(2)), + "throughput_mbit_s": float(m.group(3)), + "latency_mean_ms": float(m.group(4)), + "latency_p50_ms": float(m.group(5)), + "latency_p95_ms": float(m.group(6)), + "latency_p99_ms": float(m.group(7)), + "latency_max_ms": float(m.group(8)), + } + ) + return rows + + +def _parse_consumer_intervals(text: str) -> list[dict[str, float]]: + rows: list[dict[str, float]] = [] + for m in _INTERVAL_CONSUMER.finditer(text): + rows.append( + { + "records_cumulative": float(m.group(1)), + "throughput_msg_s": float(m.group(2)), + "throughput_mbit_s": float(m.group(3)), + "latency_mean_ms": float(m.group(4)), + "latency_p50_ms": float(m.group(5)), + "latency_p95_ms": float(m.group(6)), + "latency_p99_ms": float(m.group(7)), + "latency_max_ms": float(m.group(8)), + } ) - if not interval_matches: - raise RuntimeError(f"failed to parse producer output:\n{text}") - ( - records, - throughput_msg_s, - throughput_mbit_s, - latency_mean_ms, - latency_p99_ms, - latency_max_ms, - ) = interval_matches[-1] # type: ignore[misc] - return { - "records": int(records), - "throughput_msg_s": float(throughput_msg_s), - "throughput_mbit_s": float(throughput_mbit_s), - "latency_mean_ms": float(latency_mean_ms), - "latency_p50_ms": None, - "latency_p95_ms": None, - "latency_p99_ms": float(latency_p99_ms), - "latency_p999_ms": None, - "latency_max_ms": float(latency_max_ms), - "partial": True, + return rows + + +def parse_producer_output(text: str) -> dict[str, Any]: + """Parse pulsar-perf producer log. + + Prefer ~10s interval windows (drop first when 2+ exist, median of rest) for + throughput/latency. Fall back to Aggregated wall-clock stats when no + interval lines exist (short -m runs). + """ + intervals = _parse_producer_intervals(text) + steady = _from_intervals(intervals) + + agg_thr = _AGG_PRODUCER_THR.search(text) + agg_lat = _AGG_LATENCY.search(text) + + if steady is not None: + records = int(steady["records_cumulative_last"]) + if agg_thr: + records = int(agg_thr.group(1)) + result = { + "records": records, + "throughput_msg_s": steady["throughput_msg_s"], + "throughput_mbit_s": steady["throughput_mbit_s"], + "latency_mean_ms": steady["latency_mean_ms"], + "latency_p50_ms": steady["latency_p50_ms"], + "latency_p95_ms": steady["latency_p95_ms"], + "latency_p99_ms": steady["latency_p99_ms"], + "latency_p999_ms": ( + float(agg_lat.group(5)) if agg_lat else None + ), + "latency_max_ms": steady["latency_max_ms"], + "partial": steady["partial"], + "metric_source": "interval_median", + "interval_count": steady["interval_count"], + "steady_interval_count": steady["steady_interval_count"], + "interval_throughput_msg_s_median": steady[ + "interval_throughput_msg_s_median" + ], + "interval_throughput_msg_s_min": steady["interval_throughput_msg_s_min"], + "interval_throughput_msg_s_max": steady["interval_throughput_msg_s_max"], } + if agg_thr: + result["aggregated_throughput_msg_s"] = float(agg_thr.group(2)) + result["aggregated_throughput_mbit_s"] = float(agg_thr.group(3)) + if agg_lat: + result["aggregated_latency_mean_ms"] = float(agg_lat.group(1)) + result["aggregated_latency_p99_ms"] = float(agg_lat.group(4)) + result["aggregated_latency_max_ms"] = float(agg_lat.group(8)) + return result + + if not agg_thr or not agg_lat: + raise RuntimeError(f"failed to parse producer output:\n{text}") + return { - "records": int(throughput.group(1)), - "throughput_msg_s": float(throughput.group(2)), - "throughput_mbit_s": float(throughput.group(3)), - "latency_mean_ms": float(latency.group(1)), - "latency_p50_ms": float(latency.group(2)), - "latency_p95_ms": float(latency.group(3)), - "latency_p99_ms": float(latency.group(4)), - "latency_p999_ms": float(latency.group(5)), - "latency_max_ms": float(latency.group(8)), + "records": int(agg_thr.group(1)), + "throughput_msg_s": float(agg_thr.group(2)), + "throughput_mbit_s": float(agg_thr.group(3)), + "latency_mean_ms": float(agg_lat.group(1)), + "latency_p50_ms": float(agg_lat.group(2)), + "latency_p95_ms": float(agg_lat.group(3)), + "latency_p99_ms": float(agg_lat.group(4)), + "latency_p999_ms": float(agg_lat.group(5)), + "latency_max_ms": float(agg_lat.group(8)), "partial": False, + "metric_source": "aggregated", + "interval_count": 0, + "steady_interval_count": 0, } def parse_consumer_output(text: str) -> dict[str, Any]: - throughput = re.search( - r"Aggregated throughput stats ---\s+(\d+) records received ---\s+([\d.]+) msg/s ---\s+([\d.]+) Mbit/s --- AckRate: ([\d.]+)\s+msg/s --- ack failed (\d+) msg", - text, - ) - latency = re.search( - r"Aggregated latency stats --- Latency: mean:\s+([\d.]+) ms - med:\s+([\d.]+) - 95pct:\s+([\d.]+) - 99pct:\s+([\d.]+) - 99\.9pct:\s+([\d.]+) - 99\.99pct:\s+([\d.]+) - 99\.999pct:\s+([\d.]+) - Max:\s+([\d.]+)", - text, - ) - if not throughput or not latency: - interval_matches = re.findall( - r"Throughput received:\s+(\d+) msg ---\s+([\d.]+)\s+msg/s ---\s+([\d.]+)\s+Mbit/s.*?Latency: mean:\s+([\d.]+)\s+ms.*?99pct:\s+([\d.]+).*?Max:\s+([\d.]+)", - text, - re.S, - ) - if not interval_matches: - raise RuntimeError(f"failed to parse consumer output:\n{text}") - ( - records, - throughput_msg_s, - throughput_mbit_s, - latency_mean_ms, - latency_p99_ms, - latency_max_ms, - ) = interval_matches[-1] # type: ignore[misc] - return { - "records": int(records), - "throughput_msg_s": float(throughput_msg_s), - "throughput_mbit_s": float(throughput_mbit_s), - "ack_rate_msg_s": None, - "ack_failed": None, - "latency_mean_ms": float(latency_mean_ms), - "latency_p50_ms": None, - "latency_p95_ms": None, - "latency_p99_ms": float(latency_p99_ms), - "latency_p999_ms": None, - "latency_max_ms": float(latency_max_ms), - "partial": True, + """Parse pulsar-perf consumer log. + + Same interval-median preference as producer. Ack fields still come from + Aggregated when present (interval lines do not include ack rate). + """ + intervals = _parse_consumer_intervals(text) + steady = _from_intervals(intervals) + + agg_thr = _AGG_CONSUMER_THR.search(text) + agg_lat = _AGG_LATENCY.search(text) + + if steady is not None: + records = int(steady["records_cumulative_last"]) + if agg_thr: + records = int(agg_thr.group(1)) + result = { + "records": records, + "throughput_msg_s": steady["throughput_msg_s"], + "throughput_mbit_s": steady["throughput_mbit_s"], + "ack_rate_msg_s": float(agg_thr.group(4)) if agg_thr else None, + "ack_failed": int(agg_thr.group(5)) if agg_thr else None, + "latency_mean_ms": steady["latency_mean_ms"], + "latency_p50_ms": steady["latency_p50_ms"], + "latency_p95_ms": steady["latency_p95_ms"], + "latency_p99_ms": steady["latency_p99_ms"], + "latency_p999_ms": float(agg_lat.group(5)) if agg_lat else None, + "latency_max_ms": steady["latency_max_ms"], + "partial": steady["partial"], + "metric_source": "interval_median", + "interval_count": steady["interval_count"], + "steady_interval_count": steady["steady_interval_count"], + "interval_throughput_msg_s_median": steady[ + "interval_throughput_msg_s_median" + ], + "interval_throughput_msg_s_min": steady["interval_throughput_msg_s_min"], + "interval_throughput_msg_s_max": steady["interval_throughput_msg_s_max"], } + if agg_thr: + result["aggregated_throughput_msg_s"] = float(agg_thr.group(2)) + result["aggregated_throughput_mbit_s"] = float(agg_thr.group(3)) + if agg_lat: + result["aggregated_latency_mean_ms"] = float(agg_lat.group(1)) + result["aggregated_latency_p99_ms"] = float(agg_lat.group(4)) + result["aggregated_latency_max_ms"] = float(agg_lat.group(8)) + return result + + if not agg_thr or not agg_lat: + raise RuntimeError(f"failed to parse consumer output:\n{text}") + return { - "records": int(throughput.group(1)), - "throughput_msg_s": float(throughput.group(2)), - "throughput_mbit_s": float(throughput.group(3)), - "ack_rate_msg_s": float(throughput.group(4)), - "ack_failed": int(throughput.group(5)), - "latency_mean_ms": float(latency.group(1)), - "latency_p50_ms": float(latency.group(2)), - "latency_p95_ms": float(latency.group(3)), - "latency_p99_ms": float(latency.group(4)), - "latency_p999_ms": float(latency.group(5)), - "latency_max_ms": float(latency.group(8)), + "records": int(agg_thr.group(1)), + "throughput_msg_s": float(agg_thr.group(2)), + "throughput_mbit_s": float(agg_thr.group(3)), + "ack_rate_msg_s": float(agg_thr.group(4)), + "ack_failed": int(agg_thr.group(5)), + "latency_mean_ms": float(agg_lat.group(1)), + "latency_p50_ms": float(agg_lat.group(2)), + "latency_p95_ms": float(agg_lat.group(3)), + "latency_p99_ms": float(agg_lat.group(4)), + "latency_p999_ms": float(agg_lat.group(5)), + "latency_max_ms": float(agg_lat.group(8)), "partial": False, + "metric_source": "aggregated", + "interval_count": 0, + "steady_interval_count": 0, } diff --git a/tests/perf/lib/perf_cmd.py b/tests/perf/lib/perf_cmd.py index a8a8ee8..c30ac64 100644 --- a/tests/perf/lib/perf_cmd.py +++ b/tests/perf/lib/perf_cmd.py @@ -99,6 +99,108 @@ def wait_for_log(path: Path, needle: str, timeout: float = 30.0) -> None: raise RuntimeError(f"timed out waiting for {needle!r} in {path}") +def explain_exit_code(rc: int | None) -> str: + """Human-readable process exit status (143 = SIGTERM from harness, etc.).""" + if rc is None: + return "still-running" + if rc == 0: + return "0 (ok)" + if rc < 0: + return f"{rc} (killed by signal {-rc})" + if rc == 143: + return ( + "143 (SIGTERM: process was terminated — often by the harness " + "after the peer failed, or on overall timeout; not a Java business error)" + ) + if rc == 137: + return "137 (SIGKILL / likely OOM killer)" + if rc > 128: + return f"{rc} (signal {rc - 128})" + return f"{rc} (non-zero process exit)" + + +def log_tail(text: str, *, max_lines: int = 50, max_chars: int = 4000) -> str: + """Tail of a perf log (errors and Aggregated lines are usually at the end).""" + lines = text.splitlines() + tail_lines = lines[-max_lines:] if lines else [] + tail = "\n".join(tail_lines) + if len(tail) > max_chars: + tail = tail[-max_chars:] + return tail if tail.strip() else "(log empty)" + + +def format_e2e_process_failure( + *, + consumer_rc: int, + producer_rc: int, + consumer_out: str, + producer_out: str, + first_failed: str | None = None, + consumer_label: str = "consumer", + producer_label: str = "producer", +) -> str: + """Build an error message that shows both exit codes and log tails.""" + if first_failed is None: + if producer_rc not in (0, 143) and consumer_rc == 143: + first_failed = "producer" + elif consumer_rc not in (0, 143) and producer_rc == 143: + first_failed = "consumer" + elif consumer_rc == 143 and producer_rc == 143: + first_failed = "timeout-or-both-sigterm" + elif producer_rc != 0: + first_failed = "producer" + elif consumer_rc != 0: + first_failed = "consumer" + else: + first_failed = "unknown" + + hang_note = "" + if first_failed and first_failed.startswith("peer_hang_after_"): + hang_note = ( + "\n note: one side finished successfully (rc=0); the peer was still " + "alive after peer_grace and was SIGTERM'd. Often the peer is stuck " + "reconnecting after broker drop, or its -time window ends later " + "because it started later. Check whether broker stayed up." + ) + + parts = [ + "E2E dual-process failure " + f"(first_failed={first_failed})", + f" {consumer_label}_rc={explain_exit_code(consumer_rc)}", + f" {producer_label}_rc={explain_exit_code(producer_rc)}", + ] + if hang_note: + parts.append(hang_note) + parts.extend( + [ + f"--- {producer_label} log tail ---", + log_tail(producer_out), + f"--- {consumer_label} log tail ---", + log_tail(consumer_out), + ] + ) + return "\n".join(parts) + + +def e2e_success_despite_peer_hang( + consumer_rc: int, + producer_rc: int, + first_failed: str | None, +) -> bool: + """True when the primary side finished ok and only the peer was grace-killed. + + Used so -time E2E does not hard-fail after a 600s hang when consumer already + completed successfully and producer is stuck reconnecting. + """ + if not first_failed or not first_failed.startswith("peer_hang_after_"): + return False + if first_failed == "peer_hang_after_consumer_ok": + return consumer_rc == 0 and producer_rc in (0, 143, -15) + if first_failed == "peer_hang_after_producer_ok": + return producer_rc == 0 and consumer_rc in (0, 143, -15) + return False + + def run_consumer_then_feed( consumer_cmd: list[str], producer_cmd: list[str], @@ -106,8 +208,16 @@ def run_consumer_then_feed( producer_log: Path, consumer_timeout: float = 300.0, producer_timeout: float = 300.0, -) -> tuple[str, str, int, int]: - with consumer_log.open("w", encoding="utf-8") as consumer_fh: +) -> tuple[str, str, int, int, str | None]: + """Run consumer then feed producer. + + Returns: + consumer_out, producer_out, consumer_rc, producer_rc, first_failed + first_failed is \"consumer\" | \"producer\" | \"timeout\" | None (both ok). + """ + # Line-buffered file captures so interval/error lines show up sooner if the + # process dies. + with consumer_log.open("w", encoding="utf-8", buffering=1) as consumer_fh: consumer_proc = subprocess.Popen( consumer_cmd, stdout=consumer_fh, @@ -120,9 +230,10 @@ def run_consumer_then_feed( wait_for_log(consumer_log, "Start receiving from") except Exception: _terminate_process(consumer_proc) + # Ensure FH closed and content readable for the error path. raise - with producer_log.open("w", encoding="utf-8") as producer_fh: + with producer_log.open("w", encoding="utf-8", buffering=1) as producer_fh: producer_proc = subprocess.Popen( producer_cmd, stdout=producer_fh, @@ -131,9 +242,11 @@ def run_consumer_then_feed( env=ENV_BASE, ) - consumer_rc, producer_rc = _wait_both_or_kill( + consumer_rc, producer_rc, first_failed = _wait_both_or_kill( consumer_proc, producer_proc, + consumer_log=consumer_log, + producer_log=producer_log, timeout=max(consumer_timeout, producer_timeout), ) @@ -142,12 +255,24 @@ def run_consumer_then_feed( producer_log.read_text(encoding="utf-8", errors="replace"), consumer_rc, producer_rc, + first_failed, ) +def _flush_log_path(path: Path) -> None: + """Best-effort: give the OS/Java a moment, then touch-read the log.""" + time.sleep(0.5) + try: + # Force directory entry / page cache visibility for subsequent reads. + with path.open("rb") as fh: + fh.seek(0, 2) + except OSError: + pass + + def _terminate_process(proc: subprocess.Popen) -> int: if proc.poll() is not None: - return proc.returncode + return proc.returncode if proc.returncode is not None else -1 proc.terminate() try: return proc.wait(timeout=10) @@ -159,24 +284,93 @@ def _terminate_process(proc: subprocess.Popen) -> int: def _wait_both_or_kill( consumer_proc: subprocess.Popen, producer_proc: subprocess.Popen, + *, + consumer_log: Path, + producer_log: Path, timeout: float, -) -> tuple[int, int]: + peer_grace_s: float = 45.0, +) -> tuple[int, int, str | None]: + """Wait for both processes. + + - Both exit → return their codes. + - One exits non-zero → SIGTERM the peer immediately (first_failed=that side). + - One exits zero while the other is still running → give the peer + ``peer_grace_s`` to finish (covers producer started after consumer, or + brief drain). If still alive, SIGTERM the peer and tag + first_failed=\"peer_hang_after__ok\" so callers can treat metrics + as usable instead of a mysterious 600s timeout. + - Overall ``timeout`` still bounds the whole wait. + """ deadline = time.monotonic() + timeout + first_failed: str | None = None + # When set, the named side already exited 0; peer must finish by this time. + peer_grace_deadline: float | None = None + ok_side: str | None = None while True: consumer_rc = consumer_proc.poll() producer_rc = producer_proc.poll() + now = time.monotonic() if consumer_rc is not None and producer_rc is not None: - return consumer_rc, producer_rc + return consumer_rc, producer_rc, first_failed + # One side failed hard: kill peer. if consumer_rc is not None and consumer_rc != 0: - return consumer_rc, _terminate_process(producer_proc) + if first_failed is None: + first_failed = "consumer" + _flush_log_path(consumer_log) + peer_rc = _terminate_process(producer_proc) + _flush_log_path(producer_log) + return consumer_rc, peer_rc, first_failed if producer_rc is not None and producer_rc != 0: - return _terminate_process(consumer_proc), producer_rc + if first_failed is None: + first_failed = "producer" + _flush_log_path(producer_log) + peer_rc = _terminate_process(consumer_proc) + _flush_log_path(consumer_log) + return peer_rc, producer_rc, first_failed + + # One side finished cleanly (rc=0); start/refresh grace for the peer. + if consumer_rc == 0 and producer_rc is None: + if ok_side != "consumer": + ok_side = "consumer" + peer_grace_deadline = now + peer_grace_s + elif producer_rc == 0 and consumer_rc is None: + if ok_side != "producer": + ok_side = "producer" + peer_grace_deadline = now + peer_grace_s - if time.monotonic() >= deadline: - return _terminate_process(consumer_proc), _terminate_process(producer_proc) + if peer_grace_deadline is not None and now >= peer_grace_deadline: + # Successful side is done; peer hung (often reconnect after broker drop). + first_failed = f"peer_hang_after_{ok_side}_ok" + _flush_log_path(consumer_log) + _flush_log_path(producer_log) + if consumer_rc is None: + consumer_rc = _terminate_process(consumer_proc) + if producer_rc is None: + producer_rc = _terminate_process(producer_proc) + _flush_log_path(consumer_log) + _flush_log_path(producer_log) + return consumer_rc, producer_rc, first_failed + + if now >= deadline: + first_failed = first_failed or "timeout" + _flush_log_path(consumer_log) + _flush_log_path(producer_log) + c_rc = ( + consumer_rc + if consumer_rc is not None + else _terminate_process(consumer_proc) + ) + p_rc = ( + producer_rc + if producer_rc is not None + else _terminate_process(producer_proc) + ) + _flush_log_path(consumer_log) + _flush_log_path(producer_log) + return c_rc, p_rc, first_failed time.sleep(0.2) diff --git a/tests/perf/run_non_persistent_e2e_matrix.py b/tests/perf/run_non_persistent_e2e_matrix.py index 3f4da4a..8a3eb33 100644 --- a/tests/perf/run_non_persistent_e2e_matrix.py +++ b/tests/perf/run_non_persistent_e2e_matrix.py @@ -10,9 +10,15 @@ sys.path.insert(0, str(Path(__file__).resolve().parent)) from lib import ROOT -from lib.broker import BrokerConfig, BrokerProcess +from lib.broker import BrokerConfig, BrokerProcess, ExternalBrokerProcess from lib.parsing import parse_consumer_output, parse_producer_output -from lib.perf_cmd import ensure_prereqs, perf_cmd, run_consumer_then_feed, run_sync +from lib.perf_cmd import ( + ensure_prereqs, + format_e2e_process_failure, + perf_cmd, + run_consumer_then_feed, + run_sync, +) RESULTS_PATH = ROOT / "docs" / "perf" / "data" / "non_persistent_e2e_matrix_results.json" ARTIFACTS_DIR = ROOT / "docs" / "perf" / "data" / "non_persistent_e2e_matrix_logs" @@ -329,6 +335,33 @@ def scenario_topic(run_id: str, scenario: Scenario) -> str: def main() -> int: + import argparse + + parser = argparse.ArgumentParser( + description="Run non-persistent e2e matrix scenarios for pulsar-lite" + ) + parser.add_argument( + "--broker-backend", + choices=["local", "external"], + default="local", + help="local starts rust/target/release/pulsar-lite per broker profile; " + "external targets an already-running broker via --external-url " + "(e.g. Apache Pulsar standalone); no broker lifecycle management.", + ) + parser.add_argument( + "--external-url", + default="pulsar://127.0.0.1:6650", + help="Service URL of the external broker for --broker-backend=external.", + ) + parser.add_argument( + "--external-unit", + default=None, + help="systemd unit name of the external broker (e.g. pulsar-standalone or " + "pulsar-lite) to sample CPU/memory via 'systemctl show -p MainPID' " + "+ /proc. Leave unset to disable broker metrics for external backends.", + ) + args = parser.parse_args() + ensure_prereqs() ARTIFACTS_DIR.mkdir(parents=True, exist_ok=True) run_id = time.strftime("%Y%m%d-%H%M%S") @@ -340,13 +373,30 @@ def main() -> int: broker_metrics_by_name: dict[str, dict[str, float]] = {} for broker_name, broker_cfg in BROKERS.items(): - broker = BrokerProcess(broker_cfg) + if args.broker_backend == "external": + from urllib.parse import urlparse + + parsed = urlparse(args.external_url) + if parsed.scheme != "pulsar" or not parsed.hostname: + raise ValueError( + f"--external-url must be pulsar://host:port, got {args.external_url!r}" + ) + broker = ExternalBrokerProcess( + BrokerConfig("external", parsed.port or 6650, 0), + unit=args.external_unit, + ) + else: + broker = BrokerProcess(broker_cfg) print( f"==> starting broker {broker_name} on {broker_cfg.port} (default_partitions={broker_cfg.default_partitions})", flush=True, ) broker.start() - service_url = f"pulsar://127.0.0.1:{broker_cfg.port}" + service_url = ( + args.external_url + if args.broker_backend == "external" + else f"pulsar://127.0.0.1:{broker_cfg.port}" + ) try: for scenario in [s for s in SCENARIOS if s.broker == broker_name]: topic = scenario_topic(run_id, scenario) @@ -399,15 +449,21 @@ def main() -> int: topic, scenario_dir / "feed_producer.hdr", ) - consumer_text, producer_text, consumer_rc, producer_rc = ( + consumer_text, producer_text, consumer_rc, producer_rc, first_failed = ( run_consumer_then_feed( consumer_cmd, producer_cmd, consumer_log, producer_log ) ) - if producer_rc != 0: - raise RuntimeError(f"producer failed:\n{producer_text}") - if consumer_rc != 0: - raise RuntimeError(f"consumer failed:\n{consumer_text}") + if consumer_rc != 0 or producer_rc != 0: + raise RuntimeError( + format_e2e_process_failure( + consumer_rc=consumer_rc, + producer_rc=producer_rc, + consumer_out=consumer_text, + producer_out=producer_text, + first_failed=first_failed, + ) + ) result_entry["producer_metrics"] = parse_producer_output(producer_text) result_entry["metrics"] = parse_consumer_output(consumer_text) else: diff --git a/tests/perf/run_non_persistent_stress.py b/tests/perf/run_non_persistent_stress.py index 3f1081d..f5afdc7 100644 --- a/tests/perf/run_non_persistent_stress.py +++ b/tests/perf/run_non_persistent_stress.py @@ -23,12 +23,18 @@ # --- lib imports (run from repo root or with sys.path adjusted) --- sys.path.insert(0, str(Path(__file__).resolve().parent)) from lib import ROOT -from lib.broker import BrokerConfig, BrokerProcess, DockerBrokerProcess +from lib.broker import ( + BrokerConfig, + BrokerProcess, + DockerBrokerProcess, + ExternalBrokerProcess, +) from lib.docker_image import build_broker_image from lib.observability import PerfCollector from lib.parsing import parse_consumer_output, parse_producer_output from lib.perf_cmd import ( ensure_prereqs, + format_e2e_process_failure, perf_cmd, run_consumer_then_feed, run_sync, @@ -63,19 +69,20 @@ class StressScenario: STRESS_SCENARIOS: list[StressScenario] = [ # --- Producer stress --- + # Aligned with persistent stress on -r / -o / -c / consumer -q where comparable. StressScenario( name="stress_producer_max_rate", kind="produce", broker="nonpartitioned", - description="单 producer 1M msg/s offered load 吞吐 ceiling", - producer_args=["-time", "60", "-r", "999999", "-s", "1024"], + description="单 producer 全速 60s(-r/-o 对齐 persistent)", + producer_args=["-time", "60", "-r", "999999", "-s", "1024", "-o", "1000"], estimated_duration=60, ), StressScenario( name="stress_producer_max_rate_multi_producer", kind="produce", broker="nonpartitioned", - description="4 producers 1M msg/s aggregate offered load 并发吞吐 ceiling", + description="4 producers 并发全速 60s(-r/-o 对齐 persistent)", producer_args=[ "-time", "60", @@ -84,11 +91,13 @@ class StressScenario: "-s", "1024", "-n", - "4", + "1", "-threads", "4", "-c", "4", + "-o", + "1000", ], estimated_duration=60, ), @@ -96,8 +105,8 @@ class StressScenario: name="stress_producer_large_payload", kind="produce", broker="nonpartitioned", - description="100KiB payload 1M msg/s offered load 带宽瓶颈", - producer_args=["-time", "60", "-r", "999999", "-s", "102400"], + description="100KiB payload 全速 60s(-o 对齐 persistent)", + producer_args=["-time", "60", "-r", "999999", "-s", "102400", "-o", "1000"], estimated_duration=60, ), StressScenario( @@ -105,7 +114,7 @@ class StressScenario: kind="produce", broker="nonpartitioned", description="5 分钟持续发送稳定性", - producer_args=["-time", "300", "-r", "999999", "-s", "1024"], + producer_args=["-time", "300", "-r", "999999", "-s", "1024", "-o", "1000"], estimated_duration=300, ), # --- Consumer / E2E stress --- @@ -113,67 +122,48 @@ class StressScenario: name="stress_consume_shared_max_rate", kind="consume_e2e", broker="nonpartitioned", - description="Shared 单 consumer 1M msg/s offered load 吞吐 ceiling", + description="Shared 单 consumer 全速 60s(-q/-o 对齐 persistent)", producer_args=[], - consumer_args=["-time", "60", "-q", "10000", "-st", "Shared"], - feed_producer_args=["-time", "60", "-r", "999999", "-s", "1024"], + consumer_args=["-time", "60", "-q", "1000", "-st", "Shared"], + feed_producer_args=["-time", "60", "-r", "999999", "-s", "1024", "-o", "1000"], estimated_duration=60, ), StressScenario( name="stress_consume_shared_high_fanout", kind="consume_e2e", broker="nonpartitioned", - description="Shared 16 consumers 1M msg/s offered load 高 fanout", + description="Shared 16 consumers 高扇出 60s(-q 对齐,去掉 -c)", producer_args=[], consumer_args=[ "-time", "60", "-q", - "10000", + "1000", "-st", "Shared", "-n", "16", - "-c", - "4", ], - feed_producer_args=["-time", "60", "-r", "999999", "-s", "1024", "-c", "4"], + feed_producer_args=["-time", "60", "-r", "999999", "-s", "1024", "-o", "1000"], estimated_duration=60, ), StressScenario( name="stress_consume_multi_subscription_fanout", kind="consume_e2e", broker="nonpartitioned", - description="8 subscriptions 1M msg/s offered load 高 fanout", + description="8 subscriptions 扇出 60s(-q/-o 对齐,去掉 -c/memory-limit)", producer_args=[], consumer_args=[ "-time", "60", "-q", - "10000", + "1000", "-st", "Shared", "-ns", "8", - "-c", - "4", - ], - feed_producer_args=[ - "-time", - "60", - "-r", - "999999", - "-s", - "1024", - "-c", - "4", - "--memory-limit", - "268435456", - "--max-outstanding", - "4096", - "--max-outstanding-across-partitions", - "16384", ], + feed_producer_args=["-time", "60", "-r", "999999", "-s", "1024", "-o", "1000"], estimated_duration=60, ), StressScenario( @@ -182,29 +172,27 @@ class StressScenario: broker="nonpartitioned", description="5 分钟持续消费稳定性", producer_args=[], - consumer_args=["-time", "300", "-q", "10000", "-st", "Shared", "-c", "4"], - feed_producer_args=["-time", "300", "-r", "999999", "-s", "1024", "-c", "4"], + consumer_args=["-time", "300", "-q", "1000", "-st", "Shared"], + feed_producer_args=["-time", "300", "-r", "999999", "-s", "1024", "-o", "1000"], estimated_duration=300, ), StressScenario( name="stress_consume_partitioned_max_rate", kind="consume_e2e", broker="nonpersistent_partitioned", - description="Partitioned 4 partitions Shared 4 consumers 1M msg/s offered load", + description="Partitioned 4 partitions Shared 4 consumers 60s(-q 对齐,去掉 -c)", producer_args=[], consumer_args=[ "-time", "60", "-q", - "10000", + "1000", "-st", "Shared", "-n", "4", - "-c", - "4", ], - feed_producer_args=["-time", "60", "-r", "999999", "-s", "1024", "-c", "4"], + feed_producer_args=["-time", "60", "-r", "999999", "-s", "1024", "-o", "1000"], estimated_duration=60, ), ] @@ -221,9 +209,19 @@ def build_arg_parser() -> argparse.ArgumentParser: ) parser.add_argument( "--broker-backend", - choices=["local", "docker"], + choices=["local", "docker", "external"], default="local", - help="Broker launch backend. local uses rust/target/release/pulsar-lite; docker builds and runs a constrained broker container.", + help="Broker launch backend. local uses rust/target/release/pulsar-lite; docker builds and runs a constrained broker container; external targets an already-running broker via --external-url.", + ) + parser.add_argument( + "--external-url", + default="pulsar://127.0.0.1:6650", + help="Service URL of the external broker for --broker-backend=external.", + ) + parser.add_argument( + "--external-unit", + default=None, + help="systemd unit name of the external broker (e.g. pulsar-standalone or pulsar-lite) to sample CPU/memory via 'systemctl show -p MainPID' + /proc.", ) parser.add_argument( @@ -273,7 +271,9 @@ def _topic_for(run_id: str, scenario: StressScenario) -> str: return f"non-persistent://public/default/{run_id}-{scenario.name}" -def _service_url_for(scenario: StressScenario) -> str: +def _service_url_for(scenario: StressScenario, external_url: str | None = None) -> str: + if external_url: + return external_url cfg = BROKERS[scenario.broker] return f"pulsar://127.0.0.1:{cfg.port}" @@ -284,10 +284,11 @@ def _run_produce_scenario( scenario_dir: Path, broker_proc: BrokerProcess, perf_collector: PerfCollector | None, + external_url: str | None = None, ) -> dict: """Execute a single producer-only stress scenario.""" topic = _topic_for(run_id, scenario) - service_url = _service_url_for(scenario) + service_url = _service_url_for(scenario, external_url) timeout = scenario.estimated_duration + 120 histogram_path = scenario_dir / "producer_histogram.log" @@ -337,10 +338,11 @@ def _run_consume_e2e_scenario( scenario_dir: Path, broker_proc: BrokerProcess, perf_collector: PerfCollector | None, + external_url: str | None = None, ) -> dict: """Execute a single consumer e2e stress scenario (feed-producer + consumer).""" topic = _topic_for(run_id, scenario) - service_url = _service_url_for(scenario) + service_url = _service_url_for(scenario, external_url) timeout = scenario.estimated_duration + 120 consumer_args = scenario.consumer_args or [] @@ -356,7 +358,7 @@ def _run_consume_e2e_scenario( started_at = datetime.now(timezone.utc).isoformat() t0 = time.monotonic() - consumer_out, producer_out, consumer_rc, producer_rc = run_consumer_then_feed( + consumer_out, producer_out, consumer_rc, producer_rc, first_failed = run_consumer_then_feed( consumer_cmd, producer_cmd, consumer_log, @@ -375,8 +377,23 @@ def _run_consume_e2e_scenario( status = ( "ok" if consumer_rc == 0 and producer_rc == 0 - else (f"consumer_exit:{consumer_rc},producer_exit:{producer_rc}") + else ( + f"consumer_exit:{consumer_rc},producer_exit:{producer_rc}" + f",first_failed:{first_failed}" + ) ) + if consumer_rc != 0 or producer_rc != 0: + # Keep going (non-persistent stress soft-fails), but print full dual-rc diagnosis. + print( + format_e2e_process_failure( + consumer_rc=consumer_rc, + producer_rc=producer_rc, + consumer_out=consumer_out, + producer_out=producer_out, + first_failed=first_failed, + ), + flush=True, + ) result: dict = { "name": scenario.name, "kind": scenario.kind, @@ -463,6 +480,18 @@ def main(argv: list[str] | None = None) -> None: cpuset_cpus=args.docker_cpuset, memory=args.docker_memory, ) + elif args.broker_backend == "external": + from urllib.parse import urlparse + + parsed = urlparse(args.external_url) + if parsed.scheme != "pulsar" or not parsed.hostname: + raise ValueError( + f"--external-url must be pulsar://host:port, got {args.external_url!r}" + ) + bp = ExternalBrokerProcess( + BrokerConfig("external", parsed.port or 6650, 0), + unit=args.external_unit, + ) else: bp = BrokerProcess(cfg) bp.start() @@ -482,8 +511,9 @@ def main(argv: list[str] | None = None) -> None: scenario_dir.mkdir(parents=True, exist_ok=True) # Restart broker between scenarios to clear residual topics/subscriptions - print(f" restarting broker [{scenario.broker}] ...", file=sys.stderr) - broker_proc.restart() + if args.broker_backend != "external": + print(f" restarting broker [{scenario.broker}] ...", file=sys.stderr) + broker_proc.restart() # Start perf recording (must be after restart to capture the new PID) perf_data_path = scenario_dir / "perf.data" @@ -506,6 +536,7 @@ def main(argv: list[str] | None = None) -> None: scenario_dir, broker_proc, perf_collector, + args.external_url if args.broker_backend == "external" else None, ) elif scenario.kind == "consume_e2e": result = _run_consume_e2e_scenario( @@ -514,6 +545,7 @@ def main(argv: list[str] | None = None) -> None: scenario_dir, broker_proc, perf_collector, + args.external_url if args.broker_backend == "external" else None, ) else: print(f" UNKNOWN kind: {scenario.kind}, skipping", file=sys.stderr) @@ -526,7 +558,10 @@ def main(argv: list[str] | None = None) -> None: "name": scenario.name, "kind": scenario.kind, "broker_profile": scenario.broker, - "service_url": _service_url_for(scenario), + "service_url": _service_url_for( + scenario, + args.external_url if args.broker_backend == "external" else None, + ), "description": scenario.description, "topic": _topic_for(run_id, scenario), "started_at": datetime.now(timezone.utc).isoformat(), diff --git a/tests/perf/run_persistent_e2e_matrix.py b/tests/perf/run_persistent_e2e_matrix.py index bec08fd..a9985b0 100755 --- a/tests/perf/run_persistent_e2e_matrix.py +++ b/tests/perf/run_persistent_e2e_matrix.py @@ -17,10 +17,22 @@ sys.path.insert(0, str(Path(__file__).resolve().parent)) from lib import ROOT -from lib.broker import BrokerConfig, BrokerProcess, DockerBrokerProcess +from lib.broker import ( + BrokerConfig, + BrokerProcess, + DockerBrokerProcess, + ExternalBrokerProcess, +) from lib.docker_image import build_broker_image from lib.parsing import parse_consumer_output, parse_producer_output -from lib.perf_cmd import ensure_prereqs, perf_cmd, run_consumer_then_feed, run_sync +from lib.perf_cmd import ( + ensure_prereqs, + e2e_success_despite_peer_hang, + format_e2e_process_failure, + perf_cmd, + run_consumer_then_feed, + run_sync, +) RESULTS_PATH = ROOT / "docs" / "perf" / "data" / "persistent_e2e_matrix_results.json" ARTIFACTS_DIR = ROOT / "docs" / "perf" / "data" / "persistent_e2e_matrix_logs" @@ -317,16 +329,23 @@ def run_consume_e2e_scenario( ) start = time.time() - consumer_out, producer_out, consumer_rc, producer_rc = run_consumer_then_feed( + consumer_out, producer_out, consumer_rc, producer_rc, first_failed = run_consumer_then_feed( consumer_cmd, producer_cmd, consumer_log, producer_log, consumer_timeout=120.0, producer_timeout=120.0, ) duration = time.time() - start - - if consumer_rc != 0: - raise RuntimeError(f"consumer failed with rc={consumer_rc}: {consumer_out[:500]}") - if producer_rc != 0: - raise RuntimeError(f"producer failed with rc={producer_rc}: {producer_out[:500]}") + + if consumer_rc != 0 or producer_rc != 0: + if not e2e_success_despite_peer_hang(consumer_rc, producer_rc, first_failed): + raise RuntimeError( + format_e2e_process_failure( + consumer_rc=consumer_rc, + producer_rc=producer_rc, + consumer_out=consumer_out, + producer_out=producer_out, + first_failed=first_failed, + ) + ) consumer_result = parse_consumer_output(consumer_out) producer_result = parse_producer_output(producer_out) @@ -346,6 +365,11 @@ def run_restart_smoke_scenario( run_dir: Path, ) -> dict[str, Any]: """Run restart scenario: produce → restart → consume.""" + if isinstance(broker, ExternalBrokerProcess): + raise RuntimeError( + "external broker backend cannot restart the broker; " + "use scenarios that do not restart (produce / consume_e2e)" + ) topic = f"persistent://public/default/test-{uuid.uuid4().hex[:8]}" # Step 1: Produce messages @@ -435,10 +459,24 @@ def main(argv: list[str]) -> int: ) parser.add_argument( "--broker-backend", - choices=["local","docker"], + choices=["local", "docker", "external"], default="local", help="Broker launch backend. local user rust/target/release/pulsar-lite; " - "docker builds and runs a constrained broker container.", + "docker builds and runs a constrained broker container; " + "external targets an already-running broker via --external-url " + "(e.g. Apache Pulsar standalone); no broker lifecycle management.", + ) + parser.add_argument( + "--external-url", + default="pulsar://127.0.0.1:6650", + help="Service URL of the external broker for --broker-backend=external.", + ) + parser.add_argument( + "--external-unit", + default=None, + help="systemd unit name of the external broker (e.g. pulsar-standalone or " + "pulsar-lite) to sample CPU/memory via 'systemctl show -p MainPID' " + "+ /proc. Leave unset to disable broker metrics for external backends.", ) parser.add_argument( "--docker-cpuset", @@ -502,6 +540,18 @@ def main(argv: list[str]) -> int: cpuset_cpus=args.docker_cpuset, memory=args.docker_memory, ) + elif args.broker_backend == "external": + from urllib.parse import urlparse + + parsed = urlparse(args.external_url) + if parsed.scheme != "pulsar" or not parsed.hostname: + raise ValueError( + f"--external-url must be pulsar://host:port, got {args.external_url!r}" + ) + broker = ExternalBrokerProcess( + BrokerConfig("external", parsed.port or 6650, 0), + unit=args.external_unit, + ) else: broker = BrokerProcess(broker_config) diff --git a/tests/perf/run_persistent_stress.py b/tests/perf/run_persistent_stress.py index 2083b1e..65cb65a 100755 --- a/tests/perf/run_persistent_stress.py +++ b/tests/perf/run_persistent_stress.py @@ -17,11 +17,18 @@ sys.path.insert(0, str(Path(__file__).resolve().parent)) from lib import ROOT -from lib.broker import BrokerConfig, BrokerProcess, DockerBrokerProcess +from lib.broker import BrokerConfig, BrokerProcess, DockerBrokerProcess, ExternalBrokerProcess from lib.docker_image import build_broker_image from lib.observability import PerfCollector from lib.parsing import parse_consumer_output, parse_producer_output -from lib.perf_cmd import ensure_prereqs, perf_cmd, run_consumer_then_feed, run_sync +from lib.perf_cmd import ( + ensure_prereqs, + e2e_success_despite_peer_hang, + format_e2e_process_failure, + perf_cmd, + run_consumer_then_feed, + run_sync, +) RESULTS_PATH = ROOT / "docs" / "perf" / "data" / "persistent_stress_results.json" ARTIFACTS_DIR = ROOT / "docs" / "perf" / "data" / "persistent_stress_logs" @@ -54,17 +61,17 @@ class Scenario: name="stress_persistent_producer_max_rate", kind="produce", broker="persistent_stress", - description="单 producer 全速发送 200w 条", - producer_args=["-m", "2000000", "-s", "1024", "-r", "999999", "-o", "1000"], + description="单 producer 全速 60s(对齐 non-persistent -time)", + producer_args=["-time", "60", "-s", "1024", "-r", "999999", "-o", "1000"], ), Scenario( name="stress_persistent_producer_multi_producer", kind="produce", broker="persistent_stress", - description="4 producers 并发全速发送 200w 条", + description="4 producers 并发全速 60s", producer_args=[ - "-m", - "2000000", + "-time", + "60", "-s", "1024", "-r", @@ -83,52 +90,89 @@ class Scenario: name="stress_persistent_producer_large_payload", kind="produce", broker="persistent_stress", - description="100KiB payload 发送 10k 条", - # 10k * 100KiB ≈ 1GiB payload; 200k would be ~19GiB - producer_args=["-m", "10000", "-s", "102400", "-r", "500"], + description="100KiB payload 全速 60s(对齐 non-persistent)", + producer_args=["-time", "60", "-s", "102400", "-r", "999999", "-o", "1000"], ), Scenario( name="stress_persistent_producer_sustained", kind="produce", broker="persistent_stress", - description="持续限速发送 500k 条 @ 10k msg/s (~50s)", - producer_args=["-m", "500000", "-s", "1024", "-r", "10000"], + description="限速 10k msg/s 持续 60s", + producer_args=["-time", "60", "-s", "1024", "-r", "10000", "-o", "1000"], ), - # Consumer/E2E stress (4) + # Consumer/E2E stress (4) — -time 60, no -m (steady thr via interval median) Scenario( name="stress_persistent_consume_shared_max_rate", kind="consume_e2e", broker="persistent_stress", - description="Shared 全速消费 200k 条", - consumer_args=["-m", "200000", "-q", "1000", "-st", "Shared"], - feed_producer_args=["-m", "200000", "-s", "1024", "-r", "999999"], + description="Shared 单 consumer 全速 60s", + consumer_args=["-time", "60", "-q", "1000", "-st", "Shared"], + feed_producer_args=["-time", "60", "-s", "1024", "-r", "999999", "-o", "1000"], ), Scenario( name="stress_persistent_consume_shared_high_fanout", kind="consume_e2e", broker="persistent_stress", - description="16 consumers 高扇出消费 200k 条", - consumer_args=["-m", "200000", "-q", "1000", "-st", "Shared", "-n", "16"], - feed_producer_args=["-m", "200000", "-s", "1024", "-r", "999999"], + description="16 consumers 高扇出 6s [TEMP]", + consumer_args=[ + "-time", + "6", + "-q", + "1000", + "-st", + "Shared", + "-n", + "16", + ], + feed_producer_args=["-time", "6", "-s", "1024", "-r", "999999", "-o", "1000"], ), Scenario( name="stress_persistent_consume_multi_subscription_fanout", kind="consume_e2e", broker="persistent_stress", - description="8 subscriptions 扇出:生产 100k / 消费 800k 条", - # each subscription receives a full copy; consumer -m = produce * ns - consumer_args=["-m", "800000", "-q", "1000", "-st", "Shared", "-ns", "8"], - feed_producer_args=["-m", "100000", "-s", "1024", "-r", "999999"], + description="8 subscriptions 扇出 60s", + consumer_args=[ + "-time", + "60", + "-q", + "1000", + "-st", + "Shared", + "-ns", + "8", + ], + feed_producer_args=["-time", "60", "-s", "1024", "-r", "999999", "-o", "1000"], ), Scenario( name="stress_persistent_consume_partitioned_max_rate", kind="consume_e2e", broker="persistent_stress_partitioned", - description="4 partitions + 4 consumers 消费 200k 条", - consumer_args=["-m", "200000", "-q", "1000", "-st", "Shared", "-n", "4"], - feed_producer_args=["-m", "200000", "-s", "1024", "-r", "999999"], + description="4 partitions + 4 consumers 60s", + consumer_args=[ + "-time", + "60", + "-q", + "1000", + "-st", + "Shared", + "-n", + "4", + ], + feed_producer_args=["-time", "60", "-s", "1024", "-r", "999999", "-o", "1000"], + ), + # Persistent-specific stress (4) + Scenario( + name="stress_persistent_consume_solo_10gb", + kind="backlog_drain", + broker="persistent_stress", + description="10GiB backlog(10485760×1KiB)→ 单 consumer 独立消费测速(produce 完成后才消费)", + producer_args=[ + "-m", "10485760", "-r", "999999", "-s", "1024", "-db", "-o", "1000", + ], + consumer_args=[ + "-m", "10485760", "-q", "1000", "-st", "Shared", "-sp", "Earliest", + ], ), - # Persistent-specific stress (3) Scenario( name="stress_persistent_backlog_drain", kind="backlog_drain", @@ -226,23 +270,34 @@ def run_consume_e2e_scenario( ) start = time.time() - consumer_out, producer_out, consumer_rc, producer_rc = run_consumer_then_feed( - consumer_cmd, - producer_cmd, - consumer_log, - producer_log, - consumer_timeout=600.0, - producer_timeout=600.0, + consumer_out, producer_out, consumer_rc, producer_rc, first_failed = ( + run_consumer_then_feed( + consumer_cmd, + producer_cmd, + consumer_log, + producer_log, + consumer_timeout=600.0, + producer_timeout=600.0, + ) ) duration = time.time() - start - if consumer_rc != 0: - raise RuntimeError( - f"consumer failed with rc={consumer_rc}: {consumer_out[:500]}" - ) - if producer_rc != 0: - raise RuntimeError( - f"producer failed with rc={producer_rc}: {producer_out[:500]}" + if consumer_rc != 0 or producer_rc != 0: + if not e2e_success_despite_peer_hang(consumer_rc, producer_rc, first_failed): + raise RuntimeError( + format_e2e_process_failure( + consumer_rc=consumer_rc, + producer_rc=producer_rc, + consumer_out=consumer_out, + producer_out=producer_out, + first_failed=first_failed, + ) + ) + print( + " note: peer grace-killed after other side ok " + f"(first_failed={first_failed}, " + f"consumer_rc={consumer_rc}, producer_rc={producer_rc})", + flush=True, ) consumer_result = parse_consumer_output(consumer_out) @@ -254,6 +309,7 @@ def run_consume_e2e_scenario( "producer": producer_result, "broker": broker_metrics, "duration_s": round(duration, 2), + "e2e_first_failed": first_failed, } @@ -277,7 +333,7 @@ def run_backlog_drain_scenario( ) print(" Producing backlog...") - producer_proc = run_sync(producer_cmd, producer_log, timeout=600.0) + producer_proc = run_sync(producer_cmd, producer_log, timeout=1800.0) if producer_proc.returncode != 0: raise RuntimeError(f"producer failed: {producer_log.read_text()[:500]}") @@ -297,7 +353,7 @@ def run_backlog_drain_scenario( print(" Draining backlog...") start = time.time() - consumer_proc = run_sync(consumer_cmd, consumer_log, timeout=600.0) + consumer_proc = run_sync(consumer_cmd, consumer_log, timeout=1800.0) drain_duration = time.time() - start if consumer_proc.returncode != 0: @@ -312,6 +368,7 @@ def run_backlog_drain_scenario( "broker": broker_metrics, "drain_duration_s": round(drain_duration, 2), "drain_throughput_msg_s": round(consumer_result["records"] / drain_duration, 2), + "drain_throughput_mbit_s": round(consumer_result["throughput_mbit_s"], 2), } @@ -411,19 +468,28 @@ def run_redelivery_unacked_scenario( ) print(" Partial consume...") - consumer_out, producer_out, consumer_rc, producer_rc = run_consumer_then_feed( - consumer_cmd, - producer_cmd, - consumer1_log, - producer_log, - consumer_timeout=600.0, - producer_timeout=600.0, + consumer_out, producer_out, consumer_rc, producer_rc, first_failed = ( + run_consumer_then_feed( + consumer_cmd, + producer_cmd, + consumer1_log, + producer_log, + consumer_timeout=600.0, + producer_timeout=600.0, + ) ) - if consumer_rc != 0: - raise RuntimeError(f"consumer1 failed: {consumer_out[:500]}") - if producer_rc != 0: - raise RuntimeError(f"producer failed: {producer_out[:500]}") + if consumer_rc != 0 or producer_rc != 0: + raise RuntimeError( + format_e2e_process_failure( + consumer_rc=consumer_rc, + producer_rc=producer_rc, + consumer_out=consumer_out, + producer_out=producer_out, + first_failed=first_failed, + consumer_label="consumer1", + ) + ) producer_result = parse_producer_output(producer_out) consumer1_result = parse_consumer_output(consumer_out) @@ -514,10 +580,24 @@ def main(argv: list[str]) -> int: ) parser.add_argument( "--broker-backend", - choices=["local", "docker"], + choices=["local", "docker", "external"], default="local", help="Broker launch backend. local uses rust/target/release/pulsar-lite; " - "docker builds and runs a constrained broker container.", + "docker builds and runs a constrained broker container; " + "external targets an already-running broker via --external-url " + "(e.g. Apache Pulsar standalone); no broker lifecycle management.", + ) + parser.add_argument( + "--external-url", + default="pulsar://127.0.0.1:6650", + help="Service URL of the external broker for --broker-backend=external.", + ) + parser.add_argument( + "--external-unit", + default=None, + help="systemd unit name of the external broker (e.g. pulsar-standalone or " + "pulsar-lite) to sample CPU/memory via 'systemctl show -p MainPID' " + "+ /proc. Leave unset to disable broker metrics for external backends.", ) parser.add_argument( "--docker-cpuset", @@ -529,6 +609,19 @@ def main(argv: list[str]) -> int: default="4g", help="Memory limit passed to docker run --memory when --broker-backend=docker.", ) + parser.add_argument( + "--local-cgroup-memory", + default=None, + help="local backend: MemoryMax via systemd-run --user --scope " + "(e.g. 4294967296 or 4G; MemorySwapMax=0 pinned). Requires user-scope " + "cgroup delegation for the memory controller.", + ) + parser.add_argument( + "--local-cgroup-cpus", + default=None, + help="local backend: broker CPU affinity via taskset -c (e.g. 0-3), " + "equivalent to docker --cpuset-cpus.", + ) parser.add_argument( "--skip-docker-build", action="store_true", @@ -589,8 +682,24 @@ def main(argv: list[str]) -> int: cpuset_cpus=args.docker_cpuset, memory=args.docker_memory, ) + elif args.broker_backend == "external": + from urllib.parse import urlparse + + parsed = urlparse(args.external_url) + if parsed.scheme != "pulsar" or not parsed.hostname: + raise ValueError( + f"--external-url must be pulsar://host:port, got {args.external_url!r}" + ) + broker = ExternalBrokerProcess( + BrokerConfig("external", parsed.port or 6650, 0), + unit=args.external_unit, + ) else: - broker = BrokerProcess(broker_config) + broker = BrokerProcess( + broker_config, + cgroup_memory=args.local_cgroup_memory, + cgroup_cpus=args.local_cgroup_cpus, + ) broker.start() try: @@ -599,6 +708,10 @@ def main(argv: list[str]) -> int: print(f"\n[{scenario.name}] {scenario.description}") scenario_dir = run_artifacts / scenario.name scenario_dir.mkdir(parents=True, exist_ok=True) + # Reset the broker sampler so metrics() / timeseries CSV cover + # only this scenario's window, not the whole suite run. + if broker.sampler: + broker.sampler.reset() # Start perf recording (must be after restart to capture the new PID) perf_data_path = scenario_dir / "perf.data" perf_collector: PerfCollector | None = None @@ -612,7 +725,7 @@ def main(argv: list[str]) -> int: try: result = run_scenario(scenario, broker, scenario_dir) - # Save broker log and timeseries + # Save broker log and timeseries while workdir still exists. if broker.log_path: (scenario_dir / "broker.log").write_text( broker.log_path.read_text( @@ -623,7 +736,8 @@ def main(argv: list[str]) -> int: broker.sampler.write_csv(scenario_dir / "broker_timeseries.csv") # Record artifact paths in result - perf_collector.stop() + if perf_collector is not None: + perf_collector.stop() if perf_data_path.exists(): svg_path = scenario_dir / "flamegraph.svg" ok = PerfCollector.generate_flamegraph(perf_data_path, svg_path) @@ -656,6 +770,16 @@ def main(argv: list[str]) -> int: except Exception as e: print(f" ERROR: {e}", file=sys.stderr) + # Best-effort capture before wiping storage. + try: + if broker.log_path and broker.log_path.exists(): + (scenario_dir / "broker.log").write_text( + broker.log_path.read_text( + encoding="utf-8", errors="replace" + ) + ) + except OSError: + pass if perf_collector is not None: perf_collector.stop() results["scenarios"].append( @@ -670,6 +794,24 @@ def main(argv: list[str]) -> int: ) failed += 1 print(f" ✗ FAIL: {e}") + finally: + # Drop /tmp DB+entrylog after each scenario so storage does not accumulate. + # restart_replay / redelivery manage their own in-scenario preserve restart; + # once the scenario finishes we always wipe before the next one. + # external backend owns its storage; nothing to reset. + if args.broker_backend != "external": + try: + broker.restart(preserve_storage=False) + print( + " storage cleaned for next scenario " + f"(workdir={broker.workdir})", + file=sys.stderr, + ) + except Exception as cleanup_err: + print( + f" WARNING: failed to reset broker storage: {cleanup_err}", + file=sys.stderr, + ) finally: broker.stop(cleanup=True) diff --git a/tests/perf/unit/test_parsing.py b/tests/perf/unit/test_parsing.py new file mode 100644 index 0000000..91fa29d --- /dev/null +++ b/tests/perf/unit/test_parsing.py @@ -0,0 +1,84 @@ +"""Unit tests for pulsar-perf log parsing (interval-median thr).""" + +from __future__ import annotations + +import sys +from pathlib import Path + +import pytest + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT)) + +from lib.parsing import parse_consumer_output, parse_producer_output # noqa: E402 + + +PRODUCER_LOG = """ +2026-07-30T18:00:00,000 - INFO - [main:PerformanceProducer@392] - Throughput produced: 100000 msg --- 10000.0 msg/s --- 78.1 Mbit/s --- failure 0.0 msg/s --- Latency: mean: 10.000 ms - med: 9.000 - 95pct: 20.000 - 99pct: 30.000 - 99.9pct: 40.000 - 99.99pct: 50.000 - Max: 60.000 +2026-07-30T18:00:10,000 - INFO - [main:PerformanceProducer@392] - Throughput produced: 300000 msg --- 20000.0 msg/s --- 156.2 Mbit/s --- failure 0.0 msg/s --- Latency: mean: 5.000 ms - med: 4.000 - 95pct: 10.000 - 99pct: 15.000 - 99.9pct: 20.000 - 99.99pct: 25.000 - Max: 30.000 +2026-07-30T18:00:20,000 - INFO - [main:PerformanceProducer@392] - Throughput produced: 500000 msg --- 22000.0 msg/s --- 171.9 Mbit/s --- failure 0.0 msg/s --- Latency: mean: 6.000 ms - med: 5.000 - 95pct: 11.000 - 99pct: 16.000 - 99.9pct: 21.000 - 99.99pct: 26.000 - Max: 35.000 +2026-07-30T18:00:30,000 - INFO - [main:PerformanceProducer@392] - Throughput produced: 700000 msg --- 18000.0 msg/s --- 140.6 Mbit/s --- failure 0.0 msg/s --- Latency: mean: 7.000 ms - med: 6.000 - 95pct: 12.000 - 99pct: 17.000 - 99.9pct: 22.000 - 99.99pct: 27.000 - Max: 40.000 +2026-07-30T18:00:40,000 - INFO - [perf-client-shutdown:PerformanceProducer@774] - Aggregated throughput stats --- 700000 records sent --- 11666.667 msg/s --- 91.146 Mbit/s +2026-07-30T18:00:40,000 - INFO - [perf-client-shutdown:PerformanceProducer@784] - Aggregated latency stats --- Latency: mean: 8.000 ms - med: 7.000 - 95pct: 13.000 - 99pct: 18.000 - 99.9pct: 23.000 - 99.99pct: 28.000 - 99.999pct: 33.000 - Max: 45.000 +""" + +CONSUMER_LOG = """ +2026-07-30T18:00:00,000 - INFO - [main:PerformanceConsumer@484] - Throughput received: 50000 msg --- 5000.000 msg/s --- 39.062 Mbit/s --- Latency: mean: 100.000 ms - med: 90 - 95pct: 200 - 99pct: 300 - 99.9pct: 400 - 99.99pct: 500 - Max: 600 +2026-07-30T18:00:10,000 - INFO - [main:PerformanceConsumer@484] - Throughput received: 250000 msg --- 20000.000 msg/s --- 156.250 Mbit/s --- Latency: mean: 50.000 ms - med: 40 - 95pct: 80 - 99pct: 100 - 99.9pct: 120 - 99.99pct: 140 - Max: 160 +2026-07-30T18:00:20,000 - INFO - [main:PerformanceConsumer@484] - Throughput received: 470000 msg --- 22000.000 msg/s --- 171.875 Mbit/s --- Latency: mean: 55.000 ms - med: 45 - 95pct: 85 - 99pct: 105 - 99.9pct: 125 - 99.99pct: 145 - Max: 165 +2026-07-30T18:00:30,000 - INFO - [main:PerformanceConsumer@484] - Throughput received: 650000 msg --- 18000.000 msg/s --- 140.625 Mbit/s --- Latency: mean: 60.000 ms - med: 50 - 95pct: 90 - 99pct: 110 - 99.9pct: 130 - 99.99pct: 150 - Max: 170 +2026-07-30T18:00:40,000 - INFO - [perf-client-shutdown:PerformanceConsumer@562] - Aggregated throughput stats --- 650000 records received --- 10833.333 msg/s --- 84.635 Mbit/s --- AckRate: 10833.0 msg/s --- ack failed 0 msg +2026-07-30T18:00:40,000 - INFO - [perf-client-shutdown:PerformanceConsumer@575] - Aggregated latency stats --- Latency: mean: 70.000 ms - med: 60 - 95pct: 100 - 99pct: 120 - 99.9pct: 140 - 99.99pct: 160 - 99.999pct: 180 - Max: 200 +""" + +PRODUCER_AGG_ONLY = """ +2026-07-30T18:00:10,000 - INFO - [perf-client-shutdown:PerformanceProducer@774] - Aggregated throughput stats --- 10000 records sent --- 5000.000 msg/s --- 39.062 Mbit/s +2026-07-30T18:00:10,000 - INFO - [perf-client-shutdown:PerformanceProducer@784] - Aggregated latency stats --- Latency: mean: 1.500 ms - med: 1.000 - 95pct: 2.000 - 99pct: 3.000 - 99.9pct: 4.000 - 99.99pct: 5.000 - 99.999pct: 6.000 - Max: 7.000 +""" + + +def test_producer_prefers_interval_median_dropping_first_window(): + # windows: 10k, 20k, 22k, 18k → drop first → median(20,22,18)=20k + result = parse_producer_output(PRODUCER_LOG) + assert result["metric_source"] == "interval_median" + assert result["records"] == 700000 + assert result["throughput_msg_s"] == pytest.approx(20000.0) + assert result["interval_count"] == 4 + assert result["steady_interval_count"] == 3 + assert result["partial"] is False + assert result["aggregated_throughput_msg_s"] == pytest.approx(11666.667) + assert result["latency_p99_ms"] == pytest.approx(16.0) # median of 15,16,17 + assert result["latency_max_ms"] == pytest.approx(40.0) # max of steady windows + + +def test_consumer_prefers_interval_median_dropping_first_window(): + result = parse_consumer_output(CONSUMER_LOG) + assert result["metric_source"] == "interval_median" + assert result["records"] == 650000 + assert result["throughput_msg_s"] == pytest.approx(20000.0) + assert result["ack_failed"] == 0 + assert result["ack_rate_msg_s"] == pytest.approx(10833.0) + assert result["steady_interval_count"] == 3 + assert result["partial"] is False + + +def test_producer_falls_back_to_aggregated_when_no_intervals(): + result = parse_producer_output(PRODUCER_AGG_ONLY) + assert result["metric_source"] == "aggregated" + assert result["records"] == 10000 + assert result["throughput_msg_s"] == pytest.approx(5000.0) + assert result["interval_count"] == 0 + assert result["partial"] is False + + +def test_single_interval_marked_partial(): + single = """ +2026-07-30T18:00:10,000 - INFO - [main:PerformanceProducer@392] - Throughput produced: 100000 msg --- 15000.0 msg/s --- 117.2 Mbit/s --- failure 0.0 msg/s --- Latency: mean: 2.000 ms - med: 1.500 - 95pct: 3.000 - 99pct: 4.000 - 99.9pct: 5.000 - 99.99pct: 6.000 - Max: 7.000 +2026-07-30T18:00:15,000 - INFO - [perf-client-shutdown:PerformanceProducer@774] - Aggregated throughput stats --- 100000 records sent --- 6666.667 msg/s --- 52.083 Mbit/s +2026-07-30T18:00:15,000 - INFO - [perf-client-shutdown:PerformanceProducer@784] - Aggregated latency stats --- Latency: mean: 2.000 ms - med: 1.500 - 95pct: 3.000 - 99pct: 4.000 - 99.9pct: 5.000 - 99.99pct: 6.000 - 99.999pct: 7.000 - Max: 7.000 +""" + result = parse_producer_output(single) + assert result["metric_source"] == "interval_median" + assert result["throughput_msg_s"] == pytest.approx(15000.0) + assert result["partial"] is True + assert result["steady_interval_count"] == 1 diff --git a/tests/perf/unit/test_perf_cmd_e2e_diag.py b/tests/perf/unit/test_perf_cmd_e2e_diag.py new file mode 100644 index 0000000..1707aeb --- /dev/null +++ b/tests/perf/unit/test_perf_cmd_e2e_diag.py @@ -0,0 +1,54 @@ +"""Unit tests for E2E dual-process failure diagnostics.""" + +from __future__ import annotations + +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT)) + +from lib.perf_cmd import ( # noqa: E402 + explain_exit_code, + format_e2e_process_failure, + log_tail, +) + + +def test_explain_exit_code_sigterm(): + text = explain_exit_code(143) + assert "143" in text + assert "SIGTERM" in text + + +def test_format_e2e_shows_both_rcs_and_log_tails(): + msg = format_e2e_process_failure( + consumer_rc=143, + producer_rc=1, + consumer_out="consumer head\n" + "\n".join(f"c{i}" for i in range(100)), + producer_out=( + "Started performance test thread 0\n" + "Created 1 producers\n" + "Exception in thread: boom\n" + ), + first_failed="producer", + ) + assert "first_failed=producer" in msg + assert "consumer_rc=" in msg and "143" in msg + assert "producer_rc=" in msg and "1" in msg + assert "Exception in thread: boom" in msg + assert "--- producer log tail ---" in msg + assert "--- consumer log tail ---" in msg + # tail should not be only the log head + assert "consumer head" not in log_tail("\n".join(f"line{i}" for i in range(80))) + + +def test_infer_first_failed_when_consumer_sigterm_producer_nonzero(): + msg = format_e2e_process_failure( + consumer_rc=143, + producer_rc=1, + consumer_out="c-tail", + producer_out="p-tail-error", + first_failed=None, + ) + assert "first_failed=producer" in msg