From e366988b803bf2c8a19d43804e1ef07ce3150ba9 Mon Sep 17 00:00:00 2001 From: XTLine <2668543450@qq.com> Date: Thu, 30 Jul 2026 14:59:20 +0800 Subject: [PATCH 01/44] test: add append visibility test for Latest cursor --- .../tests/rocksdb_storage.rs | 53 ++++++++++++++++++- 1 file changed, 52 insertions(+), 1 deletion(-) diff --git a/rust/storage/managed-ledger-rocksdb/tests/rocksdb_storage.rs b/rust/storage/managed-ledger-rocksdb/tests/rocksdb_storage.rs index aa5e343..946dd8d 100644 --- a/rust/storage/managed-ledger-rocksdb/tests/rocksdb_storage.rs +++ b/rust/storage/managed-ledger-rocksdb/tests/rocksdb_storage.rs @@ -3,7 +3,9 @@ mod common; use common::*; -use pulsar_lite_storage_managed_ledger::ManagedLedgerStorage; +use pulsar_lite_storage_managed_ledger::{ + CursorInitOptions, InitialPosition, ManagedLedgerStorage, +}; use pulsar_lite_storage_managed_ledger_rocksdb::{test_support::keys, RocksDbManagedLedgerStorage}; use tempfile::tempdir; @@ -102,3 +104,52 @@ fn storage_normalizes_topic_url_and_encodes_cursor_name() { .unwrap() .is_none()); } + +#[test] +fn append_after_latest_cursor_is_visible_to_reads() { + // Reproduces the consumer-0-message bug: + // 1) subscribe/Latest warms the SharedLedger cache with empty in-memory info + // 2) producer appends through the write-queue worker's owned ledger copy + // 3) dispatch reads via SharedLedger and must still see the new entries + let dir = tempdir().unwrap(); + let db_path = dir.path().join("storage-append-visibility"); + let topic = "persistent://public/default/visibility"; + let subscription = "sub"; + + let mut storage = RocksDbManagedLedgerStorage::open(&db_path).unwrap(); + storage.create_topic(topic).unwrap(); + + // Warm the reader-side SharedLedger cache the same way subscribe does. + storage + .initialize_or_open_cursor( + topic, + subscription, + CursorInitOptions { + initial_position: InitialPosition::Latest, + start_message_id: None, + }, + ) + .unwrap(); + + assert_eq!( + storage.first_unacked_position(topic, subscription).unwrap(), + None, + "empty topic with Latest cursor should have no backlog" + ); + + let message_id = storage.append_message(topic, -1, b"hello-after-subscribe").unwrap(); + + let first = storage + .first_unacked_position(topic, subscription) + .unwrap() + .expect("appended entry must be visible as first unacked"); + assert_eq!(first.ledger_id, message_id.ledger); + assert_eq!(first.entry_id, message_id.entry); + + let entries = storage + .read_entries_from(topic, &first, 1) + .unwrap(); + assert_eq!(entries.len(), 1); + assert_eq!(entries[0].message_id, message_id); + assert_eq!(entries[0].payload, b"hello-after-subscribe"); +} From 793466e94caf4b7760970bb69d14bf0e82d73511 Mon Sep 17 00:00:00 2001 From: XTLine <2668543450@qq.com> Date: Thu, 30 Jul 2026 19:09:45 +0800 Subject: [PATCH 02/44] perf(storage): publish LAC via ArcSwap and drop ledger content locks --- rust/Cargo.lock | 10 + .../storage/managed-ledger-rocksdb/Cargo.toml | 1 + .../managed-ledger-rocksdb/src/factory.rs | 24 +- .../managed-ledger-rocksdb/src/ledger.rs | 369 +++++++++--------- .../storage/managed-ledger-rocksdb/src/lib.rs | 24 ++ .../managed-ledger-rocksdb/src/store.rs | 271 ++++++------- .../managed-ledger-rocksdb/src/write_queue.rs | 50 +-- .../tests/rocksdb_cursor.rs | 32 +- .../tests/rocksdb_keys.rs | 8 +- .../tests/rocksdb_ledger.rs | 139 ++++--- 10 files changed, 460 insertions(+), 468 deletions(-) diff --git a/rust/Cargo.lock b/rust/Cargo.lock index 866777f..919c3d2 100644 --- a/rust/Cargo.lock +++ b/rust/Cargo.lock @@ -91,6 +91,15 @@ version = "1.0.102" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f202df86484c868dbad7eaa557ef785d5c66295e41b460ef922eca0723b842c" +[[package]] +name = "arc-swap" +version = "1.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c049c0be4daef0b145cb3555416b3b8ef5b7888a38aea1a3a155801fe7b0810b" +dependencies = [ + "rustversion", +] + [[package]] name = "async-trait" version = "0.1.89" @@ -865,6 +874,7 @@ name = "pulsar-lite-storage-managed-ledger-rocksdb" version = "0.1.0" dependencies = [ "anyhow", + "arc-swap", "bincode", "log", "prost", diff --git a/rust/storage/managed-ledger-rocksdb/Cargo.toml b/rust/storage/managed-ledger-rocksdb/Cargo.toml index b66a022..18cc9ee 100644 --- a/rust/storage/managed-ledger-rocksdb/Cargo.toml +++ b/rust/storage/managed-ledger-rocksdb/Cargo.toml @@ -13,6 +13,7 @@ serde.workspace = true bincode = "1.3" log.workspace = true tokio = { workspace = true } +arc-swap = "1.7" [build-dependencies] prost-build = "0.13" diff --git a/rust/storage/managed-ledger-rocksdb/src/factory.rs b/rust/storage/managed-ledger-rocksdb/src/factory.rs index b12f371..1d3aae2 100644 --- a/rust/storage/managed-ledger-rocksdb/src/factory.rs +++ b/rust/storage/managed-ledger-rocksdb/src/factory.rs @@ -9,13 +9,16 @@ use std::sync::Mutex; use crate::entrylog::EntryLogStore; use pulsar_lite_storage_managed_ledger::{ManagedLedgerConfig, ManagedLedgerFactory}; -pub(crate) type SharedLedger = Arc>; +/// Shared handle for one managed ledger (write-queue worker and store readers). +/// Contents are not wrapped in a mutex; published LAC/meta use interior atomics. +pub(crate) type SharedLedger = Arc; type LedgerCache = HashMap; #[derive(Debug, Clone)] pub struct RocksDBManagedLedgerFactory { db: Arc, entry_log: Arc, + /// Serializes cache get/insert only (one Arc per name). ledgers: Arc>, } @@ -55,9 +58,8 @@ impl RocksDBManagedLedgerFactory { return Ok(Arc::clone(ledger)); } - let ledger = Arc::new(Mutex::new(self.load_ledger(name, config)?)); + let ledger = Arc::new(self.load_ledger(name, config)?); ledgers.insert(name.to_string(), Arc::clone(&ledger)); - Ok(ledger) } @@ -77,25 +79,13 @@ impl RocksDBManagedLedgerFactory { .delete(keys::managed_cursor_key(ledger_name, cursor_name))?; Ok(()) } - - fn open_ledger_with_config( - &self, - name: &str, - config: &ManagedLedgerConfig, - ) -> Result { - RocksDBManagedLedger::open_with_config( - name, - Arc::clone(&self.db), - Arc::clone(&self.entry_log), - config, - ) - } } impl ManagedLedgerFactory for RocksDBManagedLedgerFactory { type Ledger = RocksDBManagedLedger; + /// Returns an uncached instance. Production appends must use `open_ledger` + write queue. fn open(&mut self, name: &str, config: &ManagedLedgerConfig) -> Result { - self.open_ledger_with_config(name, config) + self.load_ledger(name, config) } } diff --git a/rust/storage/managed-ledger-rocksdb/src/ledger.rs b/rust/storage/managed-ledger-rocksdb/src/ledger.rs index 303ab5b..49f2d1f 100644 --- a/rust/storage/managed-ledger-rocksdb/src/ledger.rs +++ b/rust/storage/managed-ledger-rocksdb/src/ledger.rs @@ -2,8 +2,10 @@ use super::cursor::{next_position, RocksDBManagedCursor}; use super::entrylog::{EntryIndex, EntryLogStore, EntryRecord, EntryToAppend}; use super::keys; use super::metadata::{StoredEntryLocation, StoredManagedLedgerInfo}; -use anyhow::{Ok, Result}; +use anyhow::Result; +use arc_swap::ArcSwap; use rocksdb::{WriteBatch, DB}; +use std::sync::atomic::{AtomicU64, Ordering}; use std::sync::Arc; use pulsar_lite_storage_managed_ledger::{ @@ -12,30 +14,42 @@ use pulsar_lite_storage_managed_ledger::{ const DEFAULT_MAX_ENTRIES_PER_LEDGER: u64 = 50_000; -/// Status of the ManagedLedger that currently working on -#[derive(Debug, Clone)] +/// Assignment cursor for the active ledger segment. +/// Only the managed-ledger write-queue worker updates these fields. +#[derive(Debug)] struct ManagedLedgerRuntimeState { - /// The current ledger ID that the ManagedLedger is working on. - current_ledger_id: u64, - - /// The number of entries in the current ledger. - current_ledger_entries: u64, - - /// The size of the current ledger in bytes. - current_ledger_size: u64, + current_ledger_id: AtomicU64, + current_ledger_entries: AtomicU64, + current_ledger_size: AtomicU64, +} - /// The last confirmed position in the entire managed ledger. - last_confirmed_position: Option, +impl ManagedLedgerRuntimeState { + fn from_info(info: &StoredManagedLedgerInfo) -> Self { + let current = info + .ledgers + .last() + .expect("managed ledger info is initialized"); + Self { + current_ledger_id: AtomicU64::new(current.ledger_id), + current_ledger_entries: AtomicU64::new(current.entries), + current_ledger_size: AtomicU64::new(current.size), + } + } } -#[derive(Debug, Clone)] +#[derive(Debug)] pub struct RocksDBManagedLedger { name: String, db: Arc, - pub info: StoredManagedLedgerInfo, - runtime: ManagedLedgerRuntimeState, - max_entries_per_ledger: u64, entry_log: Arc, + max_entries_per_ledger: u64, + + /// Published ledger metadata; readers load this snapshot. + info: ArcSwap, + /// Last durable position readers may observe. + lac: ArcSwap>, + /// Write assignment cursor (single-writer: write-queue worker). + runtime: ManagedLedgerRuntimeState, } impl RocksDBManagedLedger { @@ -70,61 +84,43 @@ impl RocksDBManagedLedger { db.put(&key, info.encode_to_vec())?; - let runtime = Self::runtime_from_info(&info, &db)?; + let lac = Self::lac_from_info_and_db(&info, &db)?; + let runtime = ManagedLedgerRuntimeState::from_info(&info); Ok(Self { name: name.to_string(), db, entry_log, - info, - runtime, max_entries_per_ledger, + info: ArcSwap::from_pointee(info), + lac: ArcSwap::from_pointee(lac), + runtime, }) } - fn runtime_from_info( + fn lac_from_info_and_db( info: &StoredManagedLedgerInfo, db: &DB, - ) -> Result { - let current_ledger = info - .ledgers - .last() - .expect("managed ledger info is initialized"); - - let last_confirmed_position = - match info.ledgers.iter().rev().find(|ledger| ledger.entries > 0) { - Some(last_non_empty_ledger) => { - let entry_id = last_non_empty_ledger.entries - 1; - - let Some(value) = db.get(keys::managed_entry_key( - last_non_empty_ledger.ledger_id, - entry_id, - ))? - else { - return Ok(ManagedLedgerRuntimeState { - current_ledger_id: current_ledger.ledger_id, - current_ledger_entries: current_ledger.entries, - current_ledger_size: current_ledger.size, - last_confirmed_position: None, - }); - }; - - let location: StoredEntryLocation = bincode::deserialize(&value)?; - Some(ManagedLedgerPosition { - ledger_id: last_non_empty_ledger.ledger_id, - entry_id, - partition: location.partition, - }) - } - None => None, - }; - - Ok(ManagedLedgerRuntimeState { - current_ledger_id: current_ledger.ledger_id, - current_ledger_entries: current_ledger.entries, - current_ledger_size: current_ledger.size, - last_confirmed_position, - }) + ) -> Result> { + match info.ledgers.iter().rev().find(|ledger| ledger.entries > 0) { + Some(last_non_empty_ledger) => { + let entry_id = last_non_empty_ledger.entries - 1; + let Some(value) = db.get(keys::managed_entry_key( + last_non_empty_ledger.ledger_id, + entry_id, + ))? + else { + return Ok(None); + }; + let location: StoredEntryLocation = bincode::deserialize(&value)?; + Ok(Some(ManagedLedgerPosition { + ledger_id: last_non_empty_ledger.ledger_id, + entry_id, + partition: location.partition, + })) + } + None => Ok(None), + } } fn allocate_ledger_id(db: &DB) -> Result { @@ -138,52 +134,28 @@ impl RocksDBManagedLedger { Ok(next_ledger_id) } - fn load_entry_index(&self, ledger_id: u64, entry_id: u64) -> Result> { - let Some(value) = self.db.get(keys::managed_entry_key(ledger_id, entry_id))? else { - return Ok(None); - }; - let location: StoredEntryLocation = bincode::deserialize(&value)?; - Ok(Some(EntryIndex { - ledger_id, - entry_id, - file_id: location.file_id, - offset: location.offset, - len: location.len, - checksum: location.checksum, - partition: location.partition, - })) - } - - fn read_entry_record( - &self, - ledger_id: u64, - entry_id: u64, - ) -> Result> { - let Some(index) = self.load_entry_index(ledger_id, entry_id)? else { - return Ok(None); - }; - let position = ManagedLedgerPosition { - ledger_id, - entry_id, - partition: index.partition, - }; - let record = self.entry_log.read(&index)?; - Ok(Some((position, record))) + pub fn info_snapshot(&self) -> arc_swap::Guard> { + self.info.load() } pub fn last_position(&self) -> Result> { - Ok(self.runtime.last_confirmed_position.clone()) + Ok((**self.lac.load()).clone()) } - /// Returns true if the given position is visible in the ledger. - fn is_visible(&self, pos: &ManagedLedgerPosition) -> bool { - self.info - .ledgers + fn is_visible( + &self, + pos: &ManagedLedgerPosition, + info: &StoredManagedLedgerInfo, + lac: &ManagedLedgerPosition, + ) -> bool { + if pos > lac { + return false; + } + info.ledgers .iter() .any(|l| l.ledger_id == pos.ledger_id && l.entries > pos.entry_id) } - /// Reads a batch of entries starting from the given position, up to the specified limit. pub fn read_entries_from( &self, from: &ManagedLedgerPosition, @@ -193,15 +165,23 @@ impl RocksDBManagedLedger { return Ok(Vec::new()); } + let lac_guard = self.lac.load(); + let Some(lac) = lac_guard.as_ref() else { + return Ok(Vec::new()); + }; + if from > lac { + return Ok(Vec::new()); + } + + let info = self.info.load(); let mut out = Vec::with_capacity(limit.min(64)); let mut current = Some(from.clone()); while let Some(pos) = current { - if out.len() >= limit { + if out.len() >= limit || pos > *lac { break; } - - if self.is_visible(&pos) { + if self.is_visible(&pos, info.as_ref(), lac) { if let Some((stored, record)) = self.read_entry_record(pos.ledger_id, pos.entry_id)? { @@ -212,62 +192,75 @@ impl RocksDBManagedLedger { }); } } - current = next_position(&pos, &self.info); + current = next_position(&pos, info.as_ref()); } - Ok(out) } - pub fn add_entry_with_partition( - &mut self, - partition: i32, - payload: &[u8], - ) -> Result { - self.add_entry_with_partition_and_metadata(partition, &[], payload) + fn load_entry_index(&self, ledger_id: u64, entry_id: u64) -> Result> { + let Some(value) = self.db.get(keys::managed_entry_key(ledger_id, entry_id))? else { + return Ok(None); + }; + let location: StoredEntryLocation = bincode::deserialize(&value)?; + Ok(Some(EntryIndex { + ledger_id, + entry_id, + file_id: location.file_id, + offset: location.offset, + len: location.len, + checksum: location.checksum, + partition: location.partition, + })) } - pub fn add_entry_with_partition_and_metadata( - &mut self, - partition: i32, - metadata: &[u8], - payload: &[u8], - ) -> Result { - let mut positions = self.add_entries_with_partition_and_metadata(&[(partition,metadata,payload)])?; - positions.pop().ok_or_else(|| anyhow::anyhow!("add_entries returned empty positions")) + fn read_entry_record( + &self, + ledger_id: u64, + entry_id: u64, + ) -> Result> { + let Some(index) = self.load_entry_index(ledger_id, entry_id)? else { + return Ok(None); + }; + let position = ManagedLedgerPosition { + ledger_id, + entry_id, + partition: index.partition, + }; + let record = self.entry_log.read(&index)?; + Ok(Some((position, record))) } - /// Append many entries with one entrylog flush and one RocksDB WriteBatch. + /// Durable batch append. Intended for the write-queue worker only (single writer). /// - /// - entry_id assignment stays serial (same as single append) - /// - entrylog is written once via `append_batch` (one write_all + flush) - /// - entry locations + final managed-ledger info are written once - /// - runtime state is published only after both durable steps succeed - pub fn add_entries_with_partition_and_metadata( - &mut self, + /// Order: allocate → entrylog + rocks → publish meta → publish LAC. + pub(crate) fn add_entries_with_partition_and_metadata( + &self, items: &[(i32, &[u8], &[u8])], ) -> Result> { if items.is_empty() { return Ok(Vec::new()); } - // Pass 1: allocate positions and next ledger/runtime state in memory. - let mut next_info = self.info.clone(); - let mut next_runtime = self.runtime.clone(); + let mut next_info = (**self.info.load()).clone(); + let mut cur_id = self.runtime.current_ledger_id.load(Ordering::Relaxed); + let mut cur_entries = self.runtime.current_ledger_entries.load(Ordering::Relaxed); + let mut cur_size = self.runtime.current_ledger_size.load(Ordering::Relaxed); + let mut positions = Vec::with_capacity(items.len()); let mut to_append = Vec::with_capacity(items.len()); for (partition, metadata, payload) in items { - if next_runtime.current_ledger_entries >= self.max_entries_per_ledger { + if cur_entries >= self.max_entries_per_ledger { let next_ledger_id = Self::allocate_ledger_id(&self.db)?; next_info.roll_over_current_ledger(next_ledger_id); - next_runtime.current_ledger_id = next_ledger_id; - next_runtime.current_ledger_entries = 0; - next_runtime.current_ledger_size = 0; + cur_id = next_ledger_id; + cur_entries = 0; + cur_size = 0; } let position = ManagedLedgerPosition { - ledger_id: next_runtime.current_ledger_id, - entry_id: next_runtime.current_ledger_entries, + ledger_id: cur_id, + entry_id: cur_entries, partition: *partition, }; @@ -275,16 +268,15 @@ impl RocksDBManagedLedger { let message_size = metadata.len() as u64 + payload.len() as u64; current_ledger.entries += 1; current_ledger.size += message_size; - next_runtime.current_ledger_entries += 1; - next_runtime.current_ledger_size += message_size; - next_runtime.last_confirmed_position = Some(position.clone()); + cur_entries += 1; + cur_size += message_size; - if next_runtime.current_ledger_entries >= self.max_entries_per_ledger { + if cur_entries >= self.max_entries_per_ledger { let next_ledger_id = Self::allocate_ledger_id(&self.db)?; next_info.roll_over_current_ledger(next_ledger_id); - next_runtime.current_ledger_entries = 0; - next_runtime.current_ledger_size = 0; - next_runtime.current_ledger_id = next_ledger_id; + cur_id = next_ledger_id; + cur_entries = 0; + cur_size = 0; } to_append.push(EntryToAppend { @@ -297,7 +289,8 @@ impl RocksDBManagedLedger { positions.push(position); } - // Pass 2: one entrylog IO for the whole batch. + let last_pos = positions.last().cloned(); + let indices = self.entry_log.append_batch(to_append)?; if indices.len() != positions.len() { anyhow::bail!( @@ -307,7 +300,6 @@ impl RocksDBManagedLedger { ); } - // Pass 3: one RocksDB WriteBatch for locations + ledger info. let mut batch = WriteBatch::default(); for (position, entry_index) in positions.iter().zip(indices) { let stored_entry_location = StoredEntryLocation::from(entry_index); @@ -322,21 +314,21 @@ impl RocksDBManagedLedger { ); self.db.write(batch)?; - // Pass 4: publish in-memory state only after durable writes succeed. - self.info = next_info; - self.runtime = next_runtime; + self.info.store(Arc::new(next_info)); + self.runtime + .current_ledger_id + .store(cur_id, Ordering::Relaxed); + self.runtime + .current_ledger_entries + .store(cur_entries, Ordering::Relaxed); + self.runtime + .current_ledger_size + .store(cur_size, Ordering::Relaxed); + self.lac.store(Arc::new(last_pos)); + Ok(positions) } - - #[allow(dead_code)] - pub fn ledger_info(&self) -> &StoredManagedLedgerInfo { - &self.info - } - /// Position immediately before `position` in ledger/entry order. - /// - entry_id > 0 -> same ledger, entry_id - 1 - /// - entry_id == 0 -> last entry of the previous non-empty ledger - /// - no previous -> None ("before first entry", i.e. seek to earliest) pub fn previous_position( &self, position: &ManagedLedgerPosition, @@ -348,8 +340,8 @@ impl RocksDBManagedLedger { partition: position.partition, }); } - let prev = self - .info + let info = self.info.load(); + let prev = info .ledgers .iter() .filter(|l| l.ledger_id < position.ledger_id && l.entries > 0) @@ -361,17 +353,31 @@ impl RocksDBManagedLedger { }) } + pub fn open_cursor(&self, name: &str) -> Result { + RocksDBManagedCursor::open(&self.name, name, Arc::clone(&self.db)) + } + pub fn get_message_by_id(&self, message_id: &MessageId) -> Option<(MessageId, Vec)> { self.get_message_entry_by_id(message_id) .map(|entry| (entry.message_id, entry.payload)) } pub fn get_message_entry_by_id(&self, message_id: &MessageId) -> Option { - let (position, record) = self + let pos = ManagedLedgerPosition::from(message_id); + let lac_guard = self.lac.load(); + let lac = lac_guard.as_ref().as_ref()?; + if &pos > lac { + return None; + } + let info = self.info.load(); + if !self.is_visible(&pos, info.as_ref(), lac) { + return None; + } + let (stored, record) = self .read_entry_record(message_id.ledger, message_id.entry) .ok() .flatten()?; - if position.partition != message_id.partition { + if stored.partition != message_id.partition { return None; } Some(StoredMessage::new( @@ -389,29 +395,21 @@ impl RocksDBManagedLedger { .collect()) } - /// TODO: A temporary global scanning interface. Subsequently, all related global scanning codes will be gradually migrated. pub fn message_entries(&self) -> Result> { - let Some(from) = - self.info - .ledgers - .iter() - .find(|l| l.entries > 0) - .map(|l| ManagedLedgerPosition { - ledger_id: l.ledger_id, - entry_id: 0, - partition: -1, - }) - else { - return Ok(vec![]); - }; - - let total = self - .info + let info = self.info.load(); + let Some(from) = info .ledgers .iter() - .map(|l| l.entries as usize) - .sum::(); - + .find(|l| l.entries > 0) + .map(|l| ManagedLedgerPosition { + ledger_id: l.ledger_id, + entry_id: 0, + partition: -1, + }) + else { + return Ok(Vec::new()); + }; + let total: usize = info.ledgers.iter().map(|l| l.entries as usize).sum(); self.read_entries_from(&from, total) } } @@ -423,8 +421,10 @@ impl ManagedLedger for RocksDBManagedLedger { &self.name } - fn add_entry(&mut self, payload: &[u8]) -> Result { - self.add_entry_with_partition(-1, payload) + fn add_entry(&mut self, _payload: &[u8]) -> Result { + anyhow::bail!( + "RocksDB managed-ledger appends must go through WriteQueue; direct add_entry is disabled" + ) } fn open_cursor(&mut self, name: &str) -> Result { @@ -432,6 +432,15 @@ impl ManagedLedger for RocksDBManagedLedger { } fn read_entry(&self, position: &ManagedLedgerPosition) -> Option> { + let lac_guard = self.lac.load(); + let lac = lac_guard.as_ref().as_ref()?; + if position > lac { + return None; + } + let info = self.info.load(); + if !self.is_visible(position, info.as_ref(), lac) { + return None; + } let (stored, record) = self .read_entry_record(position.ledger_id, position.entry_id) .ok() diff --git a/rust/storage/managed-ledger-rocksdb/src/lib.rs b/rust/storage/managed-ledger-rocksdb/src/lib.rs index d390526..e00575f 100644 --- a/rust/storage/managed-ledger-rocksdb/src/lib.rs +++ b/rust/storage/managed-ledger-rocksdb/src/lib.rs @@ -28,4 +28,28 @@ pub mod test_support { managed_ledger_name, }; } + + use anyhow::Result; + use pulsar_lite_storage_managed_ledger::ManagedLedgerPosition; + + /// Test-only durable append (crate-private API). Production must use WriteQueue. + pub fn append_payload( + ledger: &RocksDBManagedLedger, + payload: &[u8], + ) -> Result { + append_with_partition(ledger, -1, payload) + } + + /// Test-only durable append with partition. + pub fn append_with_partition( + ledger: &RocksDBManagedLedger, + partition: i32, + payload: &[u8], + ) -> Result { + let mut positions = + ledger.add_entries_with_partition_and_metadata(&[(partition, &[] as &[u8], payload)])?; + positions + .pop() + .ok_or_else(|| anyhow::anyhow!("add_entries returned empty positions")) + } } diff --git a/rust/storage/managed-ledger-rocksdb/src/store.rs b/rust/storage/managed-ledger-rocksdb/src/store.rs index 1bdb73a..7c2574e 100644 --- a/rust/storage/managed-ledger-rocksdb/src/store.rs +++ b/rust/storage/managed-ledger-rocksdb/src/store.rs @@ -2,18 +2,17 @@ use super::cursor::{ack_managed_cursor_shared, is_managed_position_acknowledged, use super::entrylog::EntryLogStore; use super::factory::{RocksDBManagedLedgerFactory, SharedLedger}; use super::keys; -use super::ledger::RocksDBManagedLedger; use super::write_queue::WriteQueue; use crate::cursor::first_position; use anyhow::{anyhow, Result}; use rocksdb::{Options, DB}; use std::fmt; use std::path::Path; -use std::sync::{Arc, MutexGuard}; +use std::sync::{Arc, Mutex}; use pulsar_lite_storage_managed_ledger::{ - CursorInitOptions, CursorOpenResult, InitialPosition, ManagedCursor, ManagedLedger, - ManagedLedgerPosition, ManagedLedgerStorage, MessageId, StoredMessage, + CursorInitOptions, CursorOpenResult, InitialPosition, ManagedCursor, ManagedLedgerPosition, + ManagedLedgerStorage, MessageId, StoredMessage, }; /// Cloned handle that can append without holding `Mutex`. @@ -58,8 +57,8 @@ impl ConcurrentAppender { .map_err(|e| anyhow!(e)) } - /// Enqueue without waiting (BookKeeper logAddEntry style for connections). - /// Completion is delivered on `completion_tx` from the write-queue worker thread. + /// Enqueue without waiting. Completion is delivered on `completion_tx` + /// from the write-queue worker thread. pub fn enqueue_for_connection( &self, topic: &str, @@ -94,6 +93,8 @@ impl fmt::Debug for ConcurrentAppender { pub struct RocksDbManagedLedgerStorage { factory: RocksDBManagedLedgerFactory, write_queue: WriteQueue, + /// Serializes cursor RMW (ack/seek/init). Must not cover entrylog IO. + cursor_mu: Mutex<()>, } impl fmt::Debug for RocksDbManagedLedgerStorage { @@ -116,6 +117,7 @@ impl RocksDbManagedLedgerStorage { Ok(Self { write_queue: WriteQueue::new(factory.clone()), factory, + cursor_mu: Mutex::new(()), }) } @@ -131,10 +133,12 @@ impl RocksDbManagedLedgerStorage { self.factory.open_ledger(&ledger_name) } - fn lock_ledger(ledger: &SharedLedger) -> Result> { - ledger + fn with_cursor_lock(&self, f: impl FnOnce() -> Result) -> Result { + let _guard = self + .cursor_mu .lock() - .map_err(|_| anyhow!("managed ledger lock poisoned")) + .map_err(|_| anyhow!("cursor lock poisoned"))?; + f() } fn cursor_exists(&self, topic: &str, subscription: &str) -> Result { @@ -144,25 +148,27 @@ impl RocksDbManagedLedgerStorage { } fn persist_empty_cursor(&self, topic: &str, subscription: &str) -> Result<()> { - let cursor_name = keys::encode_cursor_name(subscription); - let shared = self.topic_ledger(topic)?; - let mut ledger = Self::lock_ledger(&shared)?; - let cursor = ledger.open_cursor(&cursor_name)?; - cursor.persist_state() + self.with_cursor_lock(|| { + let cursor_name = keys::encode_cursor_name(subscription); + let ledger = self.topic_ledger(topic)?; + let mut cursor = ledger.open_cursor(&cursor_name)?; + cursor.persist_state() + }) } fn apply_latest_cursor(&self, topic: &str, subscription: &str) -> Result<()> { - let cursor_name = keys::encode_cursor_name(subscription); - let shared = self.topic_ledger(topic)?; - let mut ledger = Self::lock_ledger(&shared)?; - let last = ledger.last_position()?; - let mut cursor = ledger.open_cursor(&cursor_name)?; - - if let Some(last) = last { - cursor.mark_delete(last) - } else { - cursor.persist_state() - } + self.with_cursor_lock(|| { + let cursor_name = keys::encode_cursor_name(subscription); + let ledger = self.topic_ledger(topic)?; + let last = ledger.last_position()?; + let mut cursor = ledger.open_cursor(&cursor_name)?; + + if let Some(last) = last { + cursor.mark_delete(last) + } else { + cursor.persist_state() + } + }) } fn apply_start_message_id_cursor( @@ -171,19 +177,19 @@ impl RocksDbManagedLedgerStorage { subscription: &str, start: &MessageId, ) -> Result<()> { - let cursor_name = keys::encode_cursor_name(subscription); - let target = ManagedLedgerPosition::from(start); - let shared = self.topic_ledger(topic)?; - let mut ledger = Self::lock_ledger(&shared)?; - - let previous = ledger.previous_position(&target); - let mut cursor = ledger.open_cursor(&cursor_name)?; - - if let Some(previous) = previous { - cursor.mark_delete(previous) - } else { - cursor.persist_state() - } + self.with_cursor_lock(|| { + let cursor_name = keys::encode_cursor_name(subscription); + let target = ManagedLedgerPosition::from(start); + let ledger = self.topic_ledger(topic)?; + let previous = ledger.previous_position(&target); + let mut cursor = ledger.open_cursor(&cursor_name)?; + + if let Some(previous) = previous { + cursor.mark_delete(previous) + } else { + cursor.persist_state() + } + }) } } @@ -252,18 +258,14 @@ impl ManagedLedgerStorage for RocksDbManagedLedgerStorage { message_id: &MessageId, _shared: bool, ) -> Result<()> { - let cursor_name = keys::encode_cursor_name(subscription); - let position = ManagedLedgerPosition::from(message_id); - - let (mut cursor, marker_delete_posistion) = { - let shared = self.topic_ledger(topic)?; - let mut ledger = Self::lock_ledger(&shared)?; - + self.with_cursor_lock(|| { + let cursor_name = keys::encode_cursor_name(subscription); + let position = ManagedLedgerPosition::from(message_id); + let ledger = self.topic_ledger(topic)?; let mark_delete_position = ledger.previous_position(&position); - let cursor = ledger.open_cursor(&cursor_name)?; - (cursor, mark_delete_position) - }; - cursor.reset_cursor(marker_delete_posistion) + let mut cursor = ledger.open_cursor(&cursor_name)?; + cursor.reset_cursor(mark_delete_position) + }) } fn first_unacked_position( @@ -271,25 +273,33 @@ impl ManagedLedgerStorage for RocksDbManagedLedgerStorage { topic: &str, subscription: &str, ) -> Result> { - let cursor_name = keys::encode_cursor_name(subscription); - let shared = self.topic_ledger(topic)?; - let mut ledger = Self::lock_ledger(&shared)?; + self.with_cursor_lock(|| { + let cursor_name = keys::encode_cursor_name(subscription); + let ledger = self.topic_ledger(topic)?; + let Some(lac) = ledger.last_position()? else { + return Ok(None); + }; - let cursor = ledger.open_cursor(&cursor_name)?; - let state = cursor.state(); - - let mut candidate = match state.mark_delete.as_ref() { - Some(mark_delete) => next_position(mark_delete, &ledger.info), - None => first_position(&ledger.info, -1), - }; - - while let Some(position) = candidate { - if !is_managed_position_acknowledged(state, &position) { - return Ok(Some(position)); + let cursor = ledger.open_cursor(&cursor_name)?; + let state = cursor.state(); + let info = ledger.info_snapshot(); + + let mut candidate = match state.mark_delete.as_ref() { + Some(mark_delete) => next_position(mark_delete, info.as_ref()), + None => first_position(info.as_ref(), -1), + }; + + while let Some(position) = candidate { + if position > lac { + return Ok(None); + } + if !is_managed_position_acknowledged(state, &position) { + return Ok(Some(position)); + } + candidate = next_position(&position, info.as_ref()); } - candidate = next_position(&position, &ledger.info); - } - Ok(None) + Ok(None) + }) } fn read_from( @@ -298,9 +308,7 @@ impl ManagedLedgerStorage for RocksDbManagedLedgerStorage { from: &ManagedLedgerPosition, limit: usize, ) -> Result)>> { - let shared = self.topic_ledger(topic)?; - let ledger = Self::lock_ledger(&shared)?; - + let ledger = self.topic_ledger(topic)?; Ok(ledger .read_entries_from(from, limit)? .into_iter() @@ -314,15 +322,11 @@ impl ManagedLedgerStorage for RocksDbManagedLedgerStorage { from: &ManagedLedgerPosition, limit: usize, ) -> Result> { - let shared = self.topic_ledger(topic)?; - let ledger = Self::lock_ledger(&shared)?; - ledger.read_entries_from(from, limit) + self.topic_ledger(topic)?.read_entries_from(from, limit) } fn get_last_position(&self, topic: &str) -> Result> { - let shared = self.topic_ledger(topic)?; - let ledger = Self::lock_ledger(&shared)?; - ledger.last_position() + self.topic_ledger(topic)?.last_position() } fn get_next_position( @@ -330,9 +334,9 @@ impl ManagedLedgerStorage for RocksDbManagedLedgerStorage { topic: &str, current: &ManagedLedgerPosition, ) -> Result> { - let shared = self.topic_ledger(topic)?; - let ledger = Self::lock_ledger(&shared)?; - Ok(next_position(current, &ledger.info)) + let ledger = self.topic_ledger(topic)?; + let info = ledger.info_snapshot(); + Ok(next_position(current, info.as_ref())) } fn is_acknowledged( @@ -350,11 +354,12 @@ impl ManagedLedgerStorage for RocksDbManagedLedgerStorage { subscription: &str, message_id: MessageId, ) -> Result<()> { - let cursor_name = keys::encode_cursor_name(subscription); - let shared = self.topic_ledger(topic)?; - let mut ledger = Self::lock_ledger(&shared)?; - let mut cursor = ledger.open_cursor(&cursor_name)?; - cursor.mark_delete(ManagedLedgerPosition::from(message_id)) + self.with_cursor_lock(|| { + let cursor_name = keys::encode_cursor_name(subscription); + let ledger = self.topic_ledger(topic)?; + let mut cursor = ledger.open_cursor(&cursor_name)?; + cursor.mark_delete(ManagedLedgerPosition::from(message_id)) + }) } fn ack_message_shared( @@ -363,15 +368,17 @@ impl ManagedLedgerStorage for RocksDbManagedLedgerStorage { subscription: &str, message_id: MessageId, ) -> Result<()> { - let cursor_name = keys::encode_cursor_name(subscription); - let shared = self.topic_ledger(topic)?; - let mut ledger = Self::lock_ledger(&shared)?; - let mut cursor = ledger.open_cursor(&cursor_name)?; - ack_managed_cursor_shared( - &mut cursor, - ManagedLedgerPosition::from(message_id), - &ledger.info, - ) + self.with_cursor_lock(|| { + let cursor_name = keys::encode_cursor_name(subscription); + let ledger = self.topic_ledger(topic)?; + let mut cursor = ledger.open_cursor(&cursor_name)?; + let info = ledger.info_snapshot(); + ack_managed_cursor_shared( + &mut cursor, + ManagedLedgerPosition::from(message_id), + info.as_ref(), + ) + }) } fn get_message_by_id( @@ -379,9 +386,7 @@ impl ManagedLedgerStorage for RocksDbManagedLedgerStorage { topic: &str, message_id: &MessageId, ) -> Option<(MessageId, Vec)> { - let shared = self.topic_ledger(topic).ok()?; - let ledger = Self::lock_ledger(&shared).ok()?; - ledger.get_message_by_id(message_id) + self.topic_ledger(topic).ok()?.get_message_by_id(message_id) } fn get_message_entry_by_id( @@ -389,35 +394,13 @@ impl ManagedLedgerStorage for RocksDbManagedLedgerStorage { topic: &str, message_id: &MessageId, ) -> Option { - let shared = self.topic_ledger(topic).ok()?; - let ledger = Self::lock_ledger(&shared).ok()?; - ledger.get_message_entry_by_id(message_id) + self.topic_ledger(topic) + .ok()? + .get_message_entry_by_id(message_id) } fn get_messages(&self, topic: &str) -> Vec<(MessageId, Vec)> { - let shared = match self.topic_ledger(topic) { - Ok(shared) => shared, - Err(error) => { - log::error!( - "Failed to open managed ledger for topic '{}': {}", - topic, - error - ); - return Vec::new(); - } - }; - let ledger = match Self::lock_ledger(&shared) { - Ok(ledger) => ledger, - Err(error) => { - log::error!( - "Failed to lock managed ledger for topic '{}': {}", - topic, - error - ); - return Vec::new(); - } - }; - match ledger.messages() { + match self.topic_ledger(topic).and_then(|l| l.messages()) { Ok(messages) => messages, Err(error) => { log::error!( @@ -431,29 +414,7 @@ impl ManagedLedgerStorage for RocksDbManagedLedgerStorage { } fn get_message_entries(&self, topic: &str) -> Vec { - let shared = match self.topic_ledger(topic) { - Ok(shared) => shared, - Err(error) => { - log::error!( - "Failed to open managed ledger for topic '{}': {}", - topic, - error - ); - return Vec::new(); - } - }; - let ledger = match Self::lock_ledger(&shared) { - Ok(ledger) => ledger, - Err(error) => { - log::error!( - "Failed to lock managed ledger for topic '{}': {}", - topic, - error - ); - return Vec::new(); - } - }; - match ledger.message_entries() { + match self.topic_ledger(topic).and_then(|l| l.message_entries()) { Ok(entries) => entries, Err(error) => { log::error!( @@ -472,13 +433,11 @@ impl ManagedLedgerStorage for RocksDbManagedLedgerStorage { subscription: &str, message_id: &MessageId, ) -> bool { - let shared = match self.topic_ledger(topic) { - Ok(shared) => shared, - Err(_) => return false, + let Ok(_guard) = self.cursor_mu.lock() else { + return false; }; - let mut ledger = match Self::lock_ledger(&shared) { - Ok(ledger) => ledger, - Err(_) => return false, + let Ok(ledger) = self.topic_ledger(topic) else { + return false; }; let cursor_name = keys::encode_cursor_name(subscription); ledger @@ -493,13 +452,11 @@ impl ManagedLedgerStorage for RocksDbManagedLedgerStorage { } fn get_mark_delete_position(&self, topic: &str, subscription: &str) -> Option { - let cursor_name = keys::encode_cursor_name(subscription); - let shared = self.topic_ledger(topic).ok()?; - let mut ledger = match Self::lock_ledger(&shared) { - Ok(ledger) => ledger, - Err(_) => return None, - }; - let cursor = ledger.open_cursor(&cursor_name).ok()?; + let _guard = self.cursor_mu.lock().ok()?; + let ledger = self.topic_ledger(topic).ok()?; + let cursor = ledger + .open_cursor(&keys::encode_cursor_name(subscription)) + .ok()?; cursor .state() .mark_delete diff --git a/rust/storage/managed-ledger-rocksdb/src/write_queue.rs b/rust/storage/managed-ledger-rocksdb/src/write_queue.rs index bbdb04c..cf7da5e 100644 --- a/rust/storage/managed-ledger-rocksdb/src/write_queue.rs +++ b/rust/storage/managed-ledger-rocksdb/src/write_queue.rs @@ -1,4 +1,4 @@ -use crate::factory::RocksDBManagedLedgerFactory; +use crate::factory::{RocksDBManagedLedgerFactory, SharedLedger}; use crate::keys; use pulsar_lite_storage_managed_ledger::MessageId; use std::collections::HashMap; @@ -64,9 +64,9 @@ pub(crate) struct WriteReq { /// Single-writer queue for managed-ledger appends. /// -/// Worker owns ledgers in a thread-local HashMap and writes via `&mut` -/// (no SharedLedger lock on the append path). Reply uses tokio oneshot so -/// async callers can await without blocking the Tokio worker thread. +/// Worker caches the same `Arc` handles as store reads so +/// durable success updates one published LAC/meta view. Entry-id assignment stays +/// serial on this thread; no outer content mutex around append IO. pub(crate) struct WriteQueue { tx: Option>, worker: Option>, @@ -90,6 +90,9 @@ impl WriteQueue { } fn worker_loop(factory: RocksDBManagedLedgerFactory, rx: mpsc::Receiver) { + // Same Arc instances as store reads (singleton per ledger name). + let mut ledgers: HashMap = HashMap::new(); + // Batch-size metrics (aggregated to avoid log spam under stress). let mut metric_batches: u64 = 0; let mut metric_msgs: u64 = 0; @@ -178,26 +181,27 @@ impl WriteQueue { metric_group_eq1 += 1; } - // Route the batch through the factory's SHARED ledger cache: - // the worker-owned ledger keeps its runtime state (LAC, - // entry counters) private in memory, so appends committed - // through it were invisible to readers using the shared - // ledger until the LAC-ArcSwap rework. Locking the shared - // ledger unifies the in-memory state; batching (one entrylog - // flush + one RocksDB write per group) is preserved. - let shared = match factory.open_ledger(&ledger_name) { - Ok(shared) => shared, - Err(e) => { - let msg = e.to_string(); - for req in reqs { - req.reply.complete(Err(msg.clone())); + if !ledgers.contains_key(&ledger_name) { + match factory.open_ledger(&ledger_name) { + Ok(ledger) => { + ledgers.insert(ledger_name.clone(), ledger); + } + Err(e) => { + let msg = e.to_string(); + for req in reqs { + req.reply.complete(Err(msg.clone())); + } + continue; } - continue; } - }; - let mut ledger = match shared.lock() { - Ok(guard) => guard, - Err(poisoned) => poisoned.into_inner(), + } + + let Some(ledger) = ledgers.get(&ledger_name) else { + for req in reqs { + req.reply + .complete(Err("ledger missing from worker cache".to_string())); + } + continue; }; // Borrow metadata/payload only while reqs is still alive. @@ -206,6 +210,8 @@ impl WriteQueue { .map(|r| (r.partition, r.metadata.as_slice(), r.payload.as_slice())) .collect(); + // Append publishes meta then LAC only after durable OK; + // complete only after that returns Ok. match ledger.add_entries_with_partition_and_metadata(&inputs) { Ok(positions) => { for (req, position) in reqs.into_iter().zip(positions) { diff --git a/rust/storage/managed-ledger-rocksdb/tests/rocksdb_cursor.rs b/rust/storage/managed-ledger-rocksdb/tests/rocksdb_cursor.rs index ab9c912..dcd69d1 100644 --- a/rust/storage/managed-ledger-rocksdb/tests/rocksdb_cursor.rs +++ b/rust/storage/managed-ledger-rocksdb/tests/rocksdb_cursor.rs @@ -3,13 +3,13 @@ mod common; use common::*; -use pulsar_lite_storage_managed_ledger::{ManagedCursor, ManagedLedger}; +use pulsar_lite_storage_managed_ledger::ManagedCursor; use pulsar_lite_storage_managed_ledger::{ ManagedLedgerConfig, ManagedLedgerFactory, ManagedLedgerPosition, }; use pulsar_lite_storage_managed_ledger_rocksdb::test_support::{ ack_managed_cursor_shared, is_managed_position_acknowledged, RocksDBManagedCursor, - RocksDBManagedLedger, RocksDBManagedLedgerFactory, + append_payload, append_with_partition, RocksDBManagedLedger, RocksDBManagedLedgerFactory, }; use std::sync::Arc; use tempfile::tempdir; @@ -91,18 +91,18 @@ fn shared_ack_advances_contiguously_across_rolled_ledgers() { }; let entry_log = open_test_entry_log(&db_path); let mut factory = RocksDBManagedLedgerFactory::new(Arc::clone(&db), entry_log); - let mut ledger = factory.open("ledger-a", &config).unwrap(); - let first = ledger.add_entry(b"first").unwrap(); - let second = ledger.add_entry(b"second").unwrap(); - let third = ledger.add_entry(b"third").unwrap(); + let ledger = factory.open("ledger-a", &config).unwrap(); + let first = append_payload(&ledger, b"first").unwrap(); + let second = append_payload(&ledger, b"second").unwrap(); + let third = append_payload(&ledger, b"third").unwrap(); let mut cursor = ledger.open_cursor("sub-a").unwrap(); - ack_managed_cursor_shared(&mut cursor, third.clone(), &ledger.ledger_info()).unwrap(); + ack_managed_cursor_shared(&mut cursor, third.clone(), &ledger.info_snapshot().as_ref()).unwrap(); assert_eq!(cursor.state().mark_delete, None); assert!(cursor.state().individually_deleted_entries.contains(&third)); - ack_managed_cursor_shared(&mut cursor, first, &ledger.ledger_info()).unwrap(); - ack_managed_cursor_shared(&mut cursor, second, &ledger.ledger_info()).unwrap(); + ack_managed_cursor_shared(&mut cursor, first, &ledger.info_snapshot().as_ref()).unwrap(); + ack_managed_cursor_shared(&mut cursor, second, &ledger.info_snapshot().as_ref()).unwrap(); assert_eq!(cursor.state().mark_delete, Some(third)); assert!(cursor.state().individually_deleted_entries.is_empty()); @@ -116,14 +116,14 @@ fn managed_ledger_open_cursor_recovers_cursor_state() { { let db = open_test_db(&db_path); let entry_log = open_test_entry_log(&db_path); - let mut ledger = RocksDBManagedLedger::open("ledger-a", db, entry_log).unwrap(); + let ledger = RocksDBManagedLedger::open("ledger-a", db, entry_log).unwrap(); let mut cursor = ledger.open_cursor("sub-a").unwrap(); cursor.mark_delete(mark_delete.clone()).unwrap(); } let db = open_test_db(&db_path); let entry_log = open_test_entry_log(&db_path); - let mut ledger = RocksDBManagedLedger::open("ledger-a", db, entry_log).unwrap(); + let ledger = RocksDBManagedLedger::open("ledger-a", db, entry_log).unwrap(); let cursor = ledger.open_cursor("sub-a").unwrap(); assert_eq!(cursor.state().mark_delete, Some(mark_delete)); @@ -259,16 +259,16 @@ fn shared_ack_normalizes_partition_when_advancing_mark_delete() { let db = open_test_db(&db_path); let entry_log = open_test_entry_log(&db_path); - let mut ledger = RocksDBManagedLedger::open("ledger-a", Arc::clone(&db), entry_log).unwrap(); + let ledger = RocksDBManagedLedger::open("ledger-a", Arc::clone(&db), entry_log).unwrap(); - let first = ledger.add_entry_with_partition(7, b"first").unwrap(); + let first = append_with_partition(&ledger, 7, b"first").unwrap(); - let second = ledger.add_entry_with_partition(7, b"second").unwrap(); + let second = append_with_partition(&ledger, 7, b"second").unwrap(); let mut cursor = ledger.open_cursor("sub-a").unwrap(); // First, reorder ACK item 'second'. It should enter the individual collection - ack_managed_cursor_shared(&mut cursor, second.clone(), ledger.ledger_info()).unwrap(); + ack_managed_cursor_shared(&mut cursor, second.clone(), ledger.info_snapshot().as_ref()).unwrap(); assert_eq!(cursor.state().mark_delete, None); assert!(cursor @@ -281,7 +281,7 @@ fn shared_ack_normalizes_partition_when_advancing_mark_delete() { })); // Then move on to the first item, 'mark_delete'proceed forward - ack_managed_cursor_shared(&mut cursor, first.clone(), ledger.ledger_info()).unwrap(); + ack_managed_cursor_shared(&mut cursor, first.clone(), ledger.info_snapshot().as_ref()).unwrap(); assert_eq!( cursor.state().mark_delete, Some(ManagedLedgerPosition { diff --git a/rust/storage/managed-ledger-rocksdb/tests/rocksdb_keys.rs b/rust/storage/managed-ledger-rocksdb/tests/rocksdb_keys.rs index 5839d12..e627170 100644 --- a/rust/storage/managed-ledger-rocksdb/tests/rocksdb_keys.rs +++ b/rust/storage/managed-ledger-rocksdb/tests/rocksdb_keys.rs @@ -4,9 +4,9 @@ mod common; use common::*; use prost::Message; -use pulsar_lite_storage_managed_ledger::{ManagedCursor, ManagedLedger}; +use pulsar_lite_storage_managed_ledger::ManagedCursor; use pulsar_lite_storage_managed_ledger_rocksdb::test_support::{ - keys, proto, RocksDBManagedCursor, RocksDBManagedLedger, + append_payload, keys, proto, RocksDBManagedCursor, RocksDBManagedLedger, }; use std::sync::Arc; use tempfile::tempdir; @@ -36,9 +36,9 @@ fn managed_ledger_info_value_is_protobuf_encoded() { { let entry_log = open_test_entry_log(&db_path); - let mut ledger = + let ledger = RocksDBManagedLedger::open("ledger-a", Arc::clone(&db), entry_log).unwrap(); - ledger.add_entry(b"first").unwrap(); + append_payload(&ledger, b"first").unwrap(); } let bytes = read_raw_value(&db, keys::managed_ledger_key("ledger-a")); diff --git a/rust/storage/managed-ledger-rocksdb/tests/rocksdb_ledger.rs b/rust/storage/managed-ledger-rocksdb/tests/rocksdb_ledger.rs index 6835248..2ea93b2 100644 --- a/rust/storage/managed-ledger-rocksdb/tests/rocksdb_ledger.rs +++ b/rust/storage/managed-ledger-rocksdb/tests/rocksdb_ledger.rs @@ -3,12 +3,10 @@ mod common; use common::*; -use pulsar_lite_storage_managed_ledger::{ - ManagedLedger, ManagedLedgerConfig, ManagedLedgerFactory, -}; +use pulsar_lite_storage_managed_ledger::{ManagedLedger, ManagedLedgerConfig, ManagedLedgerFactory}; use pulsar_lite_storage_managed_ledger_rocksdb::test_support::{ - keys, RocksDBManagedLedger, RocksDBManagedLedgerFactory, StoredEntryLocation, - StoredManagedLedgerInfo, + append_payload, append_with_partition, keys, RocksDBManagedLedger, + RocksDBManagedLedgerFactory, StoredEntryLocation, StoredManagedLedgerInfo, }; use std::sync::Arc; use tempfile::tempdir; @@ -21,8 +19,8 @@ fn managed_ledger_entry_recovers_after_reopen() { let first_position = { let db = open_test_db(&db_path); let entry_log = open_test_entry_log(&db_path); - let mut ledger = RocksDBManagedLedger::open("ledger-a", db, entry_log).unwrap(); - ledger.add_entry(b"first").unwrap() + let ledger = RocksDBManagedLedger::open("ledger-a", db, entry_log).unwrap(); + append_payload(&ledger, b"first").unwrap() }; let db = open_test_db(&db_path); @@ -72,8 +70,8 @@ fn managed_ledger_entry_value_stores_location_not_payload() { let entry_log = open_test_entry_log(&db_path); let payload = b"payload-in-entrylog"; - let mut ledger = RocksDBManagedLedger::open("ledger-a", Arc::clone(&db), entry_log).unwrap(); - let position = ledger.add_entry(payload).unwrap(); + let ledger = RocksDBManagedLedger::open("ledger-a", Arc::clone(&db), entry_log).unwrap(); + let position = append_payload(&ledger, payload).unwrap(); let raw_value = read_raw_value( &db, @@ -97,9 +95,8 @@ fn managed_ledger_returns_none_for_bad_entry_location() { let entry_log = open_test_entry_log(&db_path); let position = { - let mut ledger = - RocksDBManagedLedger::open("ledger-a", Arc::clone(&db), entry_log).unwrap(); - ledger.add_entry(b"payload").unwrap() + let ledger = RocksDBManagedLedger::open("ledger-a", Arc::clone(&db), entry_log).unwrap(); + append_payload(&ledger, b"payload").unwrap() }; let mut location: StoredEntryLocation = bincode::deserialize(&read_raw_value( @@ -129,15 +126,15 @@ fn managed_ledger_next_entry_id_is_derived_from_last_ledger_entries() { { let db = open_test_db(&db_path); let entry_log = open_test_entry_log(&db_path); - let mut ledger = RocksDBManagedLedger::open("ledger-a", db, entry_log).unwrap(); - assert_eq!(ledger.add_entry(b"first").unwrap().entry_id, 0); - assert_eq!(ledger.add_entry(b"second").unwrap().entry_id, 1); + let ledger = RocksDBManagedLedger::open("ledger-a", db, entry_log).unwrap(); + assert_eq!(append_payload(&ledger, b"first").unwrap().entry_id, 0); + assert_eq!(append_payload(&ledger, b"second").unwrap().entry_id, 1); } let db = open_test_db(&db_path); let entry_log = open_test_entry_log(&db_path); - let mut ledger = RocksDBManagedLedger::open("ledger-a", db, entry_log).unwrap(); - let third_position = ledger.add_entry(b"third").unwrap(); + let ledger = RocksDBManagedLedger::open("ledger-a", db, entry_log).unwrap(); + let third_position = append_payload(&ledger, b"third").unwrap(); assert_eq!(third_position.ledger_id, 0); assert_eq!(third_position.entry_id, 2); @@ -160,19 +157,19 @@ fn managed_ledger_rolls_over_after_max_entries_like_pulsar() { let mut factory = RocksDBManagedLedgerFactory::new(Arc::clone(&db), entry_log); { - let mut ledger = factory.open("ledger-a", &config).unwrap(); - assert_eq!(ledger.add_entry(b"first").unwrap(), position(0, 0)); - assert_eq!(ledger.add_entry(b"second").unwrap(), position(0, 1)); - assert_eq!(ledger.add_entry(b"third").unwrap(), position(1, 0)); + let ledger = factory.open("ledger-a", &config).unwrap(); + assert_eq!(append_payload(&ledger, b"first").unwrap(), position(0, 0)); + assert_eq!(append_payload(&ledger, b"second").unwrap(), position(0, 1)); + assert_eq!(append_payload(&ledger, b"third").unwrap(), position(1, 0)); } let ledger = factory.open("ledger-a", &config).unwrap(); - assert_eq!(ledger.ledger_info().ledgers.len(), 2); - assert_eq!(ledger.ledger_info().ledgers[0].ledger_id, 0); - assert_eq!(ledger.ledger_info().ledgers[0].entries, 2); - assert_eq!(ledger.ledger_info().ledgers[1].ledger_id, 1); - assert_eq!(ledger.ledger_info().ledgers[1].entries, 1); + assert_eq!(ledger.info_snapshot().ledgers.len(), 2); + assert_eq!(ledger.info_snapshot().ledgers[0].ledger_id, 0); + assert_eq!(ledger.info_snapshot().ledgers[0].entries, 2); + assert_eq!(ledger.info_snapshot().ledgers[1].ledger_id, 1); + assert_eq!(ledger.info_snapshot().ledgers[1].entries, 1); assert_eq!( ledger.read_entry(&position(0, 0)).as_deref(), Some(b"first".as_slice()) @@ -200,10 +197,10 @@ fn managed_ledger_rollover_metadata_is_persisted_in_rocksdb() { let mut factory = RocksDBManagedLedgerFactory::new(Arc::clone(&db), entry_log); { - let mut ledger = factory.open("ledger-a", &config).unwrap(); - ledger.add_entry(b"first").unwrap(); - ledger.add_entry(b"second").unwrap(); - ledger.add_entry(b"third").unwrap(); + let ledger = factory.open("ledger-a", &config).unwrap(); + append_payload(&ledger, b"first").unwrap(); + append_payload(&ledger, b"second").unwrap(); + append_payload(&ledger, b"third").unwrap(); } let info = read_managed_ledger_info(&db, "ledger-a"); @@ -235,19 +232,19 @@ fn managed_ledger_reopen_continues_from_persisted_rollover_metadata() { let db = open_test_db(&db_path); let entry_log = open_test_entry_log(&db_path); let mut factory = RocksDBManagedLedgerFactory::new(db, entry_log); - let mut ledger = factory.open("ledger-a", &config).unwrap(); - assert_eq!(ledger.add_entry(b"first").unwrap(), position(0, 0)); - assert_eq!(ledger.add_entry(b"second").unwrap(), position(0, 1)); + let ledger = factory.open("ledger-a", &config).unwrap(); + assert_eq!(append_payload(&ledger, b"first").unwrap(), position(0, 0)); + assert_eq!(append_payload(&ledger, b"second").unwrap(), position(0, 1)); } { let db = open_test_db(&db_path); let entry_log = open_test_entry_log(&db_path); let mut factory = RocksDBManagedLedgerFactory::new(Arc::clone(&db), entry_log); - let mut ledger = factory.open("ledger-a", &config).unwrap(); - assert_eq!(ledger.add_entry(b"third").unwrap(), position(1, 0)); - assert_eq!(ledger.add_entry(b"fourth").unwrap(), position(1, 1)); - assert_eq!(ledger.add_entry(b"fifth").unwrap(), position(2, 0)); + let ledger = factory.open("ledger-a", &config).unwrap(); + assert_eq!(append_payload(&ledger, b"third").unwrap(), position(1, 0)); + assert_eq!(append_payload(&ledger, b"fourth").unwrap(), position(1, 1)); + assert_eq!(append_payload(&ledger, b"fifth").unwrap(), position(2, 0)); let info = read_managed_ledger_info(&db, "ledger-a"); assert_eq!(info.ledgers.len(), 3); @@ -281,21 +278,21 @@ fn managed_ledger_ids_are_global_across_topics() { let db = open_test_db(&db_path); let entry_log = open_test_entry_log(&db_path); - let mut orders = RocksDBManagedLedger::open( + let orders = RocksDBManagedLedger::open( "public/default/persistent/orders", Arc::clone(&db), Arc::clone(&entry_log), ) .unwrap(); - let mut payments = RocksDBManagedLedger::open( + let payments = RocksDBManagedLedger::open( "public/default/persistent/payments", Arc::clone(&db), Arc::clone(&entry_log), ) .unwrap(); - let orders_position = orders.add_entry(b"order-1").unwrap(); - let payments_position = payments.add_entry(b"payment-1").unwrap(); + let orders_position = append_payload(&orders, b"order-1").unwrap(); + let payments_position = append_payload(&payments, b"payment-1").unwrap(); assert_ne!(orders_position.ledger_id, payments_position.ledger_id); assert_eq!(orders_position.entry_id, 0); @@ -335,12 +332,12 @@ fn rolled_ledgers_allocate_global_ledger_ids() { let entry_log = open_test_entry_log(&db_path); let mut factory = RocksDBManagedLedgerFactory::new(Arc::clone(&db), entry_log); - let mut orders = factory.open("orders", &config).unwrap(); - let mut payments = factory.open("payments", &config).unwrap(); + let orders = factory.open("orders", &config).unwrap(); + let payments = factory.open("payments", &config).unwrap(); - let orders_first = orders.add_entry(b"order-1").unwrap(); - let payments_first = payments.add_entry(b"payment-1").unwrap(); - let orders_second = orders.add_entry(b"order-2").unwrap(); + let orders_first = append_payload(&orders, b"order-1").unwrap(); + let payments_first = append_payload(&payments, b"payment-1").unwrap(); + let orders_second = append_payload(&orders, b"order-2").unwrap(); assert_ne!(orders_first.ledger_id, payments_first.ledger_id); assert_ne!(orders_second.ledger_id, orders_first.ledger_id); @@ -366,10 +363,10 @@ fn previous_position_handles_same_ledger_cross_ledger_and_before_first() { let entry_log = open_test_entry_log(&db_path); let mut factory = RocksDBManagedLedgerFactory::new(Arc::clone(&db), entry_log); - let mut ledger = factory.open("ledger-a", &config).unwrap(); - ledger.add_entry(b"first").unwrap(); - ledger.add_entry(b"second").unwrap(); - ledger.add_entry(b"third").unwrap(); + let ledger = factory.open("ledger-a", &config).unwrap(); + append_payload(&ledger, b"first").unwrap(); + append_payload(&ledger, b"second").unwrap(); + append_payload(&ledger, b"third").unwrap(); // phase 1: entry_id > 0 -> same ledger assert_eq!( @@ -390,10 +387,10 @@ fn read_entries_from_respects_limit() { let dir = tempdir().unwrap(); let db = open_test_db(dir.path()); let entry_log = open_test_entry_log(dir.path()); - let mut ledger = RocksDBManagedLedger::open("ledger-1", db, entry_log).unwrap(); - ledger.add_entry(b"first").unwrap(); - ledger.add_entry(b"second").unwrap(); - ledger.add_entry(b"third").unwrap(); + let ledger = RocksDBManagedLedger::open("ledger-1", db, entry_log).unwrap(); + append_payload(&ledger, b"first").unwrap(); + append_payload(&ledger, b"second").unwrap(); + append_payload(&ledger, b"third").unwrap(); let entries = ledger.read_entries_from(&position(0, 0), 2).unwrap(); assert_eq!(entries.len(), 2); assert_eq!(entries[0].payload, "first".as_bytes()); @@ -408,8 +405,8 @@ fn managed_ledger_open_initializes_last_position_runtime_state() { let expected_position = { let db = open_test_db(&db_path); let entry_log = open_test_entry_log(&db_path); - let mut ledger = RocksDBManagedLedger::open("ledger-a", db, entry_log).unwrap(); - ledger.add_entry_with_partition(7, b"first").unwrap() + let ledger = RocksDBManagedLedger::open("ledger-a", db, entry_log).unwrap(); + append_with_partition(&ledger, 7, b"first").unwrap() }; let db = open_test_db(&db_path); @@ -435,10 +432,9 @@ fn message_ledger_append_updates_runtime_last_position() { let db = open_test_db(&db_path); let entry_log = open_test_entry_log(&db_path); - let mut ledger = - RocksDBManagedLedger::open("ledger-a", Arc::clone(&db), Arc::clone(&entry_log)).unwrap(); + let ledger = RocksDBManagedLedger::open("ledger-a", Arc::clone(&db), Arc::clone(&entry_log)).unwrap(); - let first = ledger.add_entry_with_partition(7, b"first").unwrap(); + let first = append_with_partition(&ledger, 7, b"first").unwrap(); // Delete the index and ensure that last_position cannot be obtained through a temporary query in RocksDB. db.delete(keys::managed_entry_key(first.ledger_id, first.entry_id)) @@ -446,11 +442,10 @@ fn message_ledger_append_updates_runtime_last_position() { assert_eq!(ledger.last_position().unwrap(), Some(first)); // Consecutive append - advancing to the last position in the runtime. - let mut ledger = - RocksDBManagedLedger::open("ledger-b", Arc::clone(&db), Arc::clone(&entry_log)).unwrap(); + let ledger = RocksDBManagedLedger::open("ledger-b", Arc::clone(&db), Arc::clone(&entry_log)).unwrap(); - let first = ledger.add_entry_with_partition(7, b"first").unwrap(); - let second = ledger.add_entry_with_partition(7, b"second").unwrap(); + let first = append_with_partition(&ledger, 7, b"first").unwrap(); + let second = append_with_partition(&ledger, 7, b"second").unwrap(); assert_eq!(ledger.last_position().unwrap(), Some(second.clone()),); assert_eq!(second.ledger_id, first.ledger_id); assert_eq!(second.entry_id, first.entry_id + 1); @@ -467,7 +462,7 @@ fn managed_ledger_runtime_last_position_survives_rollover_to_empty_ledger() { ..ManagedLedgerConfig::default() }; - let mut ledger = RocksDBManagedLedger::open_with_config( + let ledger = RocksDBManagedLedger::open_with_config( "ledger-a", Arc::clone(&db), Arc::clone(&entry_log), @@ -475,8 +470,8 @@ fn managed_ledger_runtime_last_position_survives_rollover_to_empty_ledger() { ) .unwrap(); - let first = ledger.add_entry_with_partition(7, b"first").unwrap(); - let second = ledger.add_entry_with_partition(7, b"second").unwrap(); + let first = append_with_partition(&ledger, 7, b"first").unwrap(); + let second = append_with_partition(&ledger, 7, b"second").unwrap(); assert_eq!(first.ledger_id, second.ledger_id); assert_eq!(second.ledger_id, 0); @@ -484,7 +479,7 @@ fn managed_ledger_runtime_last_position_survives_rollover_to_empty_ledger() { assert_eq!(ledger.last_position().unwrap(), Some(second),); // After adding a new entry to the new ledger, the last position should be updated. - let third = ledger.add_entry_with_partition(7, b"third").unwrap(); + let third = append_with_partition(&ledger, 7, b"third").unwrap(); assert_eq!(third.ledger_id, 1); assert_eq!(ledger.last_position().unwrap(), Some(third),) } @@ -498,20 +493,20 @@ fn managed_ledger_runtime_state_recovers_and_continues_after_reopen() { let db = open_test_db(&db_path); let entry_log = open_test_entry_log(&db_path); - let mut ledger = RocksDBManagedLedger::open("ledger-a", db, entry_log).unwrap(); + let ledger = RocksDBManagedLedger::open("ledger-a", db, entry_log).unwrap(); - ledger.add_entry_with_partition(7, b"first").unwrap(); - ledger.add_entry_with_partition(7, b"second").unwrap() + append_with_partition(&ledger, 7, b"first").unwrap(); + append_with_partition(&ledger, 7, b"second").unwrap() }; let db = open_test_db(&db_path); let entry_log = open_test_entry_log(&db_path); - let mut ledger = RocksDBManagedLedger::open("ledger-a", db, entry_log).unwrap(); + let ledger = RocksDBManagedLedger::open("ledger-a", db, entry_log).unwrap(); assert_eq!(ledger.last_position().unwrap(), Some(second.clone())); - let third = ledger.add_entry_with_partition(7, b"third").unwrap(); + let third = append_with_partition(&ledger, 7, b"third").unwrap(); assert_eq!(third.ledger_id, second.ledger_id); assert_eq!(third.entry_id, second.entry_id + 1); From 29614f198ba8726af56f14a9c0f20b1a32c05b1c Mon Sep 17 00:00:00 2001 From: XTLine <2668543450@qq.com> Date: Fri, 31 Jul 2026 15:58:13 +0800 Subject: [PATCH 03/44] Remove unneeded mut binding --- rust/storage/managed-ledger-rocksdb/src/store.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rust/storage/managed-ledger-rocksdb/src/store.rs b/rust/storage/managed-ledger-rocksdb/src/store.rs index 7c2574e..c0e1298 100644 --- a/rust/storage/managed-ledger-rocksdb/src/store.rs +++ b/rust/storage/managed-ledger-rocksdb/src/store.rs @@ -151,7 +151,7 @@ impl RocksDbManagedLedgerStorage { self.with_cursor_lock(|| { let cursor_name = keys::encode_cursor_name(subscription); let ledger = self.topic_ledger(topic)?; - let mut cursor = ledger.open_cursor(&cursor_name)?; + let cursor = ledger.open_cursor(&cursor_name)?; cursor.persist_state() }) } From bd8fe06c9fee5c63fa441d9f624ff2a174ad9aba Mon Sep 17 00:00:00 2001 From: XTLine <2668543450@qq.com> Date: Wed, 12 Aug 2026 17:13:05 +0800 Subject: [PATCH 04/44] perf(storage): cap RocksDB memory usage Default options keep every SST open (max_open_files=-1) with an unbounded block cache, so accumulated data (hundreds of GB across perf runs) let anon memory grow past the 4G cgroup wall and OOM-kill the broker during high-load fanout scenarios. - max_open_files 512: resident table readers bounded - block cache LRU 256MB, index/filter blocks included - bounded memtable (64MB x2) and background jobs (4) --- rust/storage/managed-ledger-rocksdb/src/store.rs | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/rust/storage/managed-ledger-rocksdb/src/store.rs b/rust/storage/managed-ledger-rocksdb/src/store.rs index c0e1298..9aed3f7 100644 --- a/rust/storage/managed-ledger-rocksdb/src/store.rs +++ b/rust/storage/managed-ledger-rocksdb/src/store.rs @@ -5,7 +5,7 @@ use super::keys; use super::write_queue::WriteQueue; use crate::cursor::first_position; use anyhow::{anyhow, Result}; -use rocksdb::{Options, DB}; +use rocksdb::{BlockBasedOptions, Cache, Options, DB}; use std::fmt; use std::path::Path; use std::sync::{Arc, Mutex}; @@ -110,6 +110,14 @@ impl RocksDbManagedLedgerStorage { pub fn open(path: &Path) -> Result { let mut options = Options::default(); options.create_if_missing(true); + options.set_max_open_files(512); + let mut block_opts = BlockBasedOptions::default(); + block_opts.set_block_cache(&Cache::new_lru_cache(256 * 1024 * 1024)); + block_opts.set_cache_index_and_filter_blocks(true); + options.set_block_based_table_factory(&block_opts); + options.set_write_buffer_size(64 * 1024 * 1024); + options.set_max_write_buffer_number(2); + options.set_max_background_jobs(4); let db = Arc::new(DB::open(&options, path)?); let entry_log = Arc::new(EntryLogStore::open(path)?); let factory = RocksDBManagedLedgerFactory::new(db, entry_log); From 15b430381e891bf5cd8ebdcd492ff4fad0067e25 Mon Sep 17 00:00:00 2001 From: XTLine <2668543450@qq.com> Date: Wed, 29 Jul 2026 15:13:40 +0800 Subject: [PATCH 05/44] perf storage leak between scenarios - Pass `cleanup=True` on restart in broker lib to clear /tmp data. - Ensure logs and timeseries are saved before storage is wiped. - Reset broker storage in a `finally` block in the stress test. --- tests/perf/lib/broker.py | 8 ++++---- tests/perf/run_persistent_stress.py | 31 +++++++++++++++++++++++++++-- 2 files changed, 33 insertions(+), 6 deletions(-) diff --git a/tests/perf/lib/broker.py b/tests/perf/lib/broker.py index 7f2bb24..2f7f397 100644 --- a/tests/perf/lib/broker.py +++ b/tests/perf/lib/broker.py @@ -190,8 +190,8 @@ def restart(self, preserve_storage: bool = False) -> None: self.sampler = BrokerSampler(self.broker_pid) self.sampler.start() else: - # Original behavior: fresh start - self.stop() + # Fresh storage: drop the previous /tmp workdir so disk does not accumulate. + self.stop(cleanup=True) self.start() def metrics(self) -> dict[str, float]: @@ -378,6 +378,6 @@ def restart(self, preserve_storage: bool = False) -> None: self.sampler = BrokerSampler(self.broker_pid) self.sampler.start() else: - # Original behavior: fresh start - self.stop() + # Fresh storage: drop the previous /tmp workdir so disk does not accumulate. + self.stop(cleanup=True) self.start() diff --git a/tests/perf/run_persistent_stress.py b/tests/perf/run_persistent_stress.py index 2083b1e..6a61b55 100755 --- a/tests/perf/run_persistent_stress.py +++ b/tests/perf/run_persistent_stress.py @@ -612,7 +612,7 @@ def main(argv: list[str]) -> int: try: result = run_scenario(scenario, broker, scenario_dir) - # Save broker log and timeseries + # Save broker log and timeseries while workdir still exists. if broker.log_path: (scenario_dir / "broker.log").write_text( broker.log_path.read_text( @@ -623,7 +623,8 @@ def main(argv: list[str]) -> int: broker.sampler.write_csv(scenario_dir / "broker_timeseries.csv") # Record artifact paths in result - perf_collector.stop() + if perf_collector is not None: + perf_collector.stop() if perf_data_path.exists(): svg_path = scenario_dir / "flamegraph.svg" ok = PerfCollector.generate_flamegraph(perf_data_path, svg_path) @@ -656,6 +657,16 @@ def main(argv: list[str]) -> int: except Exception as e: print(f" ERROR: {e}", file=sys.stderr) + # Best-effort capture before wiping storage. + try: + if broker.log_path and broker.log_path.exists(): + (scenario_dir / "broker.log").write_text( + broker.log_path.read_text( + encoding="utf-8", errors="replace" + ) + ) + except OSError: + pass if perf_collector is not None: perf_collector.stop() results["scenarios"].append( @@ -670,6 +681,22 @@ def main(argv: list[str]) -> int: ) failed += 1 print(f" ✗ FAIL: {e}") + finally: + # Drop /tmp DB+entrylog after each scenario so storage does not accumulate. + # restart_replay / redelivery manage their own in-scenario preserve restart; + # once the scenario finishes we always wipe before the next one. + try: + broker.restart(preserve_storage=False) + print( + " storage cleaned for next scenario " + f"(workdir={broker.workdir})", + file=sys.stderr, + ) + except Exception as cleanup_err: + print( + f" WARNING: failed to reset broker storage: {cleanup_err}", + file=sys.stderr, + ) finally: broker.stop(cleanup=True) From dba75f27b08c0e8a18802ca2a28069d0d10d78dc Mon Sep 17 00:00:00 2001 From: XTLine <2668543450@qq.com> Date: Thu, 30 Jul 2026 14:58:37 +0800 Subject: [PATCH 06/44] test: increase stress test offered load --- tests/perf/run_non_persistent_stress.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/tests/perf/run_non_persistent_stress.py b/tests/perf/run_non_persistent_stress.py index 3f1081d..3f069ed 100644 --- a/tests/perf/run_non_persistent_stress.py +++ b/tests/perf/run_non_persistent_stress.py @@ -68,7 +68,7 @@ class StressScenario: kind="produce", broker="nonpartitioned", description="单 producer 1M msg/s offered load 吞吐 ceiling", - producer_args=["-time", "60", "-r", "999999", "-s", "1024"], + producer_args=["-time", "60", "-r", "9999999", "-s", "1024"], estimated_duration=60, ), StressScenario( @@ -80,15 +80,17 @@ class StressScenario: "-time", "60", "-r", - "999999", + "9999999", "-s", "1024", "-n", - "4", + "1", "-threads", "4", "-c", "4", + "-o", + "1000", ], estimated_duration=60, ), From 21f9e3a3396aa1ee6c6a725b64e792bda6c225f2 Mon Sep 17 00:00:00 2001 From: XTLine <2668543450@qq.com> Date: Fri, 31 Jul 2026 15:57:58 +0800 Subject: [PATCH 07/44] test: Add perf log parsing and E2E diagnostics tests --- tests/perf/unit/test_parsing.py | 84 +++++++++++++++++++++++ tests/perf/unit/test_perf_cmd_e2e_diag.py | 54 +++++++++++++++ 2 files changed, 138 insertions(+) create mode 100644 tests/perf/unit/test_parsing.py create mode 100644 tests/perf/unit/test_perf_cmd_e2e_diag.py diff --git a/tests/perf/unit/test_parsing.py b/tests/perf/unit/test_parsing.py new file mode 100644 index 0000000..91fa29d --- /dev/null +++ b/tests/perf/unit/test_parsing.py @@ -0,0 +1,84 @@ +"""Unit tests for pulsar-perf log parsing (interval-median thr).""" + +from __future__ import annotations + +import sys +from pathlib import Path + +import pytest + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT)) + +from lib.parsing import parse_consumer_output, parse_producer_output # noqa: E402 + + +PRODUCER_LOG = """ +2026-07-30T18:00:00,000 - INFO - [main:PerformanceProducer@392] - Throughput produced: 100000 msg --- 10000.0 msg/s --- 78.1 Mbit/s --- failure 0.0 msg/s --- Latency: mean: 10.000 ms - med: 9.000 - 95pct: 20.000 - 99pct: 30.000 - 99.9pct: 40.000 - 99.99pct: 50.000 - Max: 60.000 +2026-07-30T18:00:10,000 - INFO - [main:PerformanceProducer@392] - Throughput produced: 300000 msg --- 20000.0 msg/s --- 156.2 Mbit/s --- failure 0.0 msg/s --- Latency: mean: 5.000 ms - med: 4.000 - 95pct: 10.000 - 99pct: 15.000 - 99.9pct: 20.000 - 99.99pct: 25.000 - Max: 30.000 +2026-07-30T18:00:20,000 - INFO - [main:PerformanceProducer@392] - Throughput produced: 500000 msg --- 22000.0 msg/s --- 171.9 Mbit/s --- failure 0.0 msg/s --- Latency: mean: 6.000 ms - med: 5.000 - 95pct: 11.000 - 99pct: 16.000 - 99.9pct: 21.000 - 99.99pct: 26.000 - Max: 35.000 +2026-07-30T18:00:30,000 - INFO - [main:PerformanceProducer@392] - Throughput produced: 700000 msg --- 18000.0 msg/s --- 140.6 Mbit/s --- failure 0.0 msg/s --- Latency: mean: 7.000 ms - med: 6.000 - 95pct: 12.000 - 99pct: 17.000 - 99.9pct: 22.000 - 99.99pct: 27.000 - Max: 40.000 +2026-07-30T18:00:40,000 - INFO - [perf-client-shutdown:PerformanceProducer@774] - Aggregated throughput stats --- 700000 records sent --- 11666.667 msg/s --- 91.146 Mbit/s +2026-07-30T18:00:40,000 - INFO - [perf-client-shutdown:PerformanceProducer@784] - Aggregated latency stats --- Latency: mean: 8.000 ms - med: 7.000 - 95pct: 13.000 - 99pct: 18.000 - 99.9pct: 23.000 - 99.99pct: 28.000 - 99.999pct: 33.000 - Max: 45.000 +""" + +CONSUMER_LOG = """ +2026-07-30T18:00:00,000 - INFO - [main:PerformanceConsumer@484] - Throughput received: 50000 msg --- 5000.000 msg/s --- 39.062 Mbit/s --- Latency: mean: 100.000 ms - med: 90 - 95pct: 200 - 99pct: 300 - 99.9pct: 400 - 99.99pct: 500 - Max: 600 +2026-07-30T18:00:10,000 - INFO - [main:PerformanceConsumer@484] - Throughput received: 250000 msg --- 20000.000 msg/s --- 156.250 Mbit/s --- Latency: mean: 50.000 ms - med: 40 - 95pct: 80 - 99pct: 100 - 99.9pct: 120 - 99.99pct: 140 - Max: 160 +2026-07-30T18:00:20,000 - INFO - [main:PerformanceConsumer@484] - Throughput received: 470000 msg --- 22000.000 msg/s --- 171.875 Mbit/s --- Latency: mean: 55.000 ms - med: 45 - 95pct: 85 - 99pct: 105 - 99.9pct: 125 - 99.99pct: 145 - Max: 165 +2026-07-30T18:00:30,000 - INFO - [main:PerformanceConsumer@484] - Throughput received: 650000 msg --- 18000.000 msg/s --- 140.625 Mbit/s --- Latency: mean: 60.000 ms - med: 50 - 95pct: 90 - 99pct: 110 - 99.9pct: 130 - 99.99pct: 150 - Max: 170 +2026-07-30T18:00:40,000 - INFO - [perf-client-shutdown:PerformanceConsumer@562] - Aggregated throughput stats --- 650000 records received --- 10833.333 msg/s --- 84.635 Mbit/s --- AckRate: 10833.0 msg/s --- ack failed 0 msg +2026-07-30T18:00:40,000 - INFO - [perf-client-shutdown:PerformanceConsumer@575] - Aggregated latency stats --- Latency: mean: 70.000 ms - med: 60 - 95pct: 100 - 99pct: 120 - 99.9pct: 140 - 99.99pct: 160 - 99.999pct: 180 - Max: 200 +""" + +PRODUCER_AGG_ONLY = """ +2026-07-30T18:00:10,000 - INFO - [perf-client-shutdown:PerformanceProducer@774] - Aggregated throughput stats --- 10000 records sent --- 5000.000 msg/s --- 39.062 Mbit/s +2026-07-30T18:00:10,000 - INFO - [perf-client-shutdown:PerformanceProducer@784] - Aggregated latency stats --- Latency: mean: 1.500 ms - med: 1.000 - 95pct: 2.000 - 99pct: 3.000 - 99.9pct: 4.000 - 99.99pct: 5.000 - 99.999pct: 6.000 - Max: 7.000 +""" + + +def test_producer_prefers_interval_median_dropping_first_window(): + # windows: 10k, 20k, 22k, 18k → drop first → median(20,22,18)=20k + result = parse_producer_output(PRODUCER_LOG) + assert result["metric_source"] == "interval_median" + assert result["records"] == 700000 + assert result["throughput_msg_s"] == pytest.approx(20000.0) + assert result["interval_count"] == 4 + assert result["steady_interval_count"] == 3 + assert result["partial"] is False + assert result["aggregated_throughput_msg_s"] == pytest.approx(11666.667) + assert result["latency_p99_ms"] == pytest.approx(16.0) # median of 15,16,17 + assert result["latency_max_ms"] == pytest.approx(40.0) # max of steady windows + + +def test_consumer_prefers_interval_median_dropping_first_window(): + result = parse_consumer_output(CONSUMER_LOG) + assert result["metric_source"] == "interval_median" + assert result["records"] == 650000 + assert result["throughput_msg_s"] == pytest.approx(20000.0) + assert result["ack_failed"] == 0 + assert result["ack_rate_msg_s"] == pytest.approx(10833.0) + assert result["steady_interval_count"] == 3 + assert result["partial"] is False + + +def test_producer_falls_back_to_aggregated_when_no_intervals(): + result = parse_producer_output(PRODUCER_AGG_ONLY) + assert result["metric_source"] == "aggregated" + assert result["records"] == 10000 + assert result["throughput_msg_s"] == pytest.approx(5000.0) + assert result["interval_count"] == 0 + assert result["partial"] is False + + +def test_single_interval_marked_partial(): + single = """ +2026-07-30T18:00:10,000 - INFO - [main:PerformanceProducer@392] - Throughput produced: 100000 msg --- 15000.0 msg/s --- 117.2 Mbit/s --- failure 0.0 msg/s --- Latency: mean: 2.000 ms - med: 1.500 - 95pct: 3.000 - 99pct: 4.000 - 99.9pct: 5.000 - 99.99pct: 6.000 - Max: 7.000 +2026-07-30T18:00:15,000 - INFO - [perf-client-shutdown:PerformanceProducer@774] - Aggregated throughput stats --- 100000 records sent --- 6666.667 msg/s --- 52.083 Mbit/s +2026-07-30T18:00:15,000 - INFO - [perf-client-shutdown:PerformanceProducer@784] - Aggregated latency stats --- Latency: mean: 2.000 ms - med: 1.500 - 95pct: 3.000 - 99pct: 4.000 - 99.9pct: 5.000 - 99.99pct: 6.000 - 99.999pct: 7.000 - Max: 7.000 +""" + result = parse_producer_output(single) + assert result["metric_source"] == "interval_median" + assert result["throughput_msg_s"] == pytest.approx(15000.0) + assert result["partial"] is True + assert result["steady_interval_count"] == 1 diff --git a/tests/perf/unit/test_perf_cmd_e2e_diag.py b/tests/perf/unit/test_perf_cmd_e2e_diag.py new file mode 100644 index 0000000..1707aeb --- /dev/null +++ b/tests/perf/unit/test_perf_cmd_e2e_diag.py @@ -0,0 +1,54 @@ +"""Unit tests for E2E dual-process failure diagnostics.""" + +from __future__ import annotations + +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT)) + +from lib.perf_cmd import ( # noqa: E402 + explain_exit_code, + format_e2e_process_failure, + log_tail, +) + + +def test_explain_exit_code_sigterm(): + text = explain_exit_code(143) + assert "143" in text + assert "SIGTERM" in text + + +def test_format_e2e_shows_both_rcs_and_log_tails(): + msg = format_e2e_process_failure( + consumer_rc=143, + producer_rc=1, + consumer_out="consumer head\n" + "\n".join(f"c{i}" for i in range(100)), + producer_out=( + "Started performance test thread 0\n" + "Created 1 producers\n" + "Exception in thread: boom\n" + ), + first_failed="producer", + ) + assert "first_failed=producer" in msg + assert "consumer_rc=" in msg and "143" in msg + assert "producer_rc=" in msg and "1" in msg + assert "Exception in thread: boom" in msg + assert "--- producer log tail ---" in msg + assert "--- consumer log tail ---" in msg + # tail should not be only the log head + assert "consumer head" not in log_tail("\n".join(f"line{i}" for i in range(80))) + + +def test_infer_first_failed_when_consumer_sigterm_producer_nonzero(): + msg = format_e2e_process_failure( + consumer_rc=143, + producer_rc=1, + consumer_out="c-tail", + producer_out="p-tail-error", + first_failed=None, + ) + assert "first_failed=producer" in msg From 1d63ef7dd85f6ca5f327b1849f4233d65a22e4fd Mon Sep 17 00:00:00 2001 From: XTLine <2668543450@qq.com> Date: Fri, 31 Jul 2026 15:58:10 +0800 Subject: [PATCH 08/44] Refactor perf parsing and error reporting Reworks metric extraction to use the median of steady-state interval windows rather than relying solely on aggregated stats. Also adds detailed logging and exit-code explanations for E2E process failures to improve debuggability. --- tests/perf/lib/parsing.py | 320 ++++++++++++++------ tests/perf/lib/perf_cmd.py | 216 ++++++++++++- tests/perf/run_non_persistent_e2e_matrix.py | 24 +- tests/perf/run_non_persistent_stress.py | 84 +++-- tests/perf/run_persistent_e2e_matrix.py | 28 +- tests/perf/run_persistent_stress.py | 153 +++++++--- 6 files changed, 615 insertions(+), 210 deletions(-) diff --git a/tests/perf/lib/parsing.py b/tests/perf/lib/parsing.py index fa75f3d..fad07f9 100644 --- a/tests/perf/lib/parsing.py +++ b/tests/perf/lib/parsing.py @@ -1,110 +1,246 @@ from __future__ import annotations import re +import statistics from typing import Any +# pulsar-perf prints one interval line about every 10s (Thread.sleep(10000)). +# First window is often cold (connect / warmup / empty feed); drop it when we +# have 2+ samples and take median of the rest as steady-state thr. -def parse_producer_output(text: str) -> dict[str, Any]: - throughput = re.search( - r"Aggregated throughput stats ---\s+(\d+) records sent ---\s+([\d.]+) msg/s ---\s+([\d.]+) Mbit/s", - text, - ) - latency = re.search( - r"Aggregated latency stats --- Latency: mean:\s+([\d.]+) ms - med:\s+([\d.]+) - 95pct:\s+([\d.]+) - 99pct:\s+([\d.]+) - 99\.9pct:\s+([\d.]+) - 99\.99pct:\s+([\d.]+) - 99\.999pct:\s+([\d.]+) - Max:\s+([\d.]+)", - text, - ) - if not throughput or not latency: - interval_matches = re.findall( - r"Throughput produced:\s+(\d+) msg ---\s+([\d.]+)\s+msg/s ---\s+([\d.]+)\s+Mbit/s.*?Latency: mean:\s+([\d.]+)\s+ms.*?99pct:\s+([\d.]+).*?Max:\s+([\d.]+)", - text, - re.S, +_AGG_PRODUCER_THR = re.compile( + r"Aggregated throughput stats ---\s+(\d+) records sent ---\s+([\d.]+) msg/s ---\s+([\d.]+) Mbit/s" +) +_AGG_CONSUMER_THR = re.compile( + r"Aggregated throughput stats ---\s+(\d+) records received ---\s+([\d.]+) msg/s ---\s+([\d.]+) Mbit/s --- AckRate: ([\d.]+)\s+msg/s --- ack failed (\d+) msg" +) +_AGG_LATENCY = re.compile( + r"Aggregated latency stats --- Latency: mean:\s+([\d.]+) ms - med:\s+([\d.]+) - 95pct:\s+([\d.]+) - 99pct:\s+([\d.]+) - 99\.9pct:\s+([\d.]+) - 99\.99pct:\s+([\d.]+) - 99\.999pct:\s+([\d.]+) - Max:\s+([\d.]+)" +) + +# Interval lines (per ~10s window). cumulative total is first number; rate is window thr. +_INTERVAL_PRODUCER = re.compile( + r"Throughput produced:\s+(\d+) msg ---\s+([\d.]+)\s+msg/s ---\s+([\d.]+)\s+Mbit/s" + r".*?Latency: mean:\s+([\d.]+)\s+ms - med:\s+([\d.]+) - 95pct:\s+([\d.]+) - 99pct:\s+([\d.]+)" + r".*?Max:\s+([\d.]+)", + re.S, +) +_INTERVAL_CONSUMER = re.compile( + r"Throughput received:\s+(\d+) msg ---\s+([\d.]+)\s+msg/s ---\s+([\d.]+)\s+Mbit/s" + r".*?Latency: mean:\s+([\d.]+)\s+ms - med:\s+([\d.]+) - 95pct:\s+([\d.]+) - 99pct:\s+([\d.]+)" + r".*?Max:\s+([\d.]+)", + re.S, +) + + +def _median(values: list[float]) -> float: + return float(statistics.median(values)) + + +def _steady_slices(intervals: list[dict[str, float]]) -> list[dict[str, float]]: + """Drop the first interval when 2+ windows exist (cold start / empty feed).""" + if len(intervals) >= 2: + return intervals[1:] + return intervals + + +def _from_intervals(intervals: list[dict[str, float]]) -> dict[str, Any] | None: + if not intervals: + return None + steady = _steady_slices(intervals) + thr = [row["throughput_msg_s"] for row in steady] + mbit = [row["throughput_mbit_s"] for row in steady] + lat_mean = [row["latency_mean_ms"] for row in steady] + lat_p50 = [row["latency_p50_ms"] for row in steady] + lat_p95 = [row["latency_p95_ms"] for row in steady] + lat_p99 = [row["latency_p99_ms"] for row in steady] + lat_max = [row["latency_max_ms"] for row in steady] + return { + "throughput_msg_s": _median(thr), + "throughput_mbit_s": _median(mbit), + "latency_mean_ms": _median(lat_mean), + "latency_p50_ms": _median(lat_p50), + "latency_p95_ms": _median(lat_p95), + "latency_p99_ms": _median(lat_p99), + "latency_max_ms": max(lat_max), + "interval_count": len(intervals), + "steady_interval_count": len(steady), + "interval_throughput_msg_s_median": _median(thr), + "interval_throughput_msg_s_min": min(thr), + "interval_throughput_msg_s_max": max(thr), + "records_cumulative_last": int(intervals[-1]["records_cumulative"]), + "partial": len(steady) < 2, + } + + +def _parse_producer_intervals(text: str) -> list[dict[str, float]]: + rows: list[dict[str, float]] = [] + for m in _INTERVAL_PRODUCER.finditer(text): + rows.append( + { + "records_cumulative": float(m.group(1)), + "throughput_msg_s": float(m.group(2)), + "throughput_mbit_s": float(m.group(3)), + "latency_mean_ms": float(m.group(4)), + "latency_p50_ms": float(m.group(5)), + "latency_p95_ms": float(m.group(6)), + "latency_p99_ms": float(m.group(7)), + "latency_max_ms": float(m.group(8)), + } + ) + return rows + + +def _parse_consumer_intervals(text: str) -> list[dict[str, float]]: + rows: list[dict[str, float]] = [] + for m in _INTERVAL_CONSUMER.finditer(text): + rows.append( + { + "records_cumulative": float(m.group(1)), + "throughput_msg_s": float(m.group(2)), + "throughput_mbit_s": float(m.group(3)), + "latency_mean_ms": float(m.group(4)), + "latency_p50_ms": float(m.group(5)), + "latency_p95_ms": float(m.group(6)), + "latency_p99_ms": float(m.group(7)), + "latency_max_ms": float(m.group(8)), + } ) - if not interval_matches: - raise RuntimeError(f"failed to parse producer output:\n{text}") - ( - records, - throughput_msg_s, - throughput_mbit_s, - latency_mean_ms, - latency_p99_ms, - latency_max_ms, - ) = interval_matches[-1] # type: ignore[misc] - return { - "records": int(records), - "throughput_msg_s": float(throughput_msg_s), - "throughput_mbit_s": float(throughput_mbit_s), - "latency_mean_ms": float(latency_mean_ms), - "latency_p50_ms": None, - "latency_p95_ms": None, - "latency_p99_ms": float(latency_p99_ms), - "latency_p999_ms": None, - "latency_max_ms": float(latency_max_ms), - "partial": True, + return rows + + +def parse_producer_output(text: str) -> dict[str, Any]: + """Parse pulsar-perf producer log. + + Prefer ~10s interval windows (drop first when 2+ exist, median of rest) for + throughput/latency. Fall back to Aggregated wall-clock stats when no + interval lines exist (short -m runs). + """ + intervals = _parse_producer_intervals(text) + steady = _from_intervals(intervals) + + agg_thr = _AGG_PRODUCER_THR.search(text) + agg_lat = _AGG_LATENCY.search(text) + + if steady is not None: + records = int(steady["records_cumulative_last"]) + if agg_thr: + records = int(agg_thr.group(1)) + result = { + "records": records, + "throughput_msg_s": steady["throughput_msg_s"], + "throughput_mbit_s": steady["throughput_mbit_s"], + "latency_mean_ms": steady["latency_mean_ms"], + "latency_p50_ms": steady["latency_p50_ms"], + "latency_p95_ms": steady["latency_p95_ms"], + "latency_p99_ms": steady["latency_p99_ms"], + "latency_p999_ms": ( + float(agg_lat.group(5)) if agg_lat else None + ), + "latency_max_ms": steady["latency_max_ms"], + "partial": steady["partial"], + "metric_source": "interval_median", + "interval_count": steady["interval_count"], + "steady_interval_count": steady["steady_interval_count"], + "interval_throughput_msg_s_median": steady[ + "interval_throughput_msg_s_median" + ], + "interval_throughput_msg_s_min": steady["interval_throughput_msg_s_min"], + "interval_throughput_msg_s_max": steady["interval_throughput_msg_s_max"], } + if agg_thr: + result["aggregated_throughput_msg_s"] = float(agg_thr.group(2)) + result["aggregated_throughput_mbit_s"] = float(agg_thr.group(3)) + if agg_lat: + result["aggregated_latency_mean_ms"] = float(agg_lat.group(1)) + result["aggregated_latency_p99_ms"] = float(agg_lat.group(4)) + result["aggregated_latency_max_ms"] = float(agg_lat.group(8)) + return result + + if not agg_thr or not agg_lat: + raise RuntimeError(f"failed to parse producer output:\n{text}") + return { - "records": int(throughput.group(1)), - "throughput_msg_s": float(throughput.group(2)), - "throughput_mbit_s": float(throughput.group(3)), - "latency_mean_ms": float(latency.group(1)), - "latency_p50_ms": float(latency.group(2)), - "latency_p95_ms": float(latency.group(3)), - "latency_p99_ms": float(latency.group(4)), - "latency_p999_ms": float(latency.group(5)), - "latency_max_ms": float(latency.group(8)), + "records": int(agg_thr.group(1)), + "throughput_msg_s": float(agg_thr.group(2)), + "throughput_mbit_s": float(agg_thr.group(3)), + "latency_mean_ms": float(agg_lat.group(1)), + "latency_p50_ms": float(agg_lat.group(2)), + "latency_p95_ms": float(agg_lat.group(3)), + "latency_p99_ms": float(agg_lat.group(4)), + "latency_p999_ms": float(agg_lat.group(5)), + "latency_max_ms": float(agg_lat.group(8)), "partial": False, + "metric_source": "aggregated", + "interval_count": 0, + "steady_interval_count": 0, } def parse_consumer_output(text: str) -> dict[str, Any]: - throughput = re.search( - r"Aggregated throughput stats ---\s+(\d+) records received ---\s+([\d.]+) msg/s ---\s+([\d.]+) Mbit/s --- AckRate: ([\d.]+)\s+msg/s --- ack failed (\d+) msg", - text, - ) - latency = re.search( - r"Aggregated latency stats --- Latency: mean:\s+([\d.]+) ms - med:\s+([\d.]+) - 95pct:\s+([\d.]+) - 99pct:\s+([\d.]+) - 99\.9pct:\s+([\d.]+) - 99\.99pct:\s+([\d.]+) - 99\.999pct:\s+([\d.]+) - Max:\s+([\d.]+)", - text, - ) - if not throughput or not latency: - interval_matches = re.findall( - r"Throughput received:\s+(\d+) msg ---\s+([\d.]+)\s+msg/s ---\s+([\d.]+)\s+Mbit/s.*?Latency: mean:\s+([\d.]+)\s+ms.*?99pct:\s+([\d.]+).*?Max:\s+([\d.]+)", - text, - re.S, - ) - if not interval_matches: - raise RuntimeError(f"failed to parse consumer output:\n{text}") - ( - records, - throughput_msg_s, - throughput_mbit_s, - latency_mean_ms, - latency_p99_ms, - latency_max_ms, - ) = interval_matches[-1] # type: ignore[misc] - return { - "records": int(records), - "throughput_msg_s": float(throughput_msg_s), - "throughput_mbit_s": float(throughput_mbit_s), - "ack_rate_msg_s": None, - "ack_failed": None, - "latency_mean_ms": float(latency_mean_ms), - "latency_p50_ms": None, - "latency_p95_ms": None, - "latency_p99_ms": float(latency_p99_ms), - "latency_p999_ms": None, - "latency_max_ms": float(latency_max_ms), - "partial": True, + """Parse pulsar-perf consumer log. + + Same interval-median preference as producer. Ack fields still come from + Aggregated when present (interval lines do not include ack rate). + """ + intervals = _parse_consumer_intervals(text) + steady = _from_intervals(intervals) + + agg_thr = _AGG_CONSUMER_THR.search(text) + agg_lat = _AGG_LATENCY.search(text) + + if steady is not None: + records = int(steady["records_cumulative_last"]) + if agg_thr: + records = int(agg_thr.group(1)) + result = { + "records": records, + "throughput_msg_s": steady["throughput_msg_s"], + "throughput_mbit_s": steady["throughput_mbit_s"], + "ack_rate_msg_s": float(agg_thr.group(4)) if agg_thr else None, + "ack_failed": int(agg_thr.group(5)) if agg_thr else None, + "latency_mean_ms": steady["latency_mean_ms"], + "latency_p50_ms": steady["latency_p50_ms"], + "latency_p95_ms": steady["latency_p95_ms"], + "latency_p99_ms": steady["latency_p99_ms"], + "latency_p999_ms": float(agg_lat.group(5)) if agg_lat else None, + "latency_max_ms": steady["latency_max_ms"], + "partial": steady["partial"], + "metric_source": "interval_median", + "interval_count": steady["interval_count"], + "steady_interval_count": steady["steady_interval_count"], + "interval_throughput_msg_s_median": steady[ + "interval_throughput_msg_s_median" + ], + "interval_throughput_msg_s_min": steady["interval_throughput_msg_s_min"], + "interval_throughput_msg_s_max": steady["interval_throughput_msg_s_max"], } + if agg_thr: + result["aggregated_throughput_msg_s"] = float(agg_thr.group(2)) + result["aggregated_throughput_mbit_s"] = float(agg_thr.group(3)) + if agg_lat: + result["aggregated_latency_mean_ms"] = float(agg_lat.group(1)) + result["aggregated_latency_p99_ms"] = float(agg_lat.group(4)) + result["aggregated_latency_max_ms"] = float(agg_lat.group(8)) + return result + + if not agg_thr or not agg_lat: + raise RuntimeError(f"failed to parse consumer output:\n{text}") + return { - "records": int(throughput.group(1)), - "throughput_msg_s": float(throughput.group(2)), - "throughput_mbit_s": float(throughput.group(3)), - "ack_rate_msg_s": float(throughput.group(4)), - "ack_failed": int(throughput.group(5)), - "latency_mean_ms": float(latency.group(1)), - "latency_p50_ms": float(latency.group(2)), - "latency_p95_ms": float(latency.group(3)), - "latency_p99_ms": float(latency.group(4)), - "latency_p999_ms": float(latency.group(5)), - "latency_max_ms": float(latency.group(8)), + "records": int(agg_thr.group(1)), + "throughput_msg_s": float(agg_thr.group(2)), + "throughput_mbit_s": float(agg_thr.group(3)), + "ack_rate_msg_s": float(agg_thr.group(4)), + "ack_failed": int(agg_thr.group(5)), + "latency_mean_ms": float(agg_lat.group(1)), + "latency_p50_ms": float(agg_lat.group(2)), + "latency_p95_ms": float(agg_lat.group(3)), + "latency_p99_ms": float(agg_lat.group(4)), + "latency_p999_ms": float(agg_lat.group(5)), + "latency_max_ms": float(agg_lat.group(8)), "partial": False, + "metric_source": "aggregated", + "interval_count": 0, + "steady_interval_count": 0, } diff --git a/tests/perf/lib/perf_cmd.py b/tests/perf/lib/perf_cmd.py index a8a8ee8..c30ac64 100644 --- a/tests/perf/lib/perf_cmd.py +++ b/tests/perf/lib/perf_cmd.py @@ -99,6 +99,108 @@ def wait_for_log(path: Path, needle: str, timeout: float = 30.0) -> None: raise RuntimeError(f"timed out waiting for {needle!r} in {path}") +def explain_exit_code(rc: int | None) -> str: + """Human-readable process exit status (143 = SIGTERM from harness, etc.).""" + if rc is None: + return "still-running" + if rc == 0: + return "0 (ok)" + if rc < 0: + return f"{rc} (killed by signal {-rc})" + if rc == 143: + return ( + "143 (SIGTERM: process was terminated — often by the harness " + "after the peer failed, or on overall timeout; not a Java business error)" + ) + if rc == 137: + return "137 (SIGKILL / likely OOM killer)" + if rc > 128: + return f"{rc} (signal {rc - 128})" + return f"{rc} (non-zero process exit)" + + +def log_tail(text: str, *, max_lines: int = 50, max_chars: int = 4000) -> str: + """Tail of a perf log (errors and Aggregated lines are usually at the end).""" + lines = text.splitlines() + tail_lines = lines[-max_lines:] if lines else [] + tail = "\n".join(tail_lines) + if len(tail) > max_chars: + tail = tail[-max_chars:] + return tail if tail.strip() else "(log empty)" + + +def format_e2e_process_failure( + *, + consumer_rc: int, + producer_rc: int, + consumer_out: str, + producer_out: str, + first_failed: str | None = None, + consumer_label: str = "consumer", + producer_label: str = "producer", +) -> str: + """Build an error message that shows both exit codes and log tails.""" + if first_failed is None: + if producer_rc not in (0, 143) and consumer_rc == 143: + first_failed = "producer" + elif consumer_rc not in (0, 143) and producer_rc == 143: + first_failed = "consumer" + elif consumer_rc == 143 and producer_rc == 143: + first_failed = "timeout-or-both-sigterm" + elif producer_rc != 0: + first_failed = "producer" + elif consumer_rc != 0: + first_failed = "consumer" + else: + first_failed = "unknown" + + hang_note = "" + if first_failed and first_failed.startswith("peer_hang_after_"): + hang_note = ( + "\n note: one side finished successfully (rc=0); the peer was still " + "alive after peer_grace and was SIGTERM'd. Often the peer is stuck " + "reconnecting after broker drop, or its -time window ends later " + "because it started later. Check whether broker stayed up." + ) + + parts = [ + "E2E dual-process failure " + f"(first_failed={first_failed})", + f" {consumer_label}_rc={explain_exit_code(consumer_rc)}", + f" {producer_label}_rc={explain_exit_code(producer_rc)}", + ] + if hang_note: + parts.append(hang_note) + parts.extend( + [ + f"--- {producer_label} log tail ---", + log_tail(producer_out), + f"--- {consumer_label} log tail ---", + log_tail(consumer_out), + ] + ) + return "\n".join(parts) + + +def e2e_success_despite_peer_hang( + consumer_rc: int, + producer_rc: int, + first_failed: str | None, +) -> bool: + """True when the primary side finished ok and only the peer was grace-killed. + + Used so -time E2E does not hard-fail after a 600s hang when consumer already + completed successfully and producer is stuck reconnecting. + """ + if not first_failed or not first_failed.startswith("peer_hang_after_"): + return False + if first_failed == "peer_hang_after_consumer_ok": + return consumer_rc == 0 and producer_rc in (0, 143, -15) + if first_failed == "peer_hang_after_producer_ok": + return producer_rc == 0 and consumer_rc in (0, 143, -15) + return False + + def run_consumer_then_feed( consumer_cmd: list[str], producer_cmd: list[str], @@ -106,8 +208,16 @@ def run_consumer_then_feed( producer_log: Path, consumer_timeout: float = 300.0, producer_timeout: float = 300.0, -) -> tuple[str, str, int, int]: - with consumer_log.open("w", encoding="utf-8") as consumer_fh: +) -> tuple[str, str, int, int, str | None]: + """Run consumer then feed producer. + + Returns: + consumer_out, producer_out, consumer_rc, producer_rc, first_failed + first_failed is \"consumer\" | \"producer\" | \"timeout\" | None (both ok). + """ + # Line-buffered file captures so interval/error lines show up sooner if the + # process dies. + with consumer_log.open("w", encoding="utf-8", buffering=1) as consumer_fh: consumer_proc = subprocess.Popen( consumer_cmd, stdout=consumer_fh, @@ -120,9 +230,10 @@ def run_consumer_then_feed( wait_for_log(consumer_log, "Start receiving from") except Exception: _terminate_process(consumer_proc) + # Ensure FH closed and content readable for the error path. raise - with producer_log.open("w", encoding="utf-8") as producer_fh: + with producer_log.open("w", encoding="utf-8", buffering=1) as producer_fh: producer_proc = subprocess.Popen( producer_cmd, stdout=producer_fh, @@ -131,9 +242,11 @@ def run_consumer_then_feed( env=ENV_BASE, ) - consumer_rc, producer_rc = _wait_both_or_kill( + consumer_rc, producer_rc, first_failed = _wait_both_or_kill( consumer_proc, producer_proc, + consumer_log=consumer_log, + producer_log=producer_log, timeout=max(consumer_timeout, producer_timeout), ) @@ -142,12 +255,24 @@ def run_consumer_then_feed( producer_log.read_text(encoding="utf-8", errors="replace"), consumer_rc, producer_rc, + first_failed, ) +def _flush_log_path(path: Path) -> None: + """Best-effort: give the OS/Java a moment, then touch-read the log.""" + time.sleep(0.5) + try: + # Force directory entry / page cache visibility for subsequent reads. + with path.open("rb") as fh: + fh.seek(0, 2) + except OSError: + pass + + def _terminate_process(proc: subprocess.Popen) -> int: if proc.poll() is not None: - return proc.returncode + return proc.returncode if proc.returncode is not None else -1 proc.terminate() try: return proc.wait(timeout=10) @@ -159,24 +284,93 @@ def _terminate_process(proc: subprocess.Popen) -> int: def _wait_both_or_kill( consumer_proc: subprocess.Popen, producer_proc: subprocess.Popen, + *, + consumer_log: Path, + producer_log: Path, timeout: float, -) -> tuple[int, int]: + peer_grace_s: float = 45.0, +) -> tuple[int, int, str | None]: + """Wait for both processes. + + - Both exit → return their codes. + - One exits non-zero → SIGTERM the peer immediately (first_failed=that side). + - One exits zero while the other is still running → give the peer + ``peer_grace_s`` to finish (covers producer started after consumer, or + brief drain). If still alive, SIGTERM the peer and tag + first_failed=\"peer_hang_after__ok\" so callers can treat metrics + as usable instead of a mysterious 600s timeout. + - Overall ``timeout`` still bounds the whole wait. + """ deadline = time.monotonic() + timeout + first_failed: str | None = None + # When set, the named side already exited 0; peer must finish by this time. + peer_grace_deadline: float | None = None + ok_side: str | None = None while True: consumer_rc = consumer_proc.poll() producer_rc = producer_proc.poll() + now = time.monotonic() if consumer_rc is not None and producer_rc is not None: - return consumer_rc, producer_rc + return consumer_rc, producer_rc, first_failed + # One side failed hard: kill peer. if consumer_rc is not None and consumer_rc != 0: - return consumer_rc, _terminate_process(producer_proc) + if first_failed is None: + first_failed = "consumer" + _flush_log_path(consumer_log) + peer_rc = _terminate_process(producer_proc) + _flush_log_path(producer_log) + return consumer_rc, peer_rc, first_failed if producer_rc is not None and producer_rc != 0: - return _terminate_process(consumer_proc), producer_rc + if first_failed is None: + first_failed = "producer" + _flush_log_path(producer_log) + peer_rc = _terminate_process(consumer_proc) + _flush_log_path(consumer_log) + return peer_rc, producer_rc, first_failed + + # One side finished cleanly (rc=0); start/refresh grace for the peer. + if consumer_rc == 0 and producer_rc is None: + if ok_side != "consumer": + ok_side = "consumer" + peer_grace_deadline = now + peer_grace_s + elif producer_rc == 0 and consumer_rc is None: + if ok_side != "producer": + ok_side = "producer" + peer_grace_deadline = now + peer_grace_s - if time.monotonic() >= deadline: - return _terminate_process(consumer_proc), _terminate_process(producer_proc) + if peer_grace_deadline is not None and now >= peer_grace_deadline: + # Successful side is done; peer hung (often reconnect after broker drop). + first_failed = f"peer_hang_after_{ok_side}_ok" + _flush_log_path(consumer_log) + _flush_log_path(producer_log) + if consumer_rc is None: + consumer_rc = _terminate_process(consumer_proc) + if producer_rc is None: + producer_rc = _terminate_process(producer_proc) + _flush_log_path(consumer_log) + _flush_log_path(producer_log) + return consumer_rc, producer_rc, first_failed + + if now >= deadline: + first_failed = first_failed or "timeout" + _flush_log_path(consumer_log) + _flush_log_path(producer_log) + c_rc = ( + consumer_rc + if consumer_rc is not None + else _terminate_process(consumer_proc) + ) + p_rc = ( + producer_rc + if producer_rc is not None + else _terminate_process(producer_proc) + ) + _flush_log_path(consumer_log) + _flush_log_path(producer_log) + return c_rc, p_rc, first_failed time.sleep(0.2) diff --git a/tests/perf/run_non_persistent_e2e_matrix.py b/tests/perf/run_non_persistent_e2e_matrix.py index 3f4da4a..e9dcbde 100644 --- a/tests/perf/run_non_persistent_e2e_matrix.py +++ b/tests/perf/run_non_persistent_e2e_matrix.py @@ -12,7 +12,13 @@ from lib import ROOT from lib.broker import BrokerConfig, BrokerProcess from lib.parsing import parse_consumer_output, parse_producer_output -from lib.perf_cmd import ensure_prereqs, perf_cmd, run_consumer_then_feed, run_sync +from lib.perf_cmd import ( + ensure_prereqs, + format_e2e_process_failure, + perf_cmd, + run_consumer_then_feed, + run_sync, +) RESULTS_PATH = ROOT / "docs" / "perf" / "data" / "non_persistent_e2e_matrix_results.json" ARTIFACTS_DIR = ROOT / "docs" / "perf" / "data" / "non_persistent_e2e_matrix_logs" @@ -399,15 +405,21 @@ def main() -> int: topic, scenario_dir / "feed_producer.hdr", ) - consumer_text, producer_text, consumer_rc, producer_rc = ( + consumer_text, producer_text, consumer_rc, producer_rc, first_failed = ( run_consumer_then_feed( consumer_cmd, producer_cmd, consumer_log, producer_log ) ) - if producer_rc != 0: - raise RuntimeError(f"producer failed:\n{producer_text}") - if consumer_rc != 0: - raise RuntimeError(f"consumer failed:\n{consumer_text}") + if consumer_rc != 0 or producer_rc != 0: + raise RuntimeError( + format_e2e_process_failure( + consumer_rc=consumer_rc, + producer_rc=producer_rc, + consumer_out=consumer_text, + producer_out=producer_text, + first_failed=first_failed, + ) + ) result_entry["producer_metrics"] = parse_producer_output(producer_text) result_entry["metrics"] = parse_consumer_output(consumer_text) else: diff --git a/tests/perf/run_non_persistent_stress.py b/tests/perf/run_non_persistent_stress.py index 3f069ed..ea2bf1f 100644 --- a/tests/perf/run_non_persistent_stress.py +++ b/tests/perf/run_non_persistent_stress.py @@ -29,6 +29,7 @@ from lib.parsing import parse_consumer_output, parse_producer_output from lib.perf_cmd import ( ensure_prereqs, + format_e2e_process_failure, perf_cmd, run_consumer_then_feed, run_sync, @@ -63,24 +64,25 @@ class StressScenario: STRESS_SCENARIOS: list[StressScenario] = [ # --- Producer stress --- + # Aligned with persistent stress on -r / -o / -c / consumer -q where comparable. StressScenario( name="stress_producer_max_rate", kind="produce", broker="nonpartitioned", - description="单 producer 1M msg/s offered load 吞吐 ceiling", - producer_args=["-time", "60", "-r", "9999999", "-s", "1024"], + description="单 producer 全速 60s（-r/-o 对齐 persistent）", + producer_args=["-time", "60", "-r", "999999", "-s", "1024", "-o", "1000"], estimated_duration=60, ), StressScenario( name="stress_producer_max_rate_multi_producer", kind="produce", broker="nonpartitioned", - description="4 producers 1M msg/s aggregate offered load 并发吞吐 ceiling", + description="4 producers 并发全速 60s（-r/-o 对齐 persistent）", producer_args=[ "-time", "60", "-r", - "9999999", + "999999", "-s", "1024", "-n", @@ -98,8 +100,8 @@ class StressScenario: name="stress_producer_large_payload", kind="produce", broker="nonpartitioned", - description="100KiB payload 1M msg/s offered load 带宽瓶颈", - producer_args=["-time", "60", "-r", "999999", "-s", "102400"], + description="100KiB payload 全速 60s（-o 对齐 persistent）", + producer_args=["-time", "60", "-r", "999999", "-s", "102400", "-o", "1000"], estimated_duration=60, ), StressScenario( @@ -107,7 +109,7 @@ class StressScenario: kind="produce", broker="nonpartitioned", description="5 分钟持续发送稳定性", - producer_args=["-time", "300", "-r", "999999", "-s", "1024"], + producer_args=["-time", "300", "-r", "999999", "-s", "1024", "-o", "1000"], estimated_duration=300, ), # --- Consumer / E2E stress --- @@ -115,67 +117,48 @@ class StressScenario: name="stress_consume_shared_max_rate", kind="consume_e2e", broker="nonpartitioned", - description="Shared 单 consumer 1M msg/s offered load 吞吐 ceiling", + description="Shared 单 consumer 全速 60s（-q/-o 对齐 persistent）", producer_args=[], - consumer_args=["-time", "60", "-q", "10000", "-st", "Shared"], - feed_producer_args=["-time", "60", "-r", "999999", "-s", "1024"], + consumer_args=["-time", "60", "-q", "1000", "-st", "Shared"], + feed_producer_args=["-time", "60", "-r", "999999", "-s", "1024", "-o", "1000"], estimated_duration=60, ), StressScenario( name="stress_consume_shared_high_fanout", kind="consume_e2e", broker="nonpartitioned", - description="Shared 16 consumers 1M msg/s offered load 高 fanout", + description="Shared 16 consumers 高扇出 60s（-q 对齐，去掉 -c）", producer_args=[], consumer_args=[ "-time", "60", "-q", - "10000", + "1000", "-st", "Shared", "-n", "16", - "-c", - "4", ], - feed_producer_args=["-time", "60", "-r", "999999", "-s", "1024", "-c", "4"], + feed_producer_args=["-time", "60", "-r", "999999", "-s", "1024", "-o", "1000"], estimated_duration=60, ), StressScenario( name="stress_consume_multi_subscription_fanout", kind="consume_e2e", broker="nonpartitioned", - description="8 subscriptions 1M msg/s offered load 高 fanout", + description="8 subscriptions 扇出 60s（-q/-o 对齐，去掉 -c/memory-limit）", producer_args=[], consumer_args=[ "-time", "60", "-q", - "10000", + "1000", "-st", "Shared", "-ns", "8", - "-c", - "4", - ], - feed_producer_args=[ - "-time", - "60", - "-r", - "999999", - "-s", - "1024", - "-c", - "4", - "--memory-limit", - "268435456", - "--max-outstanding", - "4096", - "--max-outstanding-across-partitions", - "16384", ], + feed_producer_args=["-time", "60", "-r", "999999", "-s", "1024", "-o", "1000"], estimated_duration=60, ), StressScenario( @@ -184,29 +167,27 @@ class StressScenario: broker="nonpartitioned", description="5 分钟持续消费稳定性", producer_args=[], - consumer_args=["-time", "300", "-q", "10000", "-st", "Shared", "-c", "4"], - feed_producer_args=["-time", "300", "-r", "999999", "-s", "1024", "-c", "4"], + consumer_args=["-time", "300", "-q", "1000", "-st", "Shared"], + feed_producer_args=["-time", "300", "-r", "999999", "-s", "1024", "-o", "1000"], estimated_duration=300, ), StressScenario( name="stress_consume_partitioned_max_rate", kind="consume_e2e", broker="nonpersistent_partitioned", - description="Partitioned 4 partitions Shared 4 consumers 1M msg/s offered load", + description="Partitioned 4 partitions Shared 4 consumers 60s（-q 对齐，去掉 -c）", producer_args=[], consumer_args=[ "-time", "60", "-q", - "10000", + "1000", "-st", "Shared", "-n", "4", - "-c", - "4", ], - feed_producer_args=["-time", "60", "-r", "999999", "-s", "1024", "-c", "4"], + feed_producer_args=["-time", "60", "-r", "999999", "-s", "1024", "-o", "1000"], estimated_duration=60, ), ] @@ -358,7 +339,7 @@ def _run_consume_e2e_scenario( started_at = datetime.now(timezone.utc).isoformat() t0 = time.monotonic() - consumer_out, producer_out, consumer_rc, producer_rc = run_consumer_then_feed( + consumer_out, producer_out, consumer_rc, producer_rc, first_failed = run_consumer_then_feed( consumer_cmd, producer_cmd, consumer_log, @@ -377,8 +358,23 @@ def _run_consume_e2e_scenario( status = ( "ok" if consumer_rc == 0 and producer_rc == 0 - else (f"consumer_exit:{consumer_rc},producer_exit:{producer_rc}") + else ( + f"consumer_exit:{consumer_rc},producer_exit:{producer_rc}" + f",first_failed:{first_failed}" + ) ) + if consumer_rc != 0 or producer_rc != 0: + # Keep going (non-persistent stress soft-fails), but print full dual-rc diagnosis. + print( + format_e2e_process_failure( + consumer_rc=consumer_rc, + producer_rc=producer_rc, + consumer_out=consumer_out, + producer_out=producer_out, + first_failed=first_failed, + ), + flush=True, + ) result: dict = { "name": scenario.name, "kind": scenario.kind, diff --git a/tests/perf/run_persistent_e2e_matrix.py b/tests/perf/run_persistent_e2e_matrix.py index bec08fd..03a7335 100755 --- a/tests/perf/run_persistent_e2e_matrix.py +++ b/tests/perf/run_persistent_e2e_matrix.py @@ -20,7 +20,14 @@ from lib.broker import BrokerConfig, BrokerProcess, DockerBrokerProcess from lib.docker_image import build_broker_image from lib.parsing import parse_consumer_output, parse_producer_output -from lib.perf_cmd import ensure_prereqs, perf_cmd, run_consumer_then_feed, run_sync +from lib.perf_cmd import ( + ensure_prereqs, + e2e_success_despite_peer_hang, + format_e2e_process_failure, + perf_cmd, + run_consumer_then_feed, + run_sync, +) RESULTS_PATH = ROOT / "docs" / "perf" / "data" / "persistent_e2e_matrix_results.json" ARTIFACTS_DIR = ROOT / "docs" / "perf" / "data" / "persistent_e2e_matrix_logs" @@ -317,16 +324,23 @@ def run_consume_e2e_scenario( ) start = time.time() - consumer_out, producer_out, consumer_rc, producer_rc = run_consumer_then_feed( + consumer_out, producer_out, consumer_rc, producer_rc, first_failed = run_consumer_then_feed( consumer_cmd, producer_cmd, consumer_log, producer_log, consumer_timeout=120.0, producer_timeout=120.0, ) duration = time.time() - start - - if consumer_rc != 0: - raise RuntimeError(f"consumer failed with rc={consumer_rc}: {consumer_out[:500]}") - if producer_rc != 0: - raise RuntimeError(f"producer failed with rc={producer_rc}: {producer_out[:500]}") + + if consumer_rc != 0 or producer_rc != 0: + if not e2e_success_despite_peer_hang(consumer_rc, producer_rc, first_failed): + raise RuntimeError( + format_e2e_process_failure( + consumer_rc=consumer_rc, + producer_rc=producer_rc, + consumer_out=consumer_out, + producer_out=producer_out, + first_failed=first_failed, + ) + ) consumer_result = parse_consumer_output(consumer_out) producer_result = parse_producer_output(producer_out) diff --git a/tests/perf/run_persistent_stress.py b/tests/perf/run_persistent_stress.py index 6a61b55..8114c63 100755 --- a/tests/perf/run_persistent_stress.py +++ b/tests/perf/run_persistent_stress.py @@ -21,7 +21,14 @@ from lib.docker_image import build_broker_image from lib.observability import PerfCollector from lib.parsing import parse_consumer_output, parse_producer_output -from lib.perf_cmd import ensure_prereqs, perf_cmd, run_consumer_then_feed, run_sync +from lib.perf_cmd import ( + ensure_prereqs, + e2e_success_despite_peer_hang, + format_e2e_process_failure, + perf_cmd, + run_consumer_then_feed, + run_sync, +) RESULTS_PATH = ROOT / "docs" / "perf" / "data" / "persistent_stress_results.json" ARTIFACTS_DIR = ROOT / "docs" / "perf" / "data" / "persistent_stress_logs" @@ -54,17 +61,17 @@ class Scenario: name="stress_persistent_producer_max_rate", kind="produce", broker="persistent_stress", - description="单 producer 全速发送 200w 条", - producer_args=["-m", "2000000", "-s", "1024", "-r", "999999", "-o", "1000"], + description="单 producer 全速 60s（对齐 non-persistent -time）", + producer_args=["-time", "60", "-s", "1024", "-r", "999999", "-o", "1000"], ), Scenario( name="stress_persistent_producer_multi_producer", kind="produce", broker="persistent_stress", - description="4 producers 并发全速发送 200w 条", + description="4 producers 并发全速 60s", producer_args=[ - "-m", - "2000000", + "-time", + "60", "-s", "1024", "-r", @@ -83,50 +90,75 @@ class Scenario: name="stress_persistent_producer_large_payload", kind="produce", broker="persistent_stress", - description="100KiB payload 发送 10k 条", - # 10k * 100KiB ≈ 1GiB payload; 200k would be ~19GiB - producer_args=["-m", "10000", "-s", "102400", "-r", "500"], + description="100KiB payload 全速 60s（对齐 non-persistent）", + producer_args=["-time", "60", "-s", "102400", "-r", "999999", "-o", "1000"], ), Scenario( name="stress_persistent_producer_sustained", kind="produce", broker="persistent_stress", - description="持续限速发送 500k 条 @ 10k msg/s (~50s)", - producer_args=["-m", "500000", "-s", "1024", "-r", "10000"], + description="限速 10k msg/s 持续 60s", + producer_args=["-time", "60", "-s", "1024", "-r", "10000", "-o", "1000"], ), - # Consumer/E2E stress (4) + # Consumer/E2E stress (4) — -time 60, no -m (steady thr via interval median) Scenario( name="stress_persistent_consume_shared_max_rate", kind="consume_e2e", broker="persistent_stress", - description="Shared 全速消费 200k 条", - consumer_args=["-m", "200000", "-q", "1000", "-st", "Shared"], - feed_producer_args=["-m", "200000", "-s", "1024", "-r", "999999"], + description="Shared 单 consumer 全速 60s", + consumer_args=["-time", "60", "-q", "1000", "-st", "Shared"], + feed_producer_args=["-time", "60", "-s", "1024", "-r", "999999", "-o", "1000"], ), Scenario( name="stress_persistent_consume_shared_high_fanout", kind="consume_e2e", broker="persistent_stress", - description="16 consumers 高扇出消费 200k 条", - consumer_args=["-m", "200000", "-q", "1000", "-st", "Shared", "-n", "16"], - feed_producer_args=["-m", "200000", "-s", "1024", "-r", "999999"], + description="16 consumers 高扇出 60s", + consumer_args=[ + "-time", + "60", + "-q", + "1000", + "-st", + "Shared", + "-n", + "16", + ], + feed_producer_args=["-time", "60", "-s", "1024", "-r", "999999", "-o", "1000"], ), Scenario( name="stress_persistent_consume_multi_subscription_fanout", kind="consume_e2e", broker="persistent_stress", - description="8 subscriptions 扇出：生产 100k / 消费 800k 条", - # each subscription receives a full copy; consumer -m = produce * ns - consumer_args=["-m", "800000", "-q", "1000", "-st", "Shared", "-ns", "8"], - feed_producer_args=["-m", "100000", "-s", "1024", "-r", "999999"], + description="8 subscriptions 扇出 60s", + consumer_args=[ + "-time", + "60", + "-q", + "1000", + "-st", + "Shared", + "-ns", + "8", + ], + feed_producer_args=["-time", "60", "-s", "1024", "-r", "999999", "-o", "1000"], ), Scenario( name="stress_persistent_consume_partitioned_max_rate", kind="consume_e2e", broker="persistent_stress_partitioned", - description="4 partitions + 4 consumers 消费 200k 条", - consumer_args=["-m", "200000", "-q", "1000", "-st", "Shared", "-n", "4"], - feed_producer_args=["-m", "200000", "-s", "1024", "-r", "999999"], + description="4 partitions + 4 consumers 60s", + consumer_args=[ + "-time", + "60", + "-q", + "1000", + "-st", + "Shared", + "-n", + "4", + ], + feed_producer_args=["-time", "60", "-s", "1024", "-r", "999999", "-o", "1000"], ), # Persistent-specific stress (3) Scenario( @@ -226,23 +258,34 @@ def run_consume_e2e_scenario( ) start = time.time() - consumer_out, producer_out, consumer_rc, producer_rc = run_consumer_then_feed( - consumer_cmd, - producer_cmd, - consumer_log, - producer_log, - consumer_timeout=600.0, - producer_timeout=600.0, + consumer_out, producer_out, consumer_rc, producer_rc, first_failed = ( + run_consumer_then_feed( + consumer_cmd, + producer_cmd, + consumer_log, + producer_log, + consumer_timeout=600.0, + producer_timeout=600.0, + ) ) duration = time.time() - start - if consumer_rc != 0: - raise RuntimeError( - f"consumer failed with rc={consumer_rc}: {consumer_out[:500]}" - ) - if producer_rc != 0: - raise RuntimeError( - f"producer failed with rc={producer_rc}: {producer_out[:500]}" + if consumer_rc != 0 or producer_rc != 0: + if not e2e_success_despite_peer_hang(consumer_rc, producer_rc, first_failed): + raise RuntimeError( + format_e2e_process_failure( + consumer_rc=consumer_rc, + producer_rc=producer_rc, + consumer_out=consumer_out, + producer_out=producer_out, + first_failed=first_failed, + ) + ) + print( + " note: peer grace-killed after other side ok " + f"(first_failed={first_failed}, " + f"consumer_rc={consumer_rc}, producer_rc={producer_rc})", + flush=True, ) consumer_result = parse_consumer_output(consumer_out) @@ -254,6 +297,7 @@ def run_consume_e2e_scenario( "producer": producer_result, "broker": broker_metrics, "duration_s": round(duration, 2), + "e2e_first_failed": first_failed, } @@ -411,19 +455,28 @@ def run_redelivery_unacked_scenario( ) print(" Partial consume...") - consumer_out, producer_out, consumer_rc, producer_rc = run_consumer_then_feed( - consumer_cmd, - producer_cmd, - consumer1_log, - producer_log, - consumer_timeout=600.0, - producer_timeout=600.0, + consumer_out, producer_out, consumer_rc, producer_rc, first_failed = ( + run_consumer_then_feed( + consumer_cmd, + producer_cmd, + consumer1_log, + producer_log, + consumer_timeout=600.0, + producer_timeout=600.0, + ) ) - if consumer_rc != 0: - raise RuntimeError(f"consumer1 failed: {consumer_out[:500]}") - if producer_rc != 0: - raise RuntimeError(f"producer failed: {producer_out[:500]}") + if consumer_rc != 0 or producer_rc != 0: + raise RuntimeError( + format_e2e_process_failure( + consumer_rc=consumer_rc, + producer_rc=producer_rc, + consumer_out=consumer_out, + producer_out=producer_out, + first_failed=first_failed, + consumer_label="consumer1", + ) + ) producer_result = parse_producer_output(producer_out) consumer1_result = parse_consumer_output(consumer_out) From 9867496a2b0c49a3faebbc3524a172323c9470d8 Mon Sep 17 00:00:00 2001 From: XTLine <2668543450@qq.com> Date: Tue, 4 Aug 2026 10:43:28 +0800 Subject: [PATCH 09/44] feat: add external broker backend and local cgroup limits Adds an external broker backend to target standalone instances without lifecycle management. Also introduces local resource constraints via systemd-run and taskset, and adds a new 10GB solo-consumer backlog drain scenario. --- tests/perf/lib/broker.py | 70 ++++++++++++++++++++++- tests/perf/run_persistent_stress.py | 88 ++++++++++++++++++++++------- 2 files changed, 136 insertions(+), 22 deletions(-) diff --git a/tests/perf/lib/broker.py b/tests/perf/lib/broker.py index 2f7f397..19817af 100644 --- a/tests/perf/lib/broker.py +++ b/tests/perf/lib/broker.py @@ -92,14 +92,46 @@ def _config_text(config: BrokerConfig, db_path: str) -> str: class BrokerProcess: - def __init__(self, config: BrokerConfig): + def __init__( + self, + config: BrokerConfig, + cgroup_memory: str | None = None, + cgroup_cpus: str | None = None, + ): + """Local broker process. + + cgroup_memory: MemoryMax for systemd-run --user --scope (e.g. "4294967296" + or "4G"); MemorySwapMax is pinned to 0 to match docker --memory-swap. + Requires user-scope cgroup delegation (memory controller). + cgroup_cpus: CPU affinity via taskset -c (e.g. "0-3"), equivalent to + docker --cpuset-cpus (same sched_setaffinity mechanism). + """ self.config = config + self.cgroup_memory = cgroup_memory + self.cgroup_cpus = cgroup_cpus self.proc: subprocess.Popen[str] | None = None self.broker_pid: int | None = None self.workdir: Path | None = None self.log_path: Path | None = None self.sampler: BrokerSampler | None = None + def _broker_cmd(self) -> list[str]: + cmd: list[str] = [] + if self.cgroup_memory: + cmd += [ + "systemd-run", + "--user", + "--scope", + "-p", + f"MemoryMax={self.cgroup_memory}", + "-p", + "MemorySwapMax=0", + ] + if self.cgroup_cpus: + cmd += ["taskset", "-c", self.cgroup_cpus] + cmd.append(str(BROKER_BIN)) + return cmd + def start(self) -> None: temp_dir = Path( tempfile.mkdtemp(prefix=f"pulsar-lite-{self.config.name}-", dir="/tmp") @@ -111,7 +143,7 @@ def start(self) -> None: self.log_path = temp_dir / "broker.log" log_file = self.log_path.open("w", encoding="utf-8") self.proc = subprocess.Popen( - [str(BROKER_BIN)], + self._broker_cmd(), cwd=temp_dir, stdout=log_file, stderr=subprocess.STDOUT, @@ -178,7 +210,7 @@ def restart(self, preserve_storage: bool = False) -> None: # Reopen log file for appending log_file = self.log_path.open("a", encoding="utf-8") self.proc = subprocess.Popen( - [str(BROKER_BIN)], + self._broker_cmd(), cwd=self.workdir, stdout=log_file, stderr=subprocess.STDOUT, @@ -211,6 +243,38 @@ def metrics(self) -> dict[str, float]: } +class ExternalBrokerProcess(BrokerProcess): + """Adapter for an already-running external broker (e.g. Apache Pulsar + standalone started manually with cgroup limits). + + No lifecycle management and no sampler: the harness only builds perf + commands against ``broker.config.port`` and merges empty metrics. + Scenarios that restart the broker (restart_replay, redelivery_unacked) + are not supported. + """ + + def __init__(self, config: BrokerConfig): + super().__init__(config) + self.log_path = None + self.sampler = None + + def start(self) -> None: + # External broker must already be listening; broker_pid stays None so + # the harness skips perf sampling. + self.broker_pid = None + self._wait_for_port() + + def stop(self, cleanup: bool = False) -> dict[str, float]: + return self.metrics() + + def restart(self, preserve_storage: bool = False) -> None: + raise NotImplementedError( + "external broker backend cannot restart the broker; " + "use scenarios that do not restart " + "(produce / consume_e2e / backlog_drain)" + ) + + class DockerBrokerProcess(BrokerProcess): def __init__( self, config: BrokerConfig, image_tag: str, cpuset_cpus: str, memory: str diff --git a/tests/perf/run_persistent_stress.py b/tests/perf/run_persistent_stress.py index 8114c63..b779c89 100755 --- a/tests/perf/run_persistent_stress.py +++ b/tests/perf/run_persistent_stress.py @@ -17,7 +17,7 @@ sys.path.insert(0, str(Path(__file__).resolve().parent)) from lib import ROOT -from lib.broker import BrokerConfig, BrokerProcess, DockerBrokerProcess +from lib.broker import BrokerConfig, BrokerProcess, DockerBrokerProcess, ExternalBrokerProcess from lib.docker_image import build_broker_image from lib.observability import PerfCollector from lib.parsing import parse_consumer_output, parse_producer_output @@ -160,7 +160,19 @@ class Scenario: ], feed_producer_args=["-time", "60", "-s", "1024", "-r", "999999", "-o", "1000"], ), - # Persistent-specific stress (3) + # Persistent-specific stress (4) + Scenario( + name="stress_persistent_consume_solo_10gb", + kind="backlog_drain", + broker="persistent_stress", + description="10GiB backlog（10485760×1KiB）→ 单 consumer 独立消费测速（produce 完成后才消费）", + producer_args=[ + "-m", "10485760", "-r", "999999", "-s", "1024", "-db", "-o", "1000", + ], + consumer_args=[ + "-m", "10485760", "-q", "1000", "-st", "Shared", "-sp", "Earliest", + ], + ), Scenario( name="stress_persistent_backlog_drain", kind="backlog_drain", @@ -321,7 +333,7 @@ def run_backlog_drain_scenario( ) print(" Producing backlog...") - producer_proc = run_sync(producer_cmd, producer_log, timeout=600.0) + producer_proc = run_sync(producer_cmd, producer_log, timeout=1800.0) if producer_proc.returncode != 0: raise RuntimeError(f"producer failed: {producer_log.read_text()[:500]}") @@ -341,7 +353,7 @@ def run_backlog_drain_scenario( print(" Draining backlog...") start = time.time() - consumer_proc = run_sync(consumer_cmd, consumer_log, timeout=600.0) + consumer_proc = run_sync(consumer_cmd, consumer_log, timeout=1800.0) drain_duration = time.time() - start if consumer_proc.returncode != 0: @@ -356,6 +368,7 @@ def run_backlog_drain_scenario( "broker": broker_metrics, "drain_duration_s": round(drain_duration, 2), "drain_throughput_msg_s": round(consumer_result["records"] / drain_duration, 2), + "drain_throughput_mbit_s": round(consumer_result["throughput_mbit_s"], 2), } @@ -567,10 +580,17 @@ def main(argv: list[str]) -> int: ) parser.add_argument( "--broker-backend", - choices=["local", "docker"], + choices=["local", "docker", "external"], default="local", help="Broker launch backend. local uses rust/target/release/pulsar-lite; " - "docker builds and runs a constrained broker container.", + "docker builds and runs a constrained broker container; " + "external targets an already-running broker via --external-url " + "(e.g. Apache Pulsar standalone); no broker lifecycle management.", + ) + parser.add_argument( + "--external-url", + default="pulsar://127.0.0.1:6650", + help="Service URL of the external broker for --broker-backend=external.", ) parser.add_argument( "--docker-cpuset", @@ -582,6 +602,19 @@ def main(argv: list[str]) -> int: default="4g", help="Memory limit passed to docker run --memory when --broker-backend=docker.", ) + parser.add_argument( + "--local-cgroup-memory", + default=None, + help="local backend: MemoryMax via systemd-run --user --scope " + "(e.g. 4294967296 or 4G; MemorySwapMax=0 pinned). Requires user-scope " + "cgroup delegation for the memory controller.", + ) + parser.add_argument( + "--local-cgroup-cpus", + default=None, + help="local backend: broker CPU affinity via taskset -c (e.g. 0-3), " + "equivalent to docker --cpuset-cpus.", + ) parser.add_argument( "--skip-docker-build", action="store_true", @@ -642,8 +675,23 @@ def main(argv: list[str]) -> int: cpuset_cpus=args.docker_cpuset, memory=args.docker_memory, ) + elif args.broker_backend == "external": + from urllib.parse import urlparse + + parsed = urlparse(args.external_url) + if parsed.scheme != "pulsar" or not parsed.hostname: + raise ValueError( + f"--external-url must be pulsar://host:port, got {args.external_url!r}" + ) + broker = ExternalBrokerProcess( + BrokerConfig("external", parsed.port or 6650, 0) + ) else: - broker = BrokerProcess(broker_config) + broker = BrokerProcess( + broker_config, + cgroup_memory=args.local_cgroup_memory, + cgroup_cpus=args.local_cgroup_cpus, + ) broker.start() try: @@ -738,18 +786,20 @@ def main(argv: list[str]) -> int: # Drop /tmp DB+entrylog after each scenario so storage does not accumulate. # restart_replay / redelivery manage their own in-scenario preserve restart; # once the scenario finishes we always wipe before the next one. - try: - broker.restart(preserve_storage=False) - print( - " storage cleaned for next scenario " - f"(workdir={broker.workdir})", - file=sys.stderr, - ) - except Exception as cleanup_err: - print( - f" WARNING: failed to reset broker storage: {cleanup_err}", - file=sys.stderr, - ) + # external backend owns its storage; nothing to reset. + if args.broker_backend != "external": + try: + broker.restart(preserve_storage=False) + print( + " storage cleaned for next scenario " + f"(workdir={broker.workdir})", + file=sys.stderr, + ) + except Exception as cleanup_err: + print( + f" WARNING: failed to reset broker storage: {cleanup_err}", + file=sys.stderr, + ) finally: broker.stop(cleanup=True) From 15fca7ff7631311f17886281245944cdcd6ae18c Mon Sep 17 00:00:00 2001 From: XTLine <2668543450@qq.com> Date: Fri, 7 Aug 2026 22:01:56 +0800 Subject: [PATCH 10/44] Monitor external broker via systemd unit and cgroup Sample cgroup memory stats (anon/file) when broker is managed by systemd. Reset sampler between scenarios so metrics cover only the scenario window. --- tests/perf/lib/broker.py | 102 +++++++++++++++++++++++++--- tests/perf/run_persistent_stress.py | 14 +++- 2 files changed, 104 insertions(+), 12 deletions(-) diff --git a/tests/perf/lib/broker.py b/tests/perf/lib/broker.py index 19817af..01b4757 100644 --- a/tests/perf/lib/broker.py +++ b/tests/perf/lib/broker.py @@ -11,6 +11,7 @@ import time from pathlib import Path import shutil +import sys from . import BASE_CONFIG, BROKER_BIN @@ -23,10 +24,11 @@ class BrokerConfig: class BrokerSampler(threading.Thread): - def __init__(self, pid: int, interval: float = 0.5): + def __init__(self, pid: int, interval: float = 0.5, cgroup_dir: str | None = None): super().__init__(daemon=True) self.pid = pid self.interval = interval + self.cgroup_dir = cgroup_dir self.samples: list[dict[str, float]] = [] self._stop_event = threading.Event() self._last_total = None @@ -36,6 +38,13 @@ def __init__(self, pid: int, interval: float = 0.5): def stop(self) -> None: self._stop_event.set() + def reset(self) -> None: + """Drop samples and restart the CPU delta baseline. Call before each + scenario so metrics() reflects only that scenario's window.""" + self.samples.clear() + self._last_total = None + self._last_time = None + def run(self) -> None: while not self._stop_event.is_set(): try: @@ -58,12 +67,30 @@ def run(self) -> None: rss_match = re.search(r"^VmRSS:\s+(\d+)\s+kB$", status_text, re.MULTILINE) rss_mb = (float(rss_match.group(1)) / 1024.0) if rss_match else 0.0 - self.samples.append({"cpu_pct": cpu_pct, "rss_mb": rss_mb}) + sample: dict[str, float] = {"cpu_pct": cpu_pct, "rss_mb": rss_mb} + if self.cgroup_dir: + try: + with open( + f"{self.cgroup_dir}/memory.stat", "r", encoding="utf-8" + ) as fh: + mem_stat = fh.read() + for line in mem_stat.splitlines(): + key, _, value = line.partition(" ") + if key == "anon": + sample["anon_mb"] = int(value) / 1048576.0 + elif key == "file": + sample["file_mb"] = int(value) / 1048576.0 + except (OSError, ValueError): + pass + self.samples.append(sample) time.sleep(self.interval) def write_csv(self, csv_path: Path) -> None: + fieldnames = ["cpu_pct", "rss_mb"] + if any("anon_mb" in sample for sample in self.samples): + fieldnames += ["anon_mb", "file_mb"] with csv_path.open("w", encoding="utf-8", newline="") as fh: - writer = csv.DictWriter(fh, fieldnames=["cpu_pct", "rss_mb"]) + writer = csv.DictWriter(fh, fieldnames=fieldnames) writer.writeheader() writer.writerows(self.samples) @@ -233,13 +260,19 @@ def metrics(self) -> dict[str, float]: "broker_avg_cpu_pct": 0.0, "broker_peak_cpu_pct": 0.0, "broker_peak_rss_mb": 0.0, + "broker_peak_anon_mb": 0.0, + "broker_peak_file_mb": 0.0, } cpu_values = [sample["cpu_pct"] for sample in samples[1:]] or [0.0] rss_values = [sample["rss_mb"] for sample in samples] + anon_values = [sample.get("anon_mb", 0.0) for sample in samples] + file_values = [sample.get("file_mb", 0.0) for sample in samples] return { "broker_avg_cpu_pct": round(sum(cpu_values) / len(cpu_values), 3), "broker_peak_cpu_pct": round(max(cpu_values), 3), "broker_peak_rss_mb": round(max(rss_values), 3), + "broker_peak_anon_mb": round(max(anon_values), 3), + "broker_peak_file_mb": round(max(file_values), 3), } @@ -247,25 +280,72 @@ class ExternalBrokerProcess(BrokerProcess): """Adapter for an already-running external broker (e.g. Apache Pulsar standalone started manually with cgroup limits). - No lifecycle management and no sampler: the harness only builds perf - commands against ``broker.config.port`` and merges empty metrics. - Scenarios that restart the broker (restart_replay, redelivery_unacked) - are not supported. + No lifecycle management: the harness only builds perf commands against + ``broker.config.port``. Scenarios that restart the broker + (restart_replay, redelivery_unacked) are not supported. + + If ``unit`` is given (systemd unit name, e.g. ``pulsar-standalone``), the + broker PID is resolved via ``systemctl show -p MainPID`` and CPU / + RSS / cgroup anon+file metrics are sampled like the local backend. """ - def __init__(self, config: BrokerConfig): + def __init__(self, config: BrokerConfig, unit: str | None = None): super().__init__(config) self.log_path = None + self.unit = unit self.sampler = None + def _resolve_pid(self) -> int | None: + try: + out = subprocess.run( + ["systemctl", "show", self.unit, "-p", "MainPID", "--value"], + capture_output=True, + text=True, + timeout=10, + ).stdout.strip() + except (OSError, subprocess.TimeoutExpired): + return None + if not out.isdigit(): + return None + pid = int(out) + if not os.path.exists(f"/proc/{pid}"): + return None + return pid + + def _resolve_cgroup_dir(self, pid: int) -> str | None: + try: + with open(f"/proc/{pid}/cgroup", "r", encoding="utf-8") as fh: + rel = fh.read().strip().split(":")[-1] + path = f"/sys/fs/cgroup{rel}" + if os.path.isfile(f"{path}/memory.stat"): + return path + except OSError: + pass + return None + def start(self) -> None: - # External broker must already be listening; broker_pid stays None so - # the harness skips perf sampling. + # External broker must already be listening. self.broker_pid = None self._wait_for_port() + if not self.unit: + return + pid = self._resolve_pid() + if pid is None: + print( + f" [warn] systemd unit '{self.unit}' not found; " + "broker CPU/memory metrics disabled", + file=sys.stderr, + ) + return + self.broker_pid = pid + self.sampler = BrokerSampler(pid, cgroup_dir=self._resolve_cgroup_dir(pid)) + self.sampler.start() def stop(self, cleanup: bool = False) -> dict[str, float]: - return self.metrics() + metrics = self.metrics() + if self.sampler: + self.sampler.stop() + return metrics def restart(self, preserve_storage: bool = False) -> None: raise NotImplementedError( diff --git a/tests/perf/run_persistent_stress.py b/tests/perf/run_persistent_stress.py index b779c89..9f662ef 100755 --- a/tests/perf/run_persistent_stress.py +++ b/tests/perf/run_persistent_stress.py @@ -592,6 +592,13 @@ def main(argv: list[str]) -> int: default="pulsar://127.0.0.1:6650", help="Service URL of the external broker for --broker-backend=external.", ) + parser.add_argument( + "--external-unit", + default=None, + help="systemd unit name of the external broker (e.g. pulsar-standalone or " + "pulsar-lite) to sample CPU/memory via 'systemctl show -p MainPID' " + "+ /proc. Leave unset to disable broker metrics for external backends.", + ) parser.add_argument( "--docker-cpuset", default="0-3", @@ -684,7 +691,8 @@ def main(argv: list[str]) -> int: f"--external-url must be pulsar://host:port, got {args.external_url!r}" ) broker = ExternalBrokerProcess( - BrokerConfig("external", parsed.port or 6650, 0) + BrokerConfig("external", parsed.port or 6650, 0), + unit=args.external_unit, ) else: broker = BrokerProcess( @@ -700,6 +708,10 @@ def main(argv: list[str]) -> int: print(f"\n[{scenario.name}] {scenario.description}") scenario_dir = run_artifacts / scenario.name scenario_dir.mkdir(parents=True, exist_ok=True) + # Reset the broker sampler so metrics() / timeseries CSV cover + # only this scenario's window, not the whole suite run. + if broker.sampler: + broker.sampler.reset() # Start perf recording (must be after restart to capture the new PID) perf_data_path = scenario_dir / "perf.data" perf_collector: PerfCollector | None = None From 1834646388f5c9ecdf18567e3372c74d365b7e08 Mon Sep 17 00:00:00 2001 From: XTLine <2668543450@qq.com> Date: Thu, 13 Aug 2026 16:33:02 +0800 Subject: [PATCH 11/44] Update perf stress scripts for external broker runs --- tests/perf/run_non_persistent_e2e_matrix.py | 50 +++++++++++++++++-- tests/perf/run_non_persistent_stress.py | 55 +++++++++++++++++---- tests/perf/run_persistent_e2e_matrix.py | 42 ++++++++++++++-- tests/perf/run_persistent_stress.py | 6 +-- 4 files changed, 135 insertions(+), 18 deletions(-) diff --git a/tests/perf/run_non_persistent_e2e_matrix.py b/tests/perf/run_non_persistent_e2e_matrix.py index e9dcbde..8a3eb33 100644 --- a/tests/perf/run_non_persistent_e2e_matrix.py +++ b/tests/perf/run_non_persistent_e2e_matrix.py @@ -10,7 +10,7 @@ sys.path.insert(0, str(Path(__file__).resolve().parent)) from lib import ROOT -from lib.broker import BrokerConfig, BrokerProcess +from lib.broker import BrokerConfig, BrokerProcess, ExternalBrokerProcess from lib.parsing import parse_consumer_output, parse_producer_output from lib.perf_cmd import ( ensure_prereqs, @@ -335,6 +335,33 @@ def scenario_topic(run_id: str, scenario: Scenario) -> str: def main() -> int: + import argparse + + parser = argparse.ArgumentParser( + description="Run non-persistent e2e matrix scenarios for pulsar-lite" + ) + parser.add_argument( + "--broker-backend", + choices=["local", "external"], + default="local", + help="local starts rust/target/release/pulsar-lite per broker profile; " + "external targets an already-running broker via --external-url " + "(e.g. Apache Pulsar standalone); no broker lifecycle management.", + ) + parser.add_argument( + "--external-url", + default="pulsar://127.0.0.1:6650", + help="Service URL of the external broker for --broker-backend=external.", + ) + parser.add_argument( + "--external-unit", + default=None, + help="systemd unit name of the external broker (e.g. pulsar-standalone or " + "pulsar-lite) to sample CPU/memory via 'systemctl show -p MainPID' " + "+ /proc. Leave unset to disable broker metrics for external backends.", + ) + args = parser.parse_args() + ensure_prereqs() ARTIFACTS_DIR.mkdir(parents=True, exist_ok=True) run_id = time.strftime("%Y%m%d-%H%M%S") @@ -346,13 +373,30 @@ def main() -> int: broker_metrics_by_name: dict[str, dict[str, float]] = {} for broker_name, broker_cfg in BROKERS.items(): - broker = BrokerProcess(broker_cfg) + if args.broker_backend == "external": + from urllib.parse import urlparse + + parsed = urlparse(args.external_url) + if parsed.scheme != "pulsar" or not parsed.hostname: + raise ValueError( + f"--external-url must be pulsar://host:port, got {args.external_url!r}" + ) + broker = ExternalBrokerProcess( + BrokerConfig("external", parsed.port or 6650, 0), + unit=args.external_unit, + ) + else: + broker = BrokerProcess(broker_cfg) print( f"==> starting broker {broker_name} on {broker_cfg.port} (default_partitions={broker_cfg.default_partitions})", flush=True, ) broker.start() - service_url = f"pulsar://127.0.0.1:{broker_cfg.port}" + service_url = ( + args.external_url + if args.broker_backend == "external" + else f"pulsar://127.0.0.1:{broker_cfg.port}" + ) try: for scenario in [s for s in SCENARIOS if s.broker == broker_name]: topic = scenario_topic(run_id, scenario) diff --git a/tests/perf/run_non_persistent_stress.py b/tests/perf/run_non_persistent_stress.py index ea2bf1f..f5afdc7 100644 --- a/tests/perf/run_non_persistent_stress.py +++ b/tests/perf/run_non_persistent_stress.py @@ -23,7 +23,12 @@ # --- lib imports (run from repo root or with sys.path adjusted) --- sys.path.insert(0, str(Path(__file__).resolve().parent)) from lib import ROOT -from lib.broker import BrokerConfig, BrokerProcess, DockerBrokerProcess +from lib.broker import ( + BrokerConfig, + BrokerProcess, + DockerBrokerProcess, + ExternalBrokerProcess, +) from lib.docker_image import build_broker_image from lib.observability import PerfCollector from lib.parsing import parse_consumer_output, parse_producer_output @@ -204,9 +209,19 @@ def build_arg_parser() -> argparse.ArgumentParser: ) parser.add_argument( "--broker-backend", - choices=["local", "docker"], + choices=["local", "docker", "external"], default="local", - help="Broker launch backend. local uses rust/target/release/pulsar-lite; docker builds and runs a constrained broker container.", + help="Broker launch backend. local uses rust/target/release/pulsar-lite; docker builds and runs a constrained broker container; external targets an already-running broker via --external-url.", + ) + parser.add_argument( + "--external-url", + default="pulsar://127.0.0.1:6650", + help="Service URL of the external broker for --broker-backend=external.", + ) + parser.add_argument( + "--external-unit", + default=None, + help="systemd unit name of the external broker (e.g. pulsar-standalone or pulsar-lite) to sample CPU/memory via 'systemctl show -p MainPID' + /proc.", ) parser.add_argument( @@ -256,7 +271,9 @@ def _topic_for(run_id: str, scenario: StressScenario) -> str: return f"non-persistent://public/default/{run_id}-{scenario.name}" -def _service_url_for(scenario: StressScenario) -> str: +def _service_url_for(scenario: StressScenario, external_url: str | None = None) -> str: + if external_url: + return external_url cfg = BROKERS[scenario.broker] return f"pulsar://127.0.0.1:{cfg.port}" @@ -267,10 +284,11 @@ def _run_produce_scenario( scenario_dir: Path, broker_proc: BrokerProcess, perf_collector: PerfCollector | None, + external_url: str | None = None, ) -> dict: """Execute a single producer-only stress scenario.""" topic = _topic_for(run_id, scenario) - service_url = _service_url_for(scenario) + service_url = _service_url_for(scenario, external_url) timeout = scenario.estimated_duration + 120 histogram_path = scenario_dir / "producer_histogram.log" @@ -320,10 +338,11 @@ def _run_consume_e2e_scenario( scenario_dir: Path, broker_proc: BrokerProcess, perf_collector: PerfCollector | None, + external_url: str | None = None, ) -> dict: """Execute a single consumer e2e stress scenario (feed-producer + consumer).""" topic = _topic_for(run_id, scenario) - service_url = _service_url_for(scenario) + service_url = _service_url_for(scenario, external_url) timeout = scenario.estimated_duration + 120 consumer_args = scenario.consumer_args or [] @@ -461,6 +480,18 @@ def main(argv: list[str] | None = None) -> None: cpuset_cpus=args.docker_cpuset, memory=args.docker_memory, ) + elif args.broker_backend == "external": + from urllib.parse import urlparse + + parsed = urlparse(args.external_url) + if parsed.scheme != "pulsar" or not parsed.hostname: + raise ValueError( + f"--external-url must be pulsar://host:port, got {args.external_url!r}" + ) + bp = ExternalBrokerProcess( + BrokerConfig("external", parsed.port or 6650, 0), + unit=args.external_unit, + ) else: bp = BrokerProcess(cfg) bp.start() @@ -480,8 +511,9 @@ def main(argv: list[str] | None = None) -> None: scenario_dir.mkdir(parents=True, exist_ok=True) # Restart broker between scenarios to clear residual topics/subscriptions - print(f" restarting broker [{scenario.broker}] ...", file=sys.stderr) - broker_proc.restart() + if args.broker_backend != "external": + print(f" restarting broker [{scenario.broker}] ...", file=sys.stderr) + broker_proc.restart() # Start perf recording (must be after restart to capture the new PID) perf_data_path = scenario_dir / "perf.data" @@ -504,6 +536,7 @@ def main(argv: list[str] | None = None) -> None: scenario_dir, broker_proc, perf_collector, + args.external_url if args.broker_backend == "external" else None, ) elif scenario.kind == "consume_e2e": result = _run_consume_e2e_scenario( @@ -512,6 +545,7 @@ def main(argv: list[str] | None = None) -> None: scenario_dir, broker_proc, perf_collector, + args.external_url if args.broker_backend == "external" else None, ) else: print(f" UNKNOWN kind: {scenario.kind}, skipping", file=sys.stderr) @@ -524,7 +558,10 @@ def main(argv: list[str] | None = None) -> None: "name": scenario.name, "kind": scenario.kind, "broker_profile": scenario.broker, - "service_url": _service_url_for(scenario), + "service_url": _service_url_for( + scenario, + args.external_url if args.broker_backend == "external" else None, + ), "description": scenario.description, "topic": _topic_for(run_id, scenario), "started_at": datetime.now(timezone.utc).isoformat(), diff --git a/tests/perf/run_persistent_e2e_matrix.py b/tests/perf/run_persistent_e2e_matrix.py index 03a7335..a9985b0 100755 --- a/tests/perf/run_persistent_e2e_matrix.py +++ b/tests/perf/run_persistent_e2e_matrix.py @@ -17,7 +17,12 @@ sys.path.insert(0, str(Path(__file__).resolve().parent)) from lib import ROOT -from lib.broker import BrokerConfig, BrokerProcess, DockerBrokerProcess +from lib.broker import ( + BrokerConfig, + BrokerProcess, + DockerBrokerProcess, + ExternalBrokerProcess, +) from lib.docker_image import build_broker_image from lib.parsing import parse_consumer_output, parse_producer_output from lib.perf_cmd import ( @@ -360,6 +365,11 @@ def run_restart_smoke_scenario( run_dir: Path, ) -> dict[str, Any]: """Run restart scenario: produce → restart → consume.""" + if isinstance(broker, ExternalBrokerProcess): + raise RuntimeError( + "external broker backend cannot restart the broker; " + "use scenarios that do not restart (produce / consume_e2e)" + ) topic = f"persistent://public/default/test-{uuid.uuid4().hex[:8]}" # Step 1: Produce messages @@ -449,10 +459,24 @@ def main(argv: list[str]) -> int: ) parser.add_argument( "--broker-backend", - choices=["local","docker"], + choices=["local", "docker", "external"], default="local", help="Broker launch backend. local user rust/target/release/pulsar-lite; " - "docker builds and runs a constrained broker container.", + "docker builds and runs a constrained broker container; " + "external targets an already-running broker via --external-url " + "(e.g. Apache Pulsar standalone); no broker lifecycle management.", + ) + parser.add_argument( + "--external-url", + default="pulsar://127.0.0.1:6650", + help="Service URL of the external broker for --broker-backend=external.", + ) + parser.add_argument( + "--external-unit", + default=None, + help="systemd unit name of the external broker (e.g. pulsar-standalone or " + "pulsar-lite) to sample CPU/memory via 'systemctl show -p MainPID' " + "+ /proc. Leave unset to disable broker metrics for external backends.", ) parser.add_argument( "--docker-cpuset", @@ -516,6 +540,18 @@ def main(argv: list[str]) -> int: cpuset_cpus=args.docker_cpuset, memory=args.docker_memory, ) + elif args.broker_backend == "external": + from urllib.parse import urlparse + + parsed = urlparse(args.external_url) + if parsed.scheme != "pulsar" or not parsed.hostname: + raise ValueError( + f"--external-url must be pulsar://host:port, got {args.external_url!r}" + ) + broker = ExternalBrokerProcess( + BrokerConfig("external", parsed.port or 6650, 0), + unit=args.external_unit, + ) else: broker = BrokerProcess(broker_config) diff --git a/tests/perf/run_persistent_stress.py b/tests/perf/run_persistent_stress.py index 9f662ef..65cb65a 100755 --- a/tests/perf/run_persistent_stress.py +++ b/tests/perf/run_persistent_stress.py @@ -113,10 +113,10 @@ class Scenario: name="stress_persistent_consume_shared_high_fanout", kind="consume_e2e", broker="persistent_stress", - description="16 consumers 高扇出 60s", + description="16 consumers 高扇出 6s [TEMP]", consumer_args=[ "-time", - "60", + "6", "-q", "1000", "-st", @@ -124,7 +124,7 @@ class Scenario: "-n", "16", ], - feed_producer_args=["-time", "60", "-s", "1024", "-r", "999999", "-o", "1000"], + feed_producer_args=["-time", "6", "-s", "1024", "-r", "999999", "-o", "1000"], ), Scenario( name="stress_persistent_consume_multi_subscription_fanout", From 7137100954f3748a82f1824a92e744b8cf303dcf Mon Sep 17 00:00:00 2001 From: XTLine <2668543450@qq.com> Date: Tue, 4 Aug 2026 10:43:33 +0800 Subject: [PATCH 12/44] Add TODO comment in read_position --- rust/src/broker/dispatcher/read_position.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/rust/src/broker/dispatcher/read_position.rs b/rust/src/broker/dispatcher/read_position.rs index 01e4652..be9c155 100644 --- a/rust/src/broker/dispatcher/read_position.rs +++ b/rust/src/broker/dispatcher/read_position.rs @@ -49,6 +49,7 @@ pub async fn next_unacked_candidate( let Some((entry, next_position, already_acked)) = ({ let guard = storage.lock().await; + // TODO: let batch = guard.read_entries_from(topic, &pos, 1)?; if let Some(entry) = batch.into_iter().next() { From 55906cdae95134071361b4d14112dcc6b3e5886c Mon Sep 17 00:00:00 2001 From: XTLine <2668543450@qq.com> Date: Thu, 6 Aug 2026 11:00:39 +0800 Subject: [PATCH 13/44] fix(broker): bound shared dispatcher memory with unacked gate and merge-trigger - Unacked budget gate (max_unacked_messages=50000, aligned with Pulsar maxUnackedMessagesPerConsumer): dispatch pauses when a consumer's delivered-but-unacked set exceeds the budget, cutting the fuel of the redelivery/Flow feedback loop. - send-failure branches switch continue->break: a failed send no longer re-pops the same entry within the same pass, eliminating the read-fail-restore spin (150k entrylog reads/s of allocations). - Merge-trigger dispatch (Pulsar shouldRescheduleRead pattern): triggers arriving while a loop runs only set a flag; the pass checks it after releasing the guard and runs one more pass. No dropped triggers, no fixed time constant, empty passes never read storage. - Add Consumer::pending_acks_len accessor + unit test (shared_dispatch_pauses_when_unacked_budget_exceeded). --- rust/src/broker/dispatcher/shared.rs | 142 +++++++++++++++++++++++---- rust/src/broker/service/consumer.rs | 5 + 2 files changed, 129 insertions(+), 18 deletions(-) diff --git a/rust/src/broker/dispatcher/shared.rs b/rust/src/broker/dispatcher/shared.rs index 90c2b7c..1047c12 100644 --- a/rust/src/broker/dispatcher/shared.rs +++ b/rust/src/broker/dispatcher/shared.rs @@ -33,9 +33,19 @@ pub struct SharedDispatcher { /// Total available permits across all consumers (atomic for thread safety) total_available_permits: AtomicU32, - /// Flag to prevent reentrant dispatching + /// Flag to prevent reentrant dispatching (Pulsar readMoreEntriesInProgress). dispatch_in_progress: AtomicBool, + /// Set by triggers that arrive while a dispatch loop is already running; + /// the loop checks this at the end of each pass and runs one more pass if + /// set (Pulsar shouldRescheduleRead). No triggers are lost and no fixed + /// time constants are involved. + should_reschedule: AtomicBool, + + /// Per-consumer delivered-but-unacked budget. Dispatch pauses when a + /// consumer exceeds it (aligns with Pulsar maxUnackedMessagesPerConsumer). + max_unacked_messages: usize, + // Pending messages to redeliver. Shared does not use sticky hash blocking. redelivery_controller: Arc>, @@ -43,6 +53,9 @@ pub struct SharedDispatcher { read_position: RwLock>, } +/// Aligns with Apache Pulsar broker.conf maxUnackedMessagesPerConsumer default. +const DEFAULT_MAX_UNACKED_MESSAGES_PER_CONSUMER: usize = 50_000; + impl SharedDispatcher { /// Create a new SharedDispatcher pub fn new() -> Self { @@ -52,6 +65,8 @@ impl SharedDispatcher { round_robin_index: AtomicUsize::new(0), total_available_permits: AtomicU32::new(0), dispatch_in_progress: AtomicBool::new(false), + should_reschedule: AtomicBool::new(false), + max_unacked_messages: DEFAULT_MAX_UNACKED_MESSAGES_PER_CONSUMER, redelivery_controller: Arc::new(RwLock::new(RedeliveryController::new(false))), read_position: RwLock::new(None), } @@ -260,6 +275,25 @@ impl SharedDispatcher { let consumer_id = consumer.consumer_id; + // Unacked budget gate: pause dispatch for consumers whose + // delivered-but-unacked set exceeds the budget (Pulsar + // maxUnackedMessagesPerConsumer semantics). Without this, the + // redelivery/Flow feedback loop grows permits and reads unboundedly. + // + // TODO(dispatchers): extract this gate (plus the merge-trigger in + // dispatch_messages and the send-failure break) into a shared + // module so Exclusive/Failover/KeyShared dispatchers reuse the + // same bounded-dispatch logic instead of duplicating it. + if consumer.pending_acks_len().await >= self.max_unacked_messages { + log::warn!( + "Consumer {} unacked budget exceeded ({} >= {}), pausing dispatch", + consumer_id, + consumer.pending_acks_len().await, + self.max_unacked_messages + ); + break; + } + // Use one permit if !consumer.use_permit().await { log::warn!("Consumer {} permits exhausted during dispatch", consumer_id); @@ -325,6 +359,10 @@ impl SharedDispatcher { }); consumer.add_permits(1).await; self.total_available_permits.fetch_add(1, Ordering::Relaxed); + // Send failed: connection/write path is saturated. + // Stop this pass instead of immediately re-reading the + // same entry (previous `continue` caused the spin). + break; } } else { // Message no longer exists (may have been deleted), restore permit @@ -370,6 +408,10 @@ impl SharedDispatcher { commit_read_position(&self.read_position, candidate.next_position); consumer.add_permits(1).await; self.total_available_permits.fetch_add(1, Ordering::Relaxed); + // Send failed: connection/write path is saturated. Stop + // this pass rather than queueing the whole batch for + // redelivery and re-reading in a tight loop. + break; } } else { // No more messages, restore permit @@ -626,34 +668,54 @@ impl Dispatcher for SharedDispatcher { topic: String, subscription: String, ) -> Result<(), Box> { - // Prevent reentrant dispatching + // Merge-trigger (Pulsar): only one dispatch loop runs at a time. + // Concurrent triggers set the reschedule flag instead of starting a + // new loop; the running loop checks the flag at the end of each pass + // and runs one more pass. No triggers are lost, no fixed time + // constants, and empty passes never read storage. if self.dispatch_in_progress.swap(true, Ordering::Relaxed) { - log::debug!("Dispatch already in progress, skipping"); + log::debug!("Dispatch already in progress, scheduling reschedule"); + self.should_reschedule.store(true, Ordering::Relaxed); return Ok(()); } + let result = async { - // Keep dispatching batches while we still make progress and have permits. - // read/send one batch, then readMoreEntries again loop { - let dispatcher = self - .dispatch_messages_batch(storage.clone(), topic.clone(), subscription.clone()) - .await?; - if dispatcher == 0 { - break; - } - if self.total_available_permits.load(Ordering::Relaxed) == 0 { - break; + // One pass: dispatch batches while we make progress and have permits. + loop { + let dispatcher = self + .dispatch_messages_batch(storage.clone(), topic.clone(), subscription.clone()) + .await?; + if dispatcher == 0 { + break; + } + if self.total_available_permits.load(Ordering::Relaxed) == 0 { + break; + } + + // yield so other tasks can run on large backlogs. + tokio::task::yield_now().await; } - // yield so other tasks can run on large backlogs. - tokio::task::yield_now().await; + // Pulsar ordering: release the loop guard, then check whether + // triggers arrived during the pass. Re-acquire for the + // follow-up pass; if another trigger already grabbed the guard, + // that loop owns the reschedule handling. + self.dispatch_in_progress.store(false, Ordering::Relaxed); + if !self.should_reschedule.swap(false, Ordering::Relaxed) { + return Ok(()); + } + if self.dispatch_in_progress.swap(true, Ordering::Relaxed) { + return Ok(()); + } } - Ok(()) } .await; - // Reset flag - self.dispatch_in_progress.store(false, Ordering::Relaxed); + if result.is_err() { + // Error propagated out of the pass loop with the guard still held. + self.dispatch_in_progress.store(false, Ordering::Relaxed); + } result } @@ -698,6 +760,50 @@ mod tests { )) } + #[tokio::test] + async fn shared_dispatch_pauses_when_unacked_budget_exceeded() { + let storage = create_test_storage(); + let subscription = create_test_subscription(storage.clone()); + let (tx, mut rx) = mpsc::channel(8); + let consumer = Arc::new(Consumer::new( + 1, + "consumer-1".to_string(), + subscription, + "conn-1".to_string(), + tx, + 0, + )); + consumer.add_permits(10).await; + + let mut dispatcher = SharedDispatcher::new(); + dispatcher.max_unacked_messages = 2; + dispatcher.add_consumer(consumer.clone()).unwrap(); + dispatcher.consumer_flow(1, 10); + + // Fill the delivered-but-unacked set past the budget (3 > 2). + for i in 0..3u64 { + consumer + .track_message_dispatched(&MessageId { ledger: 0, entry: i, partition: -1 }, 0) + .await; + } + + let result = dispatcher + .dispatch_messages( + storage.clone(), + "persistent://public/default/test-topic".to_string(), + "test-sub".to_string(), + ) + .await; + assert!(result.is_ok(), "dispatch should not error when budget exceeded"); + + // Budget exceeded -> dispatch paused: nothing should be delivered. + let delivered = tokio::time::timeout(std::time::Duration::from_millis(200), rx.recv()).await; + assert!( + delivered.is_err(), + "dispatch must pause when unacked budget is exceeded" + ); + } + #[tokio::test] async fn priority_dispatch_prefers_higher_priority_consumers() { let storage = create_test_storage(); diff --git a/rust/src/broker/service/consumer.rs b/rust/src/broker/service/consumer.rs index f63667f..081baea 100644 --- a/rust/src/broker/service/consumer.rs +++ b/rust/src/broker/service/consumer.rs @@ -208,6 +208,11 @@ impl Consumer { self.available_permits.fetch_add(permits, Ordering::Relaxed); } + /// Instrumentation/gate: dispatched-but-unacked message count for this consumer. + pub(crate) async fn pending_acks_len(&self) -> usize { + self.pending_acks.len().await + } + /// Use one permit when dispatching a message pub async fn use_permit(&self) -> bool { let mut current = self.available_permits.load(Ordering::Relaxed); From 1534a36d91f79de0ff46ccf47288675fe0e3b2e2 Mon Sep 17 00:00:00 2001 From: XTLine <2668543450@qq.com> Date: Thu, 6 Aug 2026 11:00:50 +0800 Subject: [PATCH 14/44] perf(broker): switch to jemalloc to avoid glibc arena memory retention glibc malloc keeps freed memory in per-thread arenas and only returns the top chunk to the OS, so high-allocation-churn workloads (300k msg/s x ~5 allocs/message) pin RSS at the peak footprint. Under a 4G cgroup limit the broker OOM'd with 4.16GB anon while every data structure was empty (verified via depth instrumentation). jemalloc uses size-class buckets plus dirty-page decay: idle pages are munmapped back to the OS within ~10s, so RSS tracks live data. Same workload drops anon from 4.16GB to ~88MB and the full 10-scenario suite passes under 4G. - #[global_allocator] Jemalloc in main.rs - unprefixed_malloc_on_supported_platforms: RocksDB and other C libs allocate through jemalloc too - tikv-jemalloc-sys profiling feature enabled for MALLOC_CONF-based heap profiling (diagnostic runs only) --- rust/Cargo.lock | 22 ++++++++++++++++++++++ rust/Cargo.toml | 6 ++++++ rust/src/main.rs | 4 ++++ 3 files changed, 32 insertions(+) diff --git a/rust/Cargo.lock b/rust/Cargo.lock index 919c3d2..0888cbc 100644 --- a/rust/Cargo.lock +++ b/rust/Cargo.lock @@ -836,6 +836,8 @@ dependencies = [ "serde_json", "tempfile", "thiserror", + "tikv-jemalloc-sys", + "tikv-jemallocator", "tokio", "tokio-stream", "tokio-util", @@ -1155,6 +1157,26 @@ dependencies = [ "syn", ] +[[package]] +name = "tikv-jemalloc-sys" +version = "0.6.1+5.3.0-1-ge13ca993e8ccb9ba9847cc330696e02839f328f7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd8aa5b2ab86a2cefa406d889139c162cbb230092f7d1d7cbc1716405d852a3b" +dependencies = [ + "cc", + "libc", +] + +[[package]] +name = "tikv-jemallocator" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0359b4327f954e0567e69fb191cf1436617748813819c94b8cd4a431422d053a" +dependencies = [ + "libc", + "tikv-jemalloc-sys", +] + [[package]] name = "tokio" version = "1.52.3" diff --git a/rust/Cargo.toml b/rust/Cargo.toml index 75a1382..ba7280c 100644 --- a/rust/Cargo.toml +++ b/rust/Cargo.toml @@ -21,6 +21,8 @@ thiserror = "1.0" tokio = { version = "1.35", features = ["full"] } anyhow = "1.0" log = "0.4" +tikv-jemallocator = { version = "0.6", features = ["unprefixed_malloc_on_supported_platforms"] } +tikv-jemalloc-sys = { version = "0.6", features = ["profiling", "unprefixed_malloc_on_supported_platforms"] } [package] name = "pulsar-lite" @@ -82,6 +84,10 @@ uuid = { version = "1.6", features = ["v4"] } tokio-stream = "0.1" futures = "0.3" +# 内存分配器：jemalloc（缓解 glibc arena 内存滞留导致的 RSS 高水位） +tikv-jemallocator.workspace = true +tikv-jemalloc-sys.workspace = true + [build-dependencies] prost-build = "0.13" diff --git a/rust/src/main.rs b/rust/src/main.rs index 3a62f2d..1b20af1 100644 --- a/rust/src/main.rs +++ b/rust/src/main.rs @@ -19,6 +19,10 @@ use tokio::net::TcpListener; use tokio::sync::{Mutex, RwLock}; use tokio::time::Duration; use tokio_util::codec::Framed; +use tikv_jemallocator::Jemalloc; + +#[global_allocator] +static GLOBAL: Jemalloc = Jemalloc; #[derive(Debug, Parser)] #[command(name = "pulsar-lite")] From bd3bf6def56282ade01ec8f1a110ae39d6095827 Mon Sep 17 00:00:00 2001 From: XTLine <2668543450@qq.com> Date: Fri, 21 Aug 2026 16:35:03 +0800 Subject: [PATCH 15/44] style: cargo fmt --- rust/src/broker/dispatcher/read_position.rs | 2 +- rust/src/broker/dispatcher/shared.rs | 23 +++++++++++++++++---- rust/src/main.rs | 2 +- 3 files changed, 21 insertions(+), 6 deletions(-) diff --git a/rust/src/broker/dispatcher/read_position.rs b/rust/src/broker/dispatcher/read_position.rs index be9c155..58d87f0 100644 --- a/rust/src/broker/dispatcher/read_position.rs +++ b/rust/src/broker/dispatcher/read_position.rs @@ -49,7 +49,7 @@ pub async fn next_unacked_candidate( let Some((entry, next_position, already_acked)) = ({ let guard = storage.lock().await; - // TODO: + // TODO: let batch = guard.read_entries_from(topic, &pos, 1)?; if let Some(entry) = batch.into_iter().next() { diff --git a/rust/src/broker/dispatcher/shared.rs b/rust/src/broker/dispatcher/shared.rs index 1047c12..fae8baf 100644 --- a/rust/src/broker/dispatcher/shared.rs +++ b/rust/src/broker/dispatcher/shared.rs @@ -684,7 +684,11 @@ impl Dispatcher for SharedDispatcher { // One pass: dispatch batches while we make progress and have permits. loop { let dispatcher = self - .dispatch_messages_batch(storage.clone(), topic.clone(), subscription.clone()) + .dispatch_messages_batch( + storage.clone(), + topic.clone(), + subscription.clone(), + ) .await?; if dispatcher == 0 { break; @@ -783,7 +787,14 @@ mod tests { // Fill the delivered-but-unacked set past the budget (3 > 2). for i in 0..3u64 { consumer - .track_message_dispatched(&MessageId { ledger: 0, entry: i, partition: -1 }, 0) + .track_message_dispatched( + &MessageId { + ledger: 0, + entry: i, + partition: -1, + }, + 0, + ) .await; } @@ -794,10 +805,14 @@ mod tests { "test-sub".to_string(), ) .await; - assert!(result.is_ok(), "dispatch should not error when budget exceeded"); + assert!( + result.is_ok(), + "dispatch should not error when budget exceeded" + ); // Budget exceeded -> dispatch paused: nothing should be delivered. - let delivered = tokio::time::timeout(std::time::Duration::from_millis(200), rx.recv()).await; + let delivered = + tokio::time::timeout(std::time::Duration::from_millis(200), rx.recv()).await; assert!( delivered.is_err(), "dispatch must pause when unacked budget is exceeded" diff --git a/rust/src/main.rs b/rust/src/main.rs index 1b20af1..478832d 100644 --- a/rust/src/main.rs +++ b/rust/src/main.rs @@ -15,11 +15,11 @@ use pulsar_lite::storage::Storage; use std::net::{IpAddr, Ipv4Addr, SocketAddr}; use std::path::PathBuf; use std::sync::Arc; +use tikv_jemallocator::Jemalloc; use tokio::net::TcpListener; use tokio::sync::{Mutex, RwLock}; use tokio::time::Duration; use tokio_util::codec::Framed; -use tikv_jemallocator::Jemalloc; #[global_allocator] static GLOBAL: Jemalloc = Jemalloc; From 180470ec7a7a509c56f68c29ffa911a7a3f25e19 Mon Sep 17 00:00:00 2001 From: XTLine <2668543450@qq.com> Date: Sat, 8 Aug 2026 21:57:02 +0800 Subject: [PATCH 16/44] feat(proto): extract pulsar binary protocol into pulsar-lite-proto crate --- rust/Cargo.lock | 14 ++ rust/Cargo.toml | 2 + rust/proto/Cargo.toml | 14 ++ rust/{ => proto}/build.rs | 2 +- rust/proto/src/codec.rs | 479 ++++++++++++++++++++++++++++++++++++++ rust/proto/src/command.rs | 335 ++++++++++++++++++++++++++ rust/proto/src/lib.rs | 6 + 7 files changed, 851 insertions(+), 1 deletion(-) create mode 100644 rust/proto/Cargo.toml rename rust/{ => proto}/build.rs (84%) create mode 100644 rust/proto/src/codec.rs create mode 100644 rust/proto/src/command.rs create mode 100644 rust/proto/src/lib.rs diff --git a/rust/Cargo.lock b/rust/Cargo.lock index 0888cbc..848eabe 100644 --- a/rust/Cargo.lock +++ b/rust/Cargo.lock @@ -827,6 +827,7 @@ dependencies = [ "prost", "prost-build", "prost-types", + "pulsar-lite-proto", "pulsar-lite-storage", "pulsar-lite-storage-managed-ledger", "pulsar-lite-storage-managed-ledger-rocksdb", @@ -845,12 +846,25 @@ dependencies = [ "uuid", ] +[[package]] +name = "pulsar-lite-proto" +version = "0.1.0" +dependencies = [ + "bytes", + "prost", + "prost-build", + "prost-types", + "tokio-util", +] + [[package]] name = "pulsar-lite-storage" version = "0.1.0" dependencies = [ "anyhow", "log", + "prost", + "pulsar-lite-proto", "pulsar-lite-storage-managed-ledger", "pulsar-lite-storage-managed-ledger-rocksdb", "pulsar-lite-storage-metadata", diff --git a/rust/Cargo.toml b/rust/Cargo.toml index ba7280c..379b921 100644 --- a/rust/Cargo.toml +++ b/rust/Cargo.toml @@ -6,6 +6,7 @@ members = [ "storage/resources", "storage/managed-ledger", "storage/managed-ledger-rocksdb", + "proto", ] resolver = "2" @@ -47,6 +48,7 @@ pulsar-lite-storage-metadata = { path = "storage/metadata" } pulsar-lite-storage-resources = { path ="storage/resources" } pulsar-lite-storage-managed-ledger = { path = "storage/managed-ledger" } pulsar-lite-storage-managed-ledger-rocksdb = { path = "storage/managed-ledger-rocksdb", optional = true } +pulsar-lite-proto = { path = "proto" } # 异步运行时 tokio = { version = "1.35", features = ["full"] } diff --git a/rust/proto/Cargo.toml b/rust/proto/Cargo.toml new file mode 100644 index 0000000..7e68933 --- /dev/null +++ b/rust/proto/Cargo.toml @@ -0,0 +1,14 @@ +[package] +name = "pulsar-lite-proto" +version = "0.1.0" +edition = "2021" + +[dependencies] +# Protobuf +prost = "0.13" +prost-types = "0.13" +bytes = "1.5" +tokio-util = { version = "0.7", features = ["codec"] } + +[build-dependencies] +prost-build = "0.13" \ No newline at end of file diff --git a/rust/build.rs b/rust/proto/build.rs similarity index 84% rename from rust/build.rs rename to rust/proto/build.rs index f1029b9..8e4ab5b 100644 --- a/rust/build.rs +++ b/rust/proto/build.rs @@ -5,7 +5,7 @@ fn main() -> Result<(), Box> { .out_dir(&out_dir) .protoc_arg("--experimental_allow_proto3_optional") .compile_protos( - &["proto/PulsarApi.proto", "proto/MLDataFormats.proto"], + &["PulsarApi.proto", "MLDataFormats.proto"], &["proto"], )?; diff --git a/rust/proto/src/codec.rs b/rust/proto/src/codec.rs new file mode 100644 index 0000000..a081ea1 --- /dev/null +++ b/rust/proto/src/codec.rs @@ -0,0 +1,479 @@ +/* + * Pulsar Binary Protocol Implementation + * + * Wire format: + * [TOTAL_SIZE (4B)] [CMD_SIZE (4B)] [CMD (Protobuf)] [MAGIC_NUMBER (2B, optional)] [CHECKSUM (4B, optional)] [METADATA_SIZE (4B)] [METADATA (Protobuf)] [PAYLOAD] + * + * Frame decoder: LengthFieldBasedFrameDecoder(maxMessageSize + 10KB, 0, 4, 0, 4) + */ + +use bytes::{Buf, BufMut, Bytes, BytesMut}; +use prost::Message; +use std::io::{self, Cursor}; +use tokio_util::codec::{Decoder, Encoder}; + +// Include the generated Pulsar API protobuf code +pub mod proto { + pub mod pulsar { + include!(concat!(env!("OUT_DIR"), "/pulsar.proto.rs")); + } +} + +// Import ServerCommand from command module +use super::command::ServerCommand; + +/// Magic number for checksum verification (0x0e01) +const MAGIC_NUMBER: u16 = 0x0e01; + +/// Maximum message size (from Pulsar defaults) +const MAX_MESSAGE_SIZE: usize = 5 * 1024 * 1024; // 5MB + +/// Frame decoder/encoder for Pulsar binary protocol +pub struct PulsarFrameCodec { + max_frame_size: usize, +} + +impl PulsarFrameCodec { + pub fn new() -> Self { + Self { + max_frame_size: MAX_MESSAGE_SIZE + 10 * 1024, // maxMessageSize + 10KB + } + } +} + +impl Default for PulsarFrameCodec { + fn default() -> Self { + Self::new() + } +} + +/// Decoded Pulsar frame +#[derive(Debug)] +pub struct PulsarFrame { + pub command: Bytes, // Serialized command protobuf + pub metadata: Option, // Serialized metadata protobuf (optional) + pub payload: Bytes, // Message payload + pub checksum: Option, // Checksum (optional) +} + +#[derive(Debug, Clone)] +pub struct EncodedMessageParts { + pub header: Bytes, + pub metadata: Bytes, + pub payload: Bytes, +} + +pub fn estimate_message_parts_size( + consumer_id: u64, + ledger_id: u64, + entry_id: u64, + partition: i32, + metadata: &Bytes, + payload: &Bytes, + redelivery_count: u32, +) -> usize { + use proto::pulsar::*; + + let command = BaseCommand { + r#type: base_command::Type::Message as i32, + message: Some(CommandMessage { + consumer_id, + message_id: MessageIdData { + ledger_id, + entry_id, + partition: Some(partition), + ..Default::default() + }, + redelivery_count: Some(redelivery_count), + ..Default::default() + }), + ..Default::default() + }; + + let cmd_len = command.encoded_len(); + let metadata_len = if metadata.is_empty() { + MessageMetadata { + sequence_id: entry_id, + ..Default::default() + } + .encoded_len() + } else { + metadata.len() + }; + + // Total encoded bytes: + // [TOTAL_SIZE(4)] [CMD_SIZE(4)] [CMD] [METADATA_SIZE(4)] [METADATA] [PAYLOAD] + (4 + 4 + cmd_len + 4) + metadata_len + payload.len() +} + +pub fn encode_message_parts( + consumer_id: u64, + ledger_id: u64, + entry_id: u64, + partition: i32, + metadata: &Bytes, + payload: &Bytes, + redelivery_count: u32, +) -> Result { + use proto::pulsar::*; + + let command = BaseCommand { + r#type: base_command::Type::Message as i32, + message: Some(CommandMessage { + consumer_id, + message_id: MessageIdData { + ledger_id, + entry_id, + partition: Some(partition), + ..Default::default() + }, + redelivery_count: Some(redelivery_count), + ..Default::default() + }), + ..Default::default() + }; + + let cmd_len = command.encoded_len(); + let metadata_bytes = if metadata.is_empty() { + let synthesized = MessageMetadata { + sequence_id: entry_id, + ..Default::default() + }; + let mut buf = BytesMut::with_capacity(synthesized.encoded_len()); + synthesized + .encode(&mut buf) + .map_err(|error| io::Error::new(io::ErrorKind::InvalidData, error.to_string()))?; + buf.freeze() + } else { + metadata.clone() + }; + + // Pulsar wire format for messages with payload: + // [TOTAL_SIZE (4B)] [CMD_SIZE (4B)] [CMD] [METADATA_SIZE (4B)] [METADATA] [PAYLOAD] + let total_size = 4 + cmd_len + 4 + metadata_bytes.len() + payload.len(); + + let mut header = BytesMut::with_capacity(4 + 4 + cmd_len + 4); + header.put_u32(total_size as u32); + header.put_u32(cmd_len as u32); + command + .encode(&mut header) + .map_err(|error| io::Error::new(io::ErrorKind::InvalidData, error.to_string()))?; + header.put_u32(metadata_bytes.len() as u32); + + Ok(EncodedMessageParts { + header: header.freeze(), + metadata: metadata_bytes, + payload: payload.clone(), + }) +} + +impl Decoder for PulsarFrameCodec { + type Item = PulsarFrame; + type Error = io::Error; + + fn decode(&mut self, src: &mut BytesMut) -> Result, Self::Error> { + // Need at least 4 bytes for total size + if src.len() < 4 { + return Ok(None); + } + + // Read total size (big-endian) + let total_size = Cursor::new(&src[..4]).get_u32() as usize; + + // Check frame size limit + if total_size > self.max_frame_size { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + format!("Frame too large: {} bytes", total_size), + )); + } + + // Wait for complete frame + if src.len() < 4 + total_size { + src.reserve(4 + total_size - src.len()); + return Ok(None); + } + + // Split off the frame + let mut frame_data = src.split_to(4 + total_size); + frame_data.advance(4); // Skip total size + + // Read command size + if frame_data.len() < 4 { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "Missing command size", + )); + } + let cmd_size = Cursor::new(&frame_data[..4]).get_u32() as usize; + frame_data.advance(4); + + // Read command + if frame_data.len() < cmd_size { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "Incomplete command data", + )); + } + let command = frame_data.split_to(cmd_size).freeze(); + + // Check for checksum (optional) + let (checksum, mut remaining) = if frame_data.len() >= 6 { + let magic = Cursor::new(&frame_data[..2]).get_u16(); + if magic == MAGIC_NUMBER { + // Has checksum + let checksum = Cursor::new(&frame_data[2..6]).get_u32(); + frame_data.advance(6); + (Some(checksum), frame_data) + } else { + (None, frame_data) + } + } else { + (None, frame_data) + }; + + // Read metadata and payload (zero-copy via freeze()) + let (metadata, payload) = if remaining.len() >= 4 { + let metadata_size = Cursor::new(&remaining[..4]).get_u32() as usize; + remaining.advance(4); + + if remaining.len() >= metadata_size { + let metadata = remaining.split_to(metadata_size).freeze(); + let payload = remaining.freeze(); + (Some(metadata), payload) + } else { + (None, remaining.freeze()) + } + } else { + (None, remaining.freeze()) + }; + + Ok(Some(PulsarFrame { + command, + metadata, + payload, + checksum, + })) + } +} + +impl Encoder for PulsarFrameCodec { + type Error = io::Error; + + fn encode(&mut self, item: ServerCommand, dst: &mut BytesMut) -> Result<(), Self::Error> { + // 特殊处理 Message 命令（包含 payload） + if let ServerCommand::Message { + consumer_id, + ledger_id, + entry_id, + partition, + metadata, + payload, + } = &item + { + return self.encode_message( + *consumer_id, + *ledger_id, + *entry_id, + *partition, + metadata, + payload, + 0, + dst, + ); + } + + let command = item.to_base_command(); + let cmd_len = command.encoded_len(); + + // Pulsar wire format for commands without payload: + // [TOTAL_SIZE (4B)] [CMD_SIZE (4B)] [CMD] + // where TOTAL_SIZE = CMD_SIZE (4) + CMD + + let total_size = 4 + cmd_len; // cmd_size field (4) + cmd + + // Write frame + dst.reserve(total_size + 4); // +4 for total_size field itself + dst.put_u32(total_size as u32); // TOTAL_SIZE + dst.put_u32(cmd_len as u32); // CMD_SIZE + command + .encode(dst) + .map_err(|error| io::Error::new(io::ErrorKind::InvalidData, error.to_string()))?; + + Ok(()) + } +} + +impl PulsarFrameCodec { + /// Encode a Message command with payload + pub fn encode_message( + &self, + consumer_id: u64, + ledger_id: u64, + entry_id: u64, + partition: i32, + metadata: &Bytes, + payload: &Bytes, + redelivery_count: u32, + dst: &mut BytesMut, + ) -> Result<(), io::Error> { + let parts = encode_message_parts( + consumer_id, + ledger_id, + entry_id, + partition, + metadata, + payload, + redelivery_count, + )?; + dst.reserve(parts.header.len() + parts.metadata.len() + parts.payload.len()); + dst.extend_from_slice(parts.header.as_ref()); + dst.extend_from_slice(parts.metadata.as_ref()); + dst.extend_from_slice(parts.payload.as_ref()); + + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use crate::codec::proto::pulsar::{BaseCommand, CompressionType, KeyValue, MessageMetadata}; + use super::*; + use crate::command::ServerCommand; + use bytes::Bytes; + use prost::Message; + + #[test] + fn test_frame_codec_simple() { + let mut codec = PulsarFrameCodec::new(); + + // Create a simple frame: total_size=12, cmd_size=4, cmd=[1,2,3,4], metadata_size=0 + let mut data = BytesMut::new(); + data.put_u32(12); // total size + data.put_u32(4); // cmd size + data.extend_from_slice(&[1, 2, 3, 4]); // cmd + data.put_u32(0); // metadata size (0) + + let frame = codec.decode(&mut data).unwrap().unwrap(); + assert_eq!(frame.command, Bytes::from(vec![1u8, 2, 3, 4])); + assert_eq!(frame.payload, Bytes::new()); + } + + #[test] + fn test_frame_codec_with_payload() { + let mut codec = PulsarFrameCodec::new(); + + // Create frame with payload + let mut data = BytesMut::new(); + data.put_u32(16); // total size + data.put_u32(4); // cmd size + data.extend_from_slice(&[1, 2, 3, 4]); // cmd + data.put_u32(0); // metadata size (0) + data.extend_from_slice(&[5, 6, 7, 8]); // payload + + let frame = codec.decode(&mut data).unwrap().unwrap(); + assert_eq!(frame.command, Bytes::from(vec![1, 2, 3, 4])); + assert_eq!(frame.payload, Bytes::from(vec![5, 6, 7, 8])); + } + + #[test] + fn test_message_encode_preserves_metadata_and_compression() { + let mut codec = PulsarFrameCodec::new(); + let metadata = MessageMetadata { + producer_name: "producer-1".to_string(), + sequence_id: 42, + publish_time: 100, + properties: vec![KeyValue { + key: "env".to_string(), + value: "test".to_string(), + }], + compression: Some(CompressionType::Lz4 as i32), + uncompressed_size: Some(128), + event_time: Some(200), + ordering_key: Some(b"order-key".to_vec()), + ..Default::default() + } + .encode_to_vec(); + + let payload = b"compressed-payload".to_vec(); + let command = ServerCommand::Message { + consumer_id: 7, + ledger_id: 9, + entry_id: 11, + partition: -1, + metadata: Bytes::from(metadata.clone()), + payload: Bytes::from(payload.clone()), + }; + + let mut encoded = BytesMut::new(); + codec.encode(command, &mut encoded).unwrap(); + let frame = codec.decode(&mut encoded).unwrap().unwrap(); + let decoded_metadata = MessageMetadata::decode(&frame.metadata.unwrap()[..]).unwrap(); + + assert_eq!(decoded_metadata.sequence_id, 42); + assert_eq!(decoded_metadata.event_time, Some(200)); + assert_eq!(decoded_metadata.ordering_key, Some(b"order-key".to_vec())); + assert_eq!( + decoded_metadata.compression, + Some(CompressionType::Lz4 as i32) + ); + assert_eq!(decoded_metadata.uncompressed_size, Some(128)); + assert_eq!(decoded_metadata.properties.len(), 1); + assert_eq!(decoded_metadata.properties[0].key, "env"); + assert_eq!(decoded_metadata.properties[0].value, "test"); + assert_eq!(frame.payload, Bytes::from(payload)); + } + + #[test] + fn test_encode_message_parts_matches_codec_output() { + let mut codec = PulsarFrameCodec::new(); + let command = ServerCommand::Message { + consumer_id: 7, + ledger_id: 11, + entry_id: 22, + partition: -1, + metadata: Bytes::new(), + payload: Bytes::from_static(b"payload"), + }; + + let mut encoded = BytesMut::new(); + codec.encode(command, &mut encoded).unwrap(); + + let parts = encode_message_parts( + 7, + 11, + 22, + -1, + &Bytes::new(), + &Bytes::from_static(b"payload"), + 0, + ) + .unwrap(); + let mut rebuilt = BytesMut::new(); + rebuilt.extend_from_slice(parts.header.as_ref()); + rebuilt.extend_from_slice(parts.metadata.as_ref()); + rebuilt.extend_from_slice(parts.payload.as_ref()); + + assert_eq!(encoded.freeze(), rebuilt.freeze()); + } + + #[test] + fn message_command_encodes_redelivery_count() { + let parts = encode_message_parts( + 7, + 11, + 22, + -1, + &Bytes::new(), + &Bytes::from_static(b"payload"), + 3, + ) + .unwrap(); + + let header = parts.header.as_ref(); + let command_size = u32::from_be_bytes(header[4..8].try_into().unwrap()) as usize; + let command = BaseCommand::decode(&header[8..8 + command_size]).unwrap(); + + assert_eq!(command.message.unwrap().redelivery_count, Some(3)); + } +} diff --git a/rust/proto/src/command.rs b/rust/proto/src/command.rs new file mode 100644 index 0000000..8ee137f --- /dev/null +++ b/rust/proto/src/command.rs @@ -0,0 +1,335 @@ +/* + * Server Command Types + * Defines server-side commands for Pulsar protocol + */ + +use super::codec::proto::pulsar::{ + base_command, command_lookup_topic_response, command_partitioned_topic_metadata_response, + BaseCommand, CommandAckResponse, CommandConnected, CommandConsumerStatsResponse, CommandError, + CommandGetLastMessageIdResponse, CommandLookupTopicResponse, CommandMessage, + CommandPartitionedTopicMetadataResponse, CommandPing, CommandPong, CommandProducerSuccess, + CommandSendError, CommandSendReceipt, CommandSuccess, MessageIdData, ServerError, +}; +use bytes::Bytes; +use prost::Message; + +/// Command to send to client +#[derive(Debug)] +pub enum ServerCommand { + Connected { + server_version: String, + protocol_version: i32, + }, + LookupResponse { + request_id: u64, + broker_service_url: String, + }, + PartitionMetadataResponse { + request_id: u64, + partitions: i32, + }, + SendReceipt { + producer_id: u64, + sequence_id: u64, + ledger_id: u64, + entry_id: u64, + partition: i32, + }, + SendError { + producer_id: u64, + sequence_id: u64, + error: ServerError, + message: String, + }, + Success { + request_id: u64, + }, + Error { + request_id: u64, + error: String, + }, + ProducerSuccess { + request_id: u64, + producer_name: String, + producer_id: u64, + }, + ConsumerStatsResponse { + request_id: u64, + consumer_name: String, + }, + Message { + consumer_id: u64, + ledger_id: u64, + entry_id: u64, + partition: i32, + metadata: Bytes, + payload: Bytes, + }, + AckResponse { + consumer_id: u64, + request_id: u64, + }, + LastMessageIdResponse { + request_id: u64, + ledger_id: u64, + entry_id: u64, + partition: i32, + }, + Ping, + Pong, +} + +impl ServerCommand { + /// Serialize command to bytes + pub fn to_bytes(&self) -> Vec { + let command = self.to_base_command(); + command.encode_to_vec() + } + + /// Convert to protobuf BaseCommand + pub fn to_base_command(&self) -> BaseCommand { + use base_command::Type; + + match self { + ServerCommand::Connected { + server_version, + protocol_version, + } => BaseCommand { + r#type: Type::Connected as i32, + connected: Some(CommandConnected { + server_version: server_version.clone(), + protocol_version: Some(*protocol_version), + ..Default::default() + }), + ..Default::default() + }, + ServerCommand::LookupResponse { + request_id, + broker_service_url, + } => BaseCommand { + r#type: Type::LookupResponse as i32, + lookup_topic_response: Some(CommandLookupTopicResponse { + request_id: *request_id, + broker_service_url: Some(broker_service_url.clone()), + response: Some(command_lookup_topic_response::LookupType::Connect as i32), + authoritative: Some(false), + proxy_through_service_url: Some(false), + ..Default::default() + }), + ..Default::default() + }, + ServerCommand::PartitionMetadataResponse { + request_id, + partitions, + } => BaseCommand { + r#type: Type::PartitionedMetadataResponse as i32, + partition_metadata_response: Some(CommandPartitionedTopicMetadataResponse { + request_id: *request_id, + partitions: Some(*partitions as u32), + response: Some( + command_partitioned_topic_metadata_response::LookupType::Success as i32, + ), + ..Default::default() + }), + ..Default::default() + }, + ServerCommand::SendReceipt { + producer_id, + sequence_id, + ledger_id, + entry_id, + partition, + } => BaseCommand { + r#type: Type::SendReceipt as i32, + send_receipt: Some(CommandSendReceipt { + producer_id: *producer_id, + sequence_id: *sequence_id, + message_id: Some(MessageIdData { + ledger_id: *ledger_id, + entry_id: *entry_id, + partition: Some(*partition), + ..Default::default() + }), + ..Default::default() + }), + ..Default::default() + }, + ServerCommand::SendError { + producer_id, + sequence_id, + error, + message, + } => BaseCommand { + r#type: Type::SendError as i32, + send_error: Some(CommandSendError { + producer_id: *producer_id, + sequence_id: *sequence_id, + error: *error as i32, + message: message.clone(), + }), + ..Default::default() + }, + ServerCommand::Success { request_id } => BaseCommand { + r#type: Type::Success as i32, + success: Some(CommandSuccess { + request_id: *request_id, + ..Default::default() + }), + ..Default::default() + }, + ServerCommand::Error { request_id, error } => BaseCommand { + r#type: Type::Error as i32, + error: Some(CommandError { + request_id: *request_id, + message: error.clone(), + ..Default::default() + }), + ..Default::default() + }, + ServerCommand::ProducerSuccess { + request_id, + producer_name, + producer_id: _, + } => BaseCommand { + r#type: Type::ProducerSuccess as i32, + producer_success: Some(CommandProducerSuccess { + request_id: *request_id, + producer_name: producer_name.clone(), + last_sequence_id: Some(-1), + schema_version: Some(Vec::new()), + producer_ready: Some(true), + ..Default::default() + }), + ..Default::default() + }, + ServerCommand::ConsumerStatsResponse { + request_id, + consumer_name, + } => BaseCommand { + r#type: Type::ConsumerStatsResponse as i32, + consumer_stats_response: Some(CommandConsumerStatsResponse { + request_id: *request_id, + consumer_name: Some(consumer_name.clone()), + ..Default::default() + }), + ..Default::default() + }, + ServerCommand::Message { + consumer_id, + ledger_id, + entry_id, + partition, + metadata: _, + payload: _, + } => BaseCommand { + r#type: Type::Message as i32, + message: Some(CommandMessage { + consumer_id: *consumer_id, + message_id: MessageIdData { + ledger_id: *ledger_id, + entry_id: *entry_id, + partition: Some(*partition), + ..Default::default() + }, + ..Default::default() + }), + ..Default::default() + }, + ServerCommand::AckResponse { + consumer_id, + request_id, + } => BaseCommand { + r#type: Type::AckResponse as i32, + ack_response: Some(CommandAckResponse { + consumer_id: *consumer_id, + request_id: Some(*request_id), + ..Default::default() + }), + ..Default::default() + }, + ServerCommand::LastMessageIdResponse { + request_id, + ledger_id, + entry_id, + partition, + } => BaseCommand { + r#type: Type::GetLastMessageIdResponse as i32, + get_last_message_id_response: Some(CommandGetLastMessageIdResponse { + request_id: *request_id, + last_message_id: MessageIdData { + ledger_id: *ledger_id, + entry_id: *entry_id, + partition: Some(*partition), + ..Default::default() + }, + ..Default::default() + }), + ..Default::default() + }, + ServerCommand::Ping => BaseCommand { + r#type: Type::Ping as i32, + ping: Some(CommandPing {}), + ..Default::default() + }, + ServerCommand::Pong => BaseCommand { + r#type: Type::Pong as i32, + pong: Some(CommandPong {}), + ..Default::default() + }, + } + } + + /// Check if this command has payload (Message command) + pub fn has_payload(&self) -> bool { + matches!(self, ServerCommand::Message { .. }) + } + + /// Get payload reference if available + pub fn get_payload(&self) -> Option<&[u8]> { + match self { + ServerCommand::Message { payload, .. } => Some(payload.as_ref()), + _ => None, + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use prost::Message; + + #[test] + fn test_server_command_serialization() { + let cmd = ServerCommand::ProducerSuccess { + request_id: 1, + producer_name: "test-producer".to_string(), + producer_id: 100, + }; + + let bytes = cmd.to_bytes(); + assert!(!bytes.is_empty()); + + // Verify we can deserialize it + let decoded = super::super::codec::proto::pulsar::BaseCommand::decode(&bytes[..]).unwrap(); + assert_eq!( + decoded.r#type, + super::super::codec::proto::pulsar::base_command::Type::ProducerSuccess as i32 + ); + } + + #[test] + fn test_has_payload() { + let msg_cmd = ServerCommand::Message { + consumer_id: 1, + ledger_id: 2, + entry_id: 3, + partition: 0, + metadata: Bytes::from_static(&[4, 5, 6]), + payload: Bytes::from_static(&[1, 2, 3]), + }; + assert!(msg_cmd.has_payload()); + + let success_cmd = ServerCommand::Success { request_id: 1 }; + assert!(!success_cmd.has_payload()); + } +} diff --git a/rust/proto/src/lib.rs b/rust/proto/src/lib.rs new file mode 100644 index 0000000..2313f66 --- /dev/null +++ b/rust/proto/src/lib.rs @@ -0,0 +1,6 @@ +pub mod codec; +pub mod command; + +// Re-export commonly used types +pub use codec::{PulsarFrame, PulsarFrameCodec}; +pub use command::ServerCommand; From 2c73988507e63b35c8058bfc6811108308cc1267 Mon Sep 17 00:00:00 2001 From: XTLine <2668543450@qq.com> Date: Sat, 8 Aug 2026 21:57:02 +0800 Subject: [PATCH 17/44] refactor(broker): remove duplicated protocol/storage modules from main crate --- rust/src/lib.rs | 3 - rust/src/protocol/codec.rs | 502 ------------------------- rust/src/protocol/command.rs | 335 ----------------- rust/src/protocol/mod.rs | 6 - rust/src/storage/managed_ledger/mod.rs | 8 - rust/src/storage/mod.rs | 66 ---- 6 files changed, 920 deletions(-) delete mode 100644 rust/src/protocol/codec.rs delete mode 100644 rust/src/protocol/command.rs delete mode 100644 rust/src/protocol/mod.rs delete mode 100644 rust/src/storage/managed_ledger/mod.rs delete mode 100644 rust/src/storage/mod.rs diff --git a/rust/src/lib.rs b/rust/src/lib.rs index 0c3ac21..bfee99a 100644 --- a/rust/src/lib.rs +++ b/rust/src/lib.rs @@ -1,11 +1,8 @@ pub mod broker; pub mod config; pub mod error; -pub mod protocol; -pub mod storage; // Re-export commonly used types pub use broker::BrokerService; pub use config::Config; pub use error::{Error, Result}; -pub use storage::Storage; diff --git a/rust/src/protocol/codec.rs b/rust/src/protocol/codec.rs deleted file mode 100644 index 0b8fb9c..0000000 --- a/rust/src/protocol/codec.rs +++ /dev/null @@ -1,502 +0,0 @@ -/* - * Pulsar Binary Protocol Implementation - * - * Wire format: - * [TOTAL_SIZE (4B)] [CMD_SIZE (4B)] [CMD (Protobuf)] [MAGIC_NUMBER (2B, optional)] [CHECKSUM (4B, optional)] [METADATA_SIZE (4B)] [METADATA (Protobuf)] [PAYLOAD] - * - * Frame decoder: LengthFieldBasedFrameDecoder(maxMessageSize + 10KB, 0, 4, 0, 4) - */ - -use bytes::{Buf, BufMut, Bytes, BytesMut}; -use prost::Message; -use std::io::{self, Cursor}; -use tokio_util::codec::{Decoder, Encoder}; - -// Include the generated Pulsar API protobuf code -pub mod proto { - pub mod pulsar { - include!(concat!(env!("OUT_DIR"), "/pulsar.proto.rs")); - } -} - -// Import ServerCommand from command module -use super::command::ServerCommand; -use crate::broker::service::PendingMessage; - -/// Magic number for checksum verification (0x0e01) -const MAGIC_NUMBER: u16 = 0x0e01; - -/// Maximum message size (from Pulsar defaults) -const MAX_MESSAGE_SIZE: usize = 5 * 1024 * 1024; // 5MB - -/// Frame decoder/encoder for Pulsar binary protocol -pub struct PulsarFrameCodec { - max_frame_size: usize, -} - -impl PulsarFrameCodec { - pub fn new() -> Self { - Self { - max_frame_size: MAX_MESSAGE_SIZE + 10 * 1024, // maxMessageSize + 10KB - } - } -} - -impl Default for PulsarFrameCodec { - fn default() -> Self { - Self::new() - } -} - -/// Decoded Pulsar frame -#[derive(Debug)] -pub struct PulsarFrame { - pub command: Bytes, // Serialized command protobuf - pub metadata: Option, // Serialized metadata protobuf (optional) - pub payload: Bytes, // Message payload - pub checksum: Option, // Checksum (optional) -} - -#[derive(Debug, Clone)] -pub struct EncodedMessageParts { - pub header: Bytes, - pub metadata: Bytes, - pub payload: Bytes, -} - -pub fn estimate_message_parts_size( - consumer_id: u64, - ledger_id: u64, - entry_id: u64, - partition: i32, - metadata: &Bytes, - payload: &Bytes, - redelivery_count: u32, -) -> usize { - use proto::pulsar::*; - - let command = BaseCommand { - r#type: base_command::Type::Message as i32, - message: Some(CommandMessage { - consumer_id, - message_id: MessageIdData { - ledger_id, - entry_id, - partition: Some(partition), - ..Default::default() - }, - redelivery_count: Some(redelivery_count), - ..Default::default() - }), - ..Default::default() - }; - - let cmd_len = command.encoded_len(); - let metadata_len = if metadata.is_empty() { - MessageMetadata { - sequence_id: entry_id, - ..Default::default() - } - .encoded_len() - } else { - metadata.len() - }; - - // Total encoded bytes: - // [TOTAL_SIZE(4)] [CMD_SIZE(4)] [CMD] [METADATA_SIZE(4)] [METADATA] [PAYLOAD] - (4 + 4 + cmd_len + 4) + metadata_len + payload.len() -} - -pub fn encode_message_parts( - consumer_id: u64, - ledger_id: u64, - entry_id: u64, - partition: i32, - metadata: &Bytes, - payload: &Bytes, - redelivery_count: u32, -) -> Result { - use proto::pulsar::*; - - let command = BaseCommand { - r#type: base_command::Type::Message as i32, - message: Some(CommandMessage { - consumer_id, - message_id: MessageIdData { - ledger_id, - entry_id, - partition: Some(partition), - ..Default::default() - }, - redelivery_count: Some(redelivery_count), - ..Default::default() - }), - ..Default::default() - }; - - let cmd_len = command.encoded_len(); - let metadata_bytes = if metadata.is_empty() { - let synthesized = MessageMetadata { - sequence_id: entry_id, - ..Default::default() - }; - let mut buf = BytesMut::with_capacity(synthesized.encoded_len()); - synthesized - .encode(&mut buf) - .map_err(|error| io::Error::new(io::ErrorKind::InvalidData, error.to_string()))?; - buf.freeze() - } else { - metadata.clone() - }; - - // Pulsar wire format for messages with payload: - // [TOTAL_SIZE (4B)] [CMD_SIZE (4B)] [CMD] [METADATA_SIZE (4B)] [METADATA] [PAYLOAD] - let total_size = 4 + cmd_len + 4 + metadata_bytes.len() + payload.len(); - - let mut header = BytesMut::with_capacity(4 + 4 + cmd_len + 4); - header.put_u32(total_size as u32); - header.put_u32(cmd_len as u32); - command - .encode(&mut header) - .map_err(|error| io::Error::new(io::ErrorKind::InvalidData, error.to_string()))?; - header.put_u32(metadata_bytes.len() as u32); - - Ok(EncodedMessageParts { - header: header.freeze(), - metadata: metadata_bytes, - payload: payload.clone(), - }) -} - -impl Decoder for PulsarFrameCodec { - type Item = PulsarFrame; - type Error = io::Error; - - fn decode(&mut self, src: &mut BytesMut) -> Result, Self::Error> { - // Need at least 4 bytes for total size - if src.len() < 4 { - return Ok(None); - } - - // Read total size (big-endian) - let total_size = Cursor::new(&src[..4]).get_u32() as usize; - - // Check frame size limit - if total_size > self.max_frame_size { - return Err(io::Error::new( - io::ErrorKind::InvalidData, - format!("Frame too large: {} bytes", total_size), - )); - } - - // Wait for complete frame - if src.len() < 4 + total_size { - src.reserve(4 + total_size - src.len()); - return Ok(None); - } - - // Split off the frame - let mut frame_data = src.split_to(4 + total_size); - frame_data.advance(4); // Skip total size - - // Read command size - if frame_data.len() < 4 { - return Err(io::Error::new( - io::ErrorKind::InvalidData, - "Missing command size", - )); - } - let cmd_size = Cursor::new(&frame_data[..4]).get_u32() as usize; - frame_data.advance(4); - - // Read command - if frame_data.len() < cmd_size { - return Err(io::Error::new( - io::ErrorKind::InvalidData, - "Incomplete command data", - )); - } - let command = frame_data.split_to(cmd_size).freeze(); - - // Check for checksum (optional) - let (checksum, mut remaining) = if frame_data.len() >= 6 { - let magic = Cursor::new(&frame_data[..2]).get_u16(); - if magic == MAGIC_NUMBER { - // Has checksum - let checksum = Cursor::new(&frame_data[2..6]).get_u32(); - frame_data.advance(6); - (Some(checksum), frame_data) - } else { - (None, frame_data) - } - } else { - (None, frame_data) - }; - - // Read metadata and payload (zero-copy via freeze()) - let (metadata, payload) = if remaining.len() >= 4 { - let metadata_size = Cursor::new(&remaining[..4]).get_u32() as usize; - remaining.advance(4); - - if remaining.len() >= metadata_size { - let metadata = remaining.split_to(metadata_size).freeze(); - let payload = remaining.freeze(); - (Some(metadata), payload) - } else { - (None, remaining.freeze()) - } - } else { - (None, remaining.freeze()) - }; - - Ok(Some(PulsarFrame { - command, - metadata, - payload, - checksum, - })) - } -} - -impl Encoder for PulsarFrameCodec { - type Error = io::Error; - - fn encode(&mut self, item: ServerCommand, dst: &mut BytesMut) -> Result<(), Self::Error> { - // 特殊处理 Message 命令（包含 payload） - if let ServerCommand::Message { - consumer_id, - ledger_id, - entry_id, - partition, - metadata, - payload, - } = &item - { - return self.encode_message( - *consumer_id, - *ledger_id, - *entry_id, - *partition, - metadata, - payload, - 0, - dst, - ); - } - - let command = item.to_base_command(); - let cmd_len = command.encoded_len(); - - // Pulsar wire format for commands without payload: - // [TOTAL_SIZE (4B)] [CMD_SIZE (4B)] [CMD] - // where TOTAL_SIZE = CMD_SIZE (4) + CMD - - let total_size = 4 + cmd_len; // cmd_size field (4) + cmd - - // Write frame - dst.reserve(total_size + 4); // +4 for total_size field itself - dst.put_u32(total_size as u32); // TOTAL_SIZE - dst.put_u32(cmd_len as u32); // CMD_SIZE - command - .encode(dst) - .map_err(|error| io::Error::new(io::ErrorKind::InvalidData, error.to_string()))?; - - Ok(()) - } -} - -impl Encoder<(u64, PendingMessage)> for PulsarFrameCodec { - type Error = io::Error; - - fn encode( - &mut self, - item: (u64, PendingMessage), - dst: &mut BytesMut, - ) -> Result<(), Self::Error> { - let (consumer_id, msg) = item; - self.encode_message( - consumer_id, - msg.message_id.ledger, - msg.message_id.entry, - msg.message_id.partition, - &msg.metadata, - &msg.payload, - msg.redelivery_count, - dst, - ) - } -} - -impl PulsarFrameCodec { - /// Encode a Message command with payload - fn encode_message( - &self, - consumer_id: u64, - ledger_id: u64, - entry_id: u64, - partition: i32, - metadata: &Bytes, - payload: &Bytes, - redelivery_count: u32, - dst: &mut BytesMut, - ) -> Result<(), io::Error> { - let parts = encode_message_parts( - consumer_id, - ledger_id, - entry_id, - partition, - metadata, - payload, - redelivery_count, - )?; - dst.reserve(parts.header.len() + parts.metadata.len() + parts.payload.len()); - dst.extend_from_slice(parts.header.as_ref()); - dst.extend_from_slice(parts.metadata.as_ref()); - dst.extend_from_slice(parts.payload.as_ref()); - - Ok(()) - } -} - -#[cfg(test)] -mod tests { - use super::proto::pulsar::{BaseCommand, CompressionType, KeyValue, MessageMetadata}; - use super::*; - use crate::protocol::command::ServerCommand; - use bytes::Bytes; - use prost::Message; - - #[test] - fn test_frame_codec_simple() { - let mut codec = PulsarFrameCodec::new(); - - // Create a simple frame: total_size=12, cmd_size=4, cmd=[1,2,3,4], metadata_size=0 - let mut data = BytesMut::new(); - data.put_u32(12); // total size - data.put_u32(4); // cmd size - data.extend_from_slice(&[1, 2, 3, 4]); // cmd - data.put_u32(0); // metadata size (0) - - let frame = codec.decode(&mut data).unwrap().unwrap(); - assert_eq!(frame.command, Bytes::from(vec![1u8, 2, 3, 4])); - assert_eq!(frame.payload, Bytes::new()); - } - - #[test] - fn test_frame_codec_with_payload() { - let mut codec = PulsarFrameCodec::new(); - - // Create frame with payload - let mut data = BytesMut::new(); - data.put_u32(16); // total size - data.put_u32(4); // cmd size - data.extend_from_slice(&[1, 2, 3, 4]); // cmd - data.put_u32(0); // metadata size (0) - data.extend_from_slice(&[5, 6, 7, 8]); // payload - - let frame = codec.decode(&mut data).unwrap().unwrap(); - assert_eq!(frame.command, Bytes::from(vec![1, 2, 3, 4])); - assert_eq!(frame.payload, Bytes::from(vec![5, 6, 7, 8])); - } - - #[test] - fn test_message_encode_preserves_metadata_and_compression() { - let mut codec = PulsarFrameCodec::new(); - let metadata = MessageMetadata { - producer_name: "producer-1".to_string(), - sequence_id: 42, - publish_time: 100, - properties: vec![KeyValue { - key: "env".to_string(), - value: "test".to_string(), - }], - compression: Some(CompressionType::Lz4 as i32), - uncompressed_size: Some(128), - event_time: Some(200), - ordering_key: Some(b"order-key".to_vec()), - ..Default::default() - } - .encode_to_vec(); - - let payload = b"compressed-payload".to_vec(); - let command = ServerCommand::Message { - consumer_id: 7, - ledger_id: 9, - entry_id: 11, - partition: -1, - metadata: Bytes::from(metadata.clone()), - payload: Bytes::from(payload.clone()), - }; - - let mut encoded = BytesMut::new(); - codec.encode(command, &mut encoded).unwrap(); - let frame = codec.decode(&mut encoded).unwrap().unwrap(); - let decoded_metadata = MessageMetadata::decode(&frame.metadata.unwrap()[..]).unwrap(); - - assert_eq!(decoded_metadata.sequence_id, 42); - assert_eq!(decoded_metadata.event_time, Some(200)); - assert_eq!(decoded_metadata.ordering_key, Some(b"order-key".to_vec())); - assert_eq!( - decoded_metadata.compression, - Some(CompressionType::Lz4 as i32) - ); - assert_eq!(decoded_metadata.uncompressed_size, Some(128)); - assert_eq!(decoded_metadata.properties.len(), 1); - assert_eq!(decoded_metadata.properties[0].key, "env"); - assert_eq!(decoded_metadata.properties[0].value, "test"); - assert_eq!(frame.payload, Bytes::from(payload)); - } - - #[test] - fn test_encode_message_parts_matches_codec_output() { - let mut codec = PulsarFrameCodec::new(); - let command = ServerCommand::Message { - consumer_id: 7, - ledger_id: 11, - entry_id: 22, - partition: -1, - metadata: Bytes::new(), - payload: Bytes::from_static(b"payload"), - }; - - let mut encoded = BytesMut::new(); - codec.encode(command, &mut encoded).unwrap(); - - let parts = encode_message_parts( - 7, - 11, - 22, - -1, - &Bytes::new(), - &Bytes::from_static(b"payload"), - 0, - ) - .unwrap(); - let mut rebuilt = BytesMut::new(); - rebuilt.extend_from_slice(parts.header.as_ref()); - rebuilt.extend_from_slice(parts.metadata.as_ref()); - rebuilt.extend_from_slice(parts.payload.as_ref()); - - assert_eq!(encoded.freeze(), rebuilt.freeze()); - } - - #[test] - fn message_command_encodes_redelivery_count() { - let parts = encode_message_parts( - 7, - 11, - 22, - -1, - &Bytes::new(), - &Bytes::from_static(b"payload"), - 3, - ) - .unwrap(); - - let header = parts.header.as_ref(); - let command_size = u32::from_be_bytes(header[4..8].try_into().unwrap()) as usize; - let command = BaseCommand::decode(&header[8..8 + command_size]).unwrap(); - - assert_eq!(command.message.unwrap().redelivery_count, Some(3)); - } -} diff --git a/rust/src/protocol/command.rs b/rust/src/protocol/command.rs deleted file mode 100644 index 8ee137f..0000000 --- a/rust/src/protocol/command.rs +++ /dev/null @@ -1,335 +0,0 @@ -/* - * Server Command Types - * Defines server-side commands for Pulsar protocol - */ - -use super::codec::proto::pulsar::{ - base_command, command_lookup_topic_response, command_partitioned_topic_metadata_response, - BaseCommand, CommandAckResponse, CommandConnected, CommandConsumerStatsResponse, CommandError, - CommandGetLastMessageIdResponse, CommandLookupTopicResponse, CommandMessage, - CommandPartitionedTopicMetadataResponse, CommandPing, CommandPong, CommandProducerSuccess, - CommandSendError, CommandSendReceipt, CommandSuccess, MessageIdData, ServerError, -}; -use bytes::Bytes; -use prost::Message; - -/// Command to send to client -#[derive(Debug)] -pub enum ServerCommand { - Connected { - server_version: String, - protocol_version: i32, - }, - LookupResponse { - request_id: u64, - broker_service_url: String, - }, - PartitionMetadataResponse { - request_id: u64, - partitions: i32, - }, - SendReceipt { - producer_id: u64, - sequence_id: u64, - ledger_id: u64, - entry_id: u64, - partition: i32, - }, - SendError { - producer_id: u64, - sequence_id: u64, - error: ServerError, - message: String, - }, - Success { - request_id: u64, - }, - Error { - request_id: u64, - error: String, - }, - ProducerSuccess { - request_id: u64, - producer_name: String, - producer_id: u64, - }, - ConsumerStatsResponse { - request_id: u64, - consumer_name: String, - }, - Message { - consumer_id: u64, - ledger_id: u64, - entry_id: u64, - partition: i32, - metadata: Bytes, - payload: Bytes, - }, - AckResponse { - consumer_id: u64, - request_id: u64, - }, - LastMessageIdResponse { - request_id: u64, - ledger_id: u64, - entry_id: u64, - partition: i32, - }, - Ping, - Pong, -} - -impl ServerCommand { - /// Serialize command to bytes - pub fn to_bytes(&self) -> Vec { - let command = self.to_base_command(); - command.encode_to_vec() - } - - /// Convert to protobuf BaseCommand - pub fn to_base_command(&self) -> BaseCommand { - use base_command::Type; - - match self { - ServerCommand::Connected { - server_version, - protocol_version, - } => BaseCommand { - r#type: Type::Connected as i32, - connected: Some(CommandConnected { - server_version: server_version.clone(), - protocol_version: Some(*protocol_version), - ..Default::default() - }), - ..Default::default() - }, - ServerCommand::LookupResponse { - request_id, - broker_service_url, - } => BaseCommand { - r#type: Type::LookupResponse as i32, - lookup_topic_response: Some(CommandLookupTopicResponse { - request_id: *request_id, - broker_service_url: Some(broker_service_url.clone()), - response: Some(command_lookup_topic_response::LookupType::Connect as i32), - authoritative: Some(false), - proxy_through_service_url: Some(false), - ..Default::default() - }), - ..Default::default() - }, - ServerCommand::PartitionMetadataResponse { - request_id, - partitions, - } => BaseCommand { - r#type: Type::PartitionedMetadataResponse as i32, - partition_metadata_response: Some(CommandPartitionedTopicMetadataResponse { - request_id: *request_id, - partitions: Some(*partitions as u32), - response: Some( - command_partitioned_topic_metadata_response::LookupType::Success as i32, - ), - ..Default::default() - }), - ..Default::default() - }, - ServerCommand::SendReceipt { - producer_id, - sequence_id, - ledger_id, - entry_id, - partition, - } => BaseCommand { - r#type: Type::SendReceipt as i32, - send_receipt: Some(CommandSendReceipt { - producer_id: *producer_id, - sequence_id: *sequence_id, - message_id: Some(MessageIdData { - ledger_id: *ledger_id, - entry_id: *entry_id, - partition: Some(*partition), - ..Default::default() - }), - ..Default::default() - }), - ..Default::default() - }, - ServerCommand::SendError { - producer_id, - sequence_id, - error, - message, - } => BaseCommand { - r#type: Type::SendError as i32, - send_error: Some(CommandSendError { - producer_id: *producer_id, - sequence_id: *sequence_id, - error: *error as i32, - message: message.clone(), - }), - ..Default::default() - }, - ServerCommand::Success { request_id } => BaseCommand { - r#type: Type::Success as i32, - success: Some(CommandSuccess { - request_id: *request_id, - ..Default::default() - }), - ..Default::default() - }, - ServerCommand::Error { request_id, error } => BaseCommand { - r#type: Type::Error as i32, - error: Some(CommandError { - request_id: *request_id, - message: error.clone(), - ..Default::default() - }), - ..Default::default() - }, - ServerCommand::ProducerSuccess { - request_id, - producer_name, - producer_id: _, - } => BaseCommand { - r#type: Type::ProducerSuccess as i32, - producer_success: Some(CommandProducerSuccess { - request_id: *request_id, - producer_name: producer_name.clone(), - last_sequence_id: Some(-1), - schema_version: Some(Vec::new()), - producer_ready: Some(true), - ..Default::default() - }), - ..Default::default() - }, - ServerCommand::ConsumerStatsResponse { - request_id, - consumer_name, - } => BaseCommand { - r#type: Type::ConsumerStatsResponse as i32, - consumer_stats_response: Some(CommandConsumerStatsResponse { - request_id: *request_id, - consumer_name: Some(consumer_name.clone()), - ..Default::default() - }), - ..Default::default() - }, - ServerCommand::Message { - consumer_id, - ledger_id, - entry_id, - partition, - metadata: _, - payload: _, - } => BaseCommand { - r#type: Type::Message as i32, - message: Some(CommandMessage { - consumer_id: *consumer_id, - message_id: MessageIdData { - ledger_id: *ledger_id, - entry_id: *entry_id, - partition: Some(*partition), - ..Default::default() - }, - ..Default::default() - }), - ..Default::default() - }, - ServerCommand::AckResponse { - consumer_id, - request_id, - } => BaseCommand { - r#type: Type::AckResponse as i32, - ack_response: Some(CommandAckResponse { - consumer_id: *consumer_id, - request_id: Some(*request_id), - ..Default::default() - }), - ..Default::default() - }, - ServerCommand::LastMessageIdResponse { - request_id, - ledger_id, - entry_id, - partition, - } => BaseCommand { - r#type: Type::GetLastMessageIdResponse as i32, - get_last_message_id_response: Some(CommandGetLastMessageIdResponse { - request_id: *request_id, - last_message_id: MessageIdData { - ledger_id: *ledger_id, - entry_id: *entry_id, - partition: Some(*partition), - ..Default::default() - }, - ..Default::default() - }), - ..Default::default() - }, - ServerCommand::Ping => BaseCommand { - r#type: Type::Ping as i32, - ping: Some(CommandPing {}), - ..Default::default() - }, - ServerCommand::Pong => BaseCommand { - r#type: Type::Pong as i32, - pong: Some(CommandPong {}), - ..Default::default() - }, - } - } - - /// Check if this command has payload (Message command) - pub fn has_payload(&self) -> bool { - matches!(self, ServerCommand::Message { .. }) - } - - /// Get payload reference if available - pub fn get_payload(&self) -> Option<&[u8]> { - match self { - ServerCommand::Message { payload, .. } => Some(payload.as_ref()), - _ => None, - } - } -} - -#[cfg(test)] -mod tests { - use super::*; - use prost::Message; - - #[test] - fn test_server_command_serialization() { - let cmd = ServerCommand::ProducerSuccess { - request_id: 1, - producer_name: "test-producer".to_string(), - producer_id: 100, - }; - - let bytes = cmd.to_bytes(); - assert!(!bytes.is_empty()); - - // Verify we can deserialize it - let decoded = super::super::codec::proto::pulsar::BaseCommand::decode(&bytes[..]).unwrap(); - assert_eq!( - decoded.r#type, - super::super::codec::proto::pulsar::base_command::Type::ProducerSuccess as i32 - ); - } - - #[test] - fn test_has_payload() { - let msg_cmd = ServerCommand::Message { - consumer_id: 1, - ledger_id: 2, - entry_id: 3, - partition: 0, - metadata: Bytes::from_static(&[4, 5, 6]), - payload: Bytes::from_static(&[1, 2, 3]), - }; - assert!(msg_cmd.has_payload()); - - let success_cmd = ServerCommand::Success { request_id: 1 }; - assert!(!success_cmd.has_payload()); - } -} diff --git a/rust/src/protocol/mod.rs b/rust/src/protocol/mod.rs deleted file mode 100644 index 2313f66..0000000 --- a/rust/src/protocol/mod.rs +++ /dev/null @@ -1,6 +0,0 @@ -pub mod codec; -pub mod command; - -// Re-export commonly used types -pub use codec::{PulsarFrame, PulsarFrameCodec}; -pub use command::ServerCommand; diff --git a/rust/src/storage/managed_ledger/mod.rs b/rust/src/storage/managed_ledger/mod.rs deleted file mode 100644 index fe4517f..0000000 --- a/rust/src/storage/managed_ledger/mod.rs +++ /dev/null @@ -1,8 +0,0 @@ -//! Managed-ledger module: re-exports the managed-ledger crate types. -pub use pulsar_lite_storage_managed_ledger::{ - CursorInitOptions, CursorOpenResult, InMemoryManagedCursor, InMemoryManagedLedger, - InMemoryManagedLedgerFactory, InMemoryManagedLedgerStorage, InitialPosition, ManagedCursor, - ManagedCursorState, ManagedLedger, ManagedLedgerConfig, ManagedLedgerFactory, - ManagedLedgerPosition, ManagedLedgerStorage, MessageId, NonPersistentEntry, StoredMessage, - SubscriptionCursor, -}; diff --git a/rust/src/storage/mod.rs b/rust/src/storage/mod.rs deleted file mode 100644 index 96ed42d..0000000 --- a/rust/src/storage/mod.rs +++ /dev/null @@ -1,66 +0,0 @@ -mod managed_ledger; - -use anyhow::Result; -pub use pulsar_lite_storage::{ManagedLedgerStore, Storage}; - -pub use managed_ledger::{ - CursorInitOptions, CursorOpenResult, InMemoryManagedCursor, InMemoryManagedLedger, - InMemoryManagedLedgerFactory, InMemoryManagedLedgerStorage, InitialPosition, ManagedCursor, - ManagedCursorState, ManagedLedger, ManagedLedgerConfig, ManagedLedgerFactory, - ManagedLedgerPosition, ManagedLedgerStorage, MessageId, NonPersistentEntry, StoredMessage, - SubscriptionCursor, -}; -pub use pulsar_lite_storage_metadata::{ - parse_topic_name, DomainNode, FileMetadataStore, MetadataDocument, MetadataFileNode, - MetadataStore, NamespaceMetadata, NamespaceNode, ParsedTopicName, PartitionedTopicNode, - SubscriptionMetadata, SubscriptionNode, TenantMetadata, TenantNode, TopicMetadata, TopicNode, -}; -pub use pulsar_lite_storage_resources::{ - NamespaceResources, PulsarResources, TenantResources, TopicResources, -}; - -pub(crate) fn decode_publish_time(metadata: &[u8]) -> Option { - use crate::protocol::codec::proto::pulsar::MessageMetadata; - use prost::Message; - if metadata.is_empty() { - return None; - } - MessageMetadata::decode(metadata) - .ok() - .map(|m| m.publish_time) -} - -/// Seek helpers that depend on broker protocol decoding (kept in the main crate). -pub trait StorageSeekExt { - fn find_message_id_by_publish_time( - &self, - topic: &str, - publish_time: u64, - ) -> Result>; -} - -impl StorageSeekExt for Storage { - fn find_message_id_by_publish_time( - &self, - topic: &str, - publish_time: u64, - ) -> Result> { - let entries = self.get_message_entries(topic); - if entries.is_empty() { - return Ok(None); - } - let mut last_earlier: Option = None; - for (i, entry) in entries.iter().enumerate() { - match decode_publish_time(&entry.metadata) { - Some(pt) if pt < publish_time => last_earlier = Some(i), - Some(_) => break, - None => {} - } - } - let target = match last_earlier { - None => Some(entries[0].message_id.clone()), - Some(i) => entries.get(i + 1).map(|e| e.message_id.clone()), - }; - Ok(target) - } -} From 407b697df7daad1735ff02e1ce80b73592a4e500 Mon Sep 17 00:00:00 2001 From: XTLine <2668543450@qq.com> Date: Sat, 8 Aug 2026 22:11:29 +0800 Subject: [PATCH 18/44] fix(broker): resolve orphan-rule violation via ConsumerMessage newtype; move seek helpers into storage core --- rust/src/broker/service/server_cnx.rs | 54 ++++++++++++++++++++------- rust/storage/core/Cargo.toml | 2 + rust/storage/core/src/service.rs | 35 +++++++++++++++++ 3 files changed, 78 insertions(+), 13 deletions(-) diff --git a/rust/src/broker/service/server_cnx.rs b/rust/src/broker/service/server_cnx.rs index 9402f68..662091e 100644 --- a/rust/src/broker/service/server_cnx.rs +++ b/rust/src/broker/service/server_cnx.rs @@ -18,11 +18,39 @@ use super::{ConnectionWriteState, Consumer, Producer, SharedStorage}; use crate::broker::broker_service::SharedBrokerService; use crate::broker::handler; use crate::broker::service::topic::TopicPublishRateExceeded; -use crate::protocol::codec::{ +use pulsar_lite_proto::codec::{ proto::pulsar::{base_command, BaseCommand, ProtocolVersion, ServerError}, PulsarFrame, PulsarFrameCodec, }; -use crate::protocol::ServerCommand; +use pulsar_lite_proto::ServerCommand; + +/// Local newtype so the orphan rule accepts the foreign-trait impl: the trait +/// parameter is now the local type itself (uncovered), not a tuple-wrapped one. +pub struct ConsumerMessage(pub u64, pub PendingMessage); + +/// Adapter impl: the protocol crate only knows raw encode primitives, the +/// business message type (PendingMessage) lives here in the broker crate. +impl tokio_util::codec::Encoder for PulsarFrameCodec { + type Error = std::io::Error; + + fn encode( + &mut self, + item: ConsumerMessage, + dst: &mut bytes::BytesMut, + ) -> Result<(), Self::Error> { + let ConsumerMessage(consumer_id, msg) = item; + self.encode_message( + consumer_id, + msg.message_id.ledger, + msg.message_id.entry, + msg.message_id.partition, + &msg.metadata, + &msg.payload, + msg.redelivery_count, + dst, + ) + } +} type CnxError = Box; type CnxResult = Result; @@ -81,7 +109,7 @@ const DEFAULT_MAX_PERSISTENT_IN_FLIGHT: usize = 1000; struct PersistentSendOutcome { producer_id: u64, sequence_id: u64, - result: Result, + result: Result, } /// Runtime context for a single broker connection. @@ -157,14 +185,14 @@ where .observe_buffered_bytes(self.framed.write_buffer().len()); } - async fn write_message_batch(&mut self, batch: Vec<(u64, PendingMessage)>) -> CnxResult<()> { + async fn write_message_batch(&mut self, batch: Vec) -> CnxResult<()> { for item in batch { self.sync_connection_writable_from_framed_buffer(); self.framed.feed(item).await.map_err(to_cnx_error)?; self.sync_connection_writable_from_framed_buffer(); } - futures::sink::SinkExt::<(u64, PendingMessage)>::flush(&mut self.framed) + futures::sink::SinkExt::::flush(&mut self.framed) .await .map_err(to_cnx_error)?; self.sync_connection_writable_from_framed_buffer(); @@ -301,9 +329,9 @@ where // into the write buffer with feed(), then flush once for a single // TCP write. This amortises syscall overhead across many messages. Some((consumer_id, pending_msg)) = self.message_rx.recv() => { - let mut batch = vec![(consumer_id, pending_msg)]; + let mut batch = vec![ConsumerMessage(consumer_id, pending_msg)]; while let Ok(next) = self.message_rx.try_recv() { - batch.push(next); + batch.push(ConsumerMessage(next.0, next.1)); if batch.len() >= 128 { break; } @@ -379,9 +407,9 @@ where // into the write buffer with feed(), then flush once for a single // TCP write. This amortises syscall overhead across many messages. Some((consumer_id, pending_msg)) = self.message_rx.recv() => { - let mut batch = vec![(consumer_id, pending_msg)]; + let mut batch = vec![ConsumerMessage(consumer_id, pending_msg)]; while let Ok(next) = self.message_rx.try_recv() { - batch.push(next); + batch.push(ConsumerMessage(next.0, next.1)); if batch.len() >= 128 { break; } @@ -1087,11 +1115,11 @@ mod tests { use super::*; use crate::broker::broker_service::{BrokerService, TopicRef}; use crate::broker::service::{topic::TopicRuntimeMode, Producer}; - use crate::protocol::codec::proto::pulsar::{ + use bytes::Bytes; + use pulsar_lite_proto::codec::proto::pulsar::{ base_command, BaseCommand, CommandConnect, CommandPing, CommandSend, }; - use crate::storage::Storage; - use bytes::Bytes; + use pulsar_lite_storage::Storage; use std::sync::Arc; use tempfile::TempDir; use tokio::io::{duplex, DuplexStream}; @@ -1377,7 +1405,7 @@ mod tests { assert_eq!(send_error.sequence_id, 11); assert_eq!( send_error.error, - crate::protocol::codec::proto::pulsar::ServerError::NotAllowedError as i32 + pulsar_lite_proto::codec::proto::pulsar::ServerError::NotAllowedError as i32 ); assert!(send_error.message.contains("Exceed maximum message size")); } diff --git a/rust/storage/core/Cargo.toml b/rust/storage/core/Cargo.toml index 98441f0..5147e53 100644 --- a/rust/storage/core/Cargo.toml +++ b/rust/storage/core/Cargo.toml @@ -13,8 +13,10 @@ pulsar-lite-storage-metadata = { path = "../metadata" } pulsar-lite-storage-resources = { path = "../resources" } pulsar-lite-storage-managed-ledger = { path = "../managed-ledger" } pulsar-lite-storage-managed-ledger-rocksdb = { path = "../managed-ledger-rocksdb", optional = true } +pulsar-lite-proto = { path = "../../proto" } anyhow.workspace = true log.workspace = true +prost = "0.13" [dev-dependencies] tempfile = "3.9" diff --git a/rust/storage/core/src/service.rs b/rust/storage/core/src/service.rs index f6a942d..4e48853 100644 --- a/rust/storage/core/src/service.rs +++ b/rust/storage/core/src/service.rs @@ -2,6 +2,8 @@ use crate::backend::ManagedLedgerStore; use crate::config::{ManagedLedgerBackendConfig, StorageConfig}; use crate::error::StorageResult; use log::{debug, info}; +use prost::Message; +use pulsar_lite_proto::codec::proto::pulsar::MessageMetadata; use pulsar_lite_storage_managed_ledger::{ CursorInitOptions, CursorOpenResult, ManagedLedgerPosition, ManagedLedgerStorage, MessageId, StoredMessage, @@ -159,6 +161,39 @@ impl Storage { .seek_cursor(topic, subscription, message_id, shared) } + pub fn decode_publish_time(metadata: &[u8]) -> Option { + if metadata.is_empty() { + return None; + } + MessageMetadata::decode(metadata) + .ok() + .map(|m| m.publish_time) + } + + pub fn find_message_id_by_publish_time( + &self, + topic: &str, + publish_time: u64, + ) -> StorageResult> { + let entries = self.get_message_entries(topic); + if entries.is_empty() { + return Ok(None); + } + let mut last_earlier: Option = None; + for (i, entry) in entries.iter().enumerate() { + match Self::decode_publish_time(&entry.metadata) { + Some(pt) if pt < publish_time => last_earlier = Some(i), + Some(_) => break, + None => {} + } + } + let target = match last_earlier { + None => Some(entries[0].message_id.clone()), + Some(i) => entries.get(i + 1).map(|e| e.message_id.clone()), + }; + Ok(target) + } + pub fn first_unacked_position( &self, topic: &str, From 11c4f26db5b1277a98caedc7a2b3b03a2761a424 Mon Sep 17 00:00:00 2001 From: XTLine <2668543450@qq.com> Date: Sat, 8 Aug 2026 22:11:29 +0800 Subject: [PATCH 19/44] refactor(broker): migrate imports to extracted pulsar-lite-proto and storage crates --- rust/proto/build.rs | 5 +---- rust/proto/src/codec.rs | 2 +- rust/src/broker/broker_service.rs | 2 +- rust/src/broker/dispatcher/enums.rs | 2 +- rust/src/broker/dispatcher/exclusive.rs | 2 +- rust/src/broker/dispatcher/failover.rs | 4 ++-- rust/src/broker/dispatcher/key_shared.rs | 11 ++++++----- rust/src/broker/dispatcher/read_position.rs | 2 +- .../src/broker/dispatcher/redelivery_controller.rs | 2 +- rust/src/broker/dispatcher/shared.rs | 5 +++-- rust/src/broker/dispatcher/single_active.rs | 2 +- rust/src/broker/dispatcher/sticky_key.rs | 2 +- rust/src/broker/dispatcher/traits.rs | 2 +- rust/src/broker/handler/connection_handler.rs | 6 +++--- rust/src/broker/handler/consumer_handler.rs | 14 +++++++------- rust/src/broker/handler/lookup_handler.rs | 8 ++++---- rust/src/broker/handler/producer_handler.rs | 6 +++--- rust/src/broker/non_persistent/dispatcher/mod.rs | 2 +- .../dispatcher/multiple_consumers.rs | 5 +++-- .../non_persistent/dispatcher/single_active.rs | 2 +- .../broker/non_persistent/dispatcher/sticky_key.rs | 6 +++--- rust/src/broker/non_persistent/runtime.rs | 2 +- rust/src/broker/service/consumer.rs | 14 +++++++------- rust/src/broker/service/mod.rs | 2 +- rust/src/broker/service/pending_acks.rs | 2 +- .../service/persistent/persistent_subscription.rs | 2 +- .../broker/service/persistent/persistent_topic.rs | 3 ++- rust/src/broker/service/producer.rs | 7 +++++-- rust/src/broker/service/topic/partitioned_topic.rs | 5 ++++- rust/src/broker/service/topic/subscription.rs | 2 +- .../service/topic/tests/partitioned_topic.rs | 2 +- .../topic/tests/subscription_non_persistence.rs | 6 +++--- .../topic/tests/subscription_persistence.rs | 5 +++-- rust/src/broker/service/topic/tests/topic.rs | 8 ++++---- rust/src/broker/service/topic/topic.rs | 13 ++++++++++--- rust/src/main.rs | 6 +++--- rust/storage/core/src/backend.rs | 4 +--- 37 files changed, 94 insertions(+), 81 deletions(-) diff --git a/rust/proto/build.rs b/rust/proto/build.rs index 8e4ab5b..4905ff7 100644 --- a/rust/proto/build.rs +++ b/rust/proto/build.rs @@ -4,10 +4,7 @@ fn main() -> Result<(), Box> { prost_build::Config::new() .out_dir(&out_dir) .protoc_arg("--experimental_allow_proto3_optional") - .compile_protos( - &["PulsarApi.proto", "MLDataFormats.proto"], - &["proto"], - )?; + .compile_protos(&["PulsarApi.proto", "MLDataFormats.proto"], &["proto"])?; Ok(()) } diff --git a/rust/proto/src/codec.rs b/rust/proto/src/codec.rs index a081ea1..bdd0ec8 100644 --- a/rust/proto/src/codec.rs +++ b/rust/proto/src/codec.rs @@ -337,8 +337,8 @@ impl PulsarFrameCodec { #[cfg(test)] mod tests { - use crate::codec::proto::pulsar::{BaseCommand, CompressionType, KeyValue, MessageMetadata}; use super::*; + use crate::codec::proto::pulsar::{BaseCommand, CompressionType, KeyValue, MessageMetadata}; use crate::command::ServerCommand; use bytes::Bytes; use prost::Message; diff --git a/rust/src/broker/broker_service.rs b/rust/src/broker/broker_service.rs index b4dcb97..474075f 100644 --- a/rust/src/broker/broker_service.rs +++ b/rust/src/broker/broker_service.rs @@ -8,7 +8,7 @@ use crate::broker::service::topic::{ PartitionedTopic, PartitionedTopicStats, SharedPartitionedTopic, Topic, TopicPublishRate, TopicStats, }; -use crate::storage::Storage; +use pulsar_lite_storage::Storage; use std::collections::HashMap; use std::sync::Arc; use tokio::sync::{Mutex, RwLock}; diff --git a/rust/src/broker/dispatcher/enums.rs b/rust/src/broker/dispatcher/enums.rs index d43d0b6..d8dc3c1 100644 --- a/rust/src/broker/dispatcher/enums.rs +++ b/rust/src/broker/dispatcher/enums.rs @@ -8,7 +8,7 @@ use super::{ExclusiveDispatcher, FailoverDispatcher, KeySharedDispatcher, Shared use crate::broker::dispatcher::redelivery_controller::RedeliveryEntry; use crate::broker::service::topic::{KeySharedPolicy, SubscriptionType}; use crate::broker::service::{Consumer, SharedStorage}; -use crate::storage::{ManagedLedgerPosition, MessageId}; +use pulsar_lite_storage_managed_ledger::{ManagedLedgerPosition, MessageId}; use std::sync::Arc; /// Dispatcher enum - holds the concrete dispatcher implementation diff --git a/rust/src/broker/dispatcher/exclusive.rs b/rust/src/broker/dispatcher/exclusive.rs index de90be9..e9ae4e1 100644 --- a/rust/src/broker/dispatcher/exclusive.rs +++ b/rust/src/broker/dispatcher/exclusive.rs @@ -10,7 +10,7 @@ use super::read_position::{commit_read_position, next_unacked_candidate}; use crate::broker::dispatcher::Dispatcher; use crate::broker::service::topic::SubscriptionType; use crate::broker::service::{Consumer, SharedStorage}; -use crate::storage::ManagedLedgerPosition; +use pulsar_lite_storage_managed_ledger::ManagedLedgerPosition; use std::sync::atomic::{AtomicU32, Ordering}; use std::sync::{Arc, RwLock}; diff --git a/rust/src/broker/dispatcher/failover.rs b/rust/src/broker/dispatcher/failover.rs index 25b8c3e..a96dd61 100644 --- a/rust/src/broker/dispatcher/failover.rs +++ b/rust/src/broker/dispatcher/failover.rs @@ -10,7 +10,7 @@ use super::rewind_read_position; use crate::broker::dispatcher::Dispatcher; use crate::broker::service::topic::SubscriptionType; use crate::broker::service::{Consumer, SharedStorage}; -use crate::storage::ManagedLedgerPosition; +use pulsar_lite_storage_managed_ledger::ManagedLedgerPosition; use std::sync::atomic::{AtomicU32, Ordering}; use std::sync::{Arc, RwLock}; @@ -325,7 +325,7 @@ impl Dispatcher for FailoverDispatcher { mod tests { use super::*; use crate::broker::service::topic::Subscription; - use crate::storage::Storage; + use pulsar_lite_storage::Storage; use std::path::Path; use tokio::sync::{mpsc, Mutex, RwLock}; diff --git a/rust/src/broker/dispatcher/key_shared.rs b/rust/src/broker/dispatcher/key_shared.rs index 58f7a4e..80d7851 100644 --- a/rust/src/broker/dispatcher/key_shared.rs +++ b/rust/src/broker/dispatcher/key_shared.rs @@ -11,7 +11,7 @@ use crate::broker::service::topic::{ KeySharedHashRange, KeySharedMode, KeySharedPolicy, SubscriptionType, }; use crate::broker::service::{Consumer, SharedStorage}; -use crate::storage::{ManagedLedgerPosition, MessageId}; +use pulsar_lite_storage_managed_ledger::{ManagedLedgerPosition, MessageId}; use std::collections::HashMap; use std::sync::atomic::{AtomicU32, Ordering}; use std::sync::{Arc, RwLock}; @@ -20,7 +20,7 @@ const DISPATCHER_MAX_ROUND_ROBIN_BATCH_SIZE: u32 = 20; type DispatchableRedelivery = Option<( RedeliveryEntry, - crate::storage::StoredMessage, + pulsar_lite_storage_managed_ledger::StoredMessage, Arc, )>; @@ -154,7 +154,7 @@ impl KeySharedDispatcher { fn pop_dispatchable_redelivery_message( &self, - storage: &crate::storage::Storage, + storage: &pulsar_lite_storage::Storage, topic: &str, subscription: &str, ) -> Result> { @@ -543,10 +543,11 @@ mod tests { use super::*; use crate::broker::service::topic::{Subscription, SubscriptionRuntimeMode}; use crate::broker::service::{ConnectionWriteState, PendingMessage}; - use crate::protocol::codec::proto::pulsar::MessageMetadata; - use crate::storage::{CursorInitOptions, InitialPosition, Storage}; use bytes::Bytes; use prost::Message; + use pulsar_lite_proto::codec::proto::pulsar::MessageMetadata; + use pulsar_lite_storage::Storage; + use pulsar_lite_storage_managed_ledger::{CursorInitOptions, InitialPosition}; use std::path::Path; use std::time::Duration; use tokio::sync::{mpsc, Mutex, RwLock}; diff --git a/rust/src/broker/dispatcher/read_position.rs b/rust/src/broker/dispatcher/read_position.rs index 58d87f0..5d982c3 100644 --- a/rust/src/broker/dispatcher/read_position.rs +++ b/rust/src/broker/dispatcher/read_position.rs @@ -1,5 +1,5 @@ use crate::broker::service::SharedStorage; -use crate::storage::{ManagedLedgerPosition, MessageId}; +use pulsar_lite_storage_managed_ledger::{ManagedLedgerPosition, MessageId}; use std::sync::RwLock; pub type DispatchError = Box; diff --git a/rust/src/broker/dispatcher/redelivery_controller.rs b/rust/src/broker/dispatcher/redelivery_controller.rs index dc56c51..72d06bb 100644 --- a/rust/src/broker/dispatcher/redelivery_controller.rs +++ b/rust/src/broker/dispatcher/redelivery_controller.rs @@ -1,4 +1,4 @@ -use crate::storage::MessageId; +use pulsar_lite_storage_managed_ledger::MessageId; use std::collections::{BTreeMap, HashMap}; #[derive(Debug, Clone, PartialEq, Eq)] diff --git a/rust/src/broker/dispatcher/shared.rs b/rust/src/broker/dispatcher/shared.rs index fae8baf..776e556 100644 --- a/rust/src/broker/dispatcher/shared.rs +++ b/rust/src/broker/dispatcher/shared.rs @@ -9,7 +9,7 @@ use super::redelivery_controller::{RedeliveryController, RedeliveryEntry}; use crate::broker::dispatcher::Dispatcher; use crate::broker::service::topic::SubscriptionType; use crate::broker::service::{Consumer, SharedStorage}; -use crate::storage::{ManagedLedgerPosition, MessageId}; +use pulsar_lite_storage_managed_ledger::{ManagedLedgerPosition, MessageId}; use std::collections::HashMap; use std::sync::atomic::{AtomicBool, AtomicU32, AtomicUsize, Ordering}; use std::sync::{Arc, RwLock}; @@ -729,7 +729,8 @@ impl Dispatcher for SharedDispatcher { mod tests { use super::*; use crate::broker::service::topic::Subscription; - use crate::storage::{CursorInitOptions, InitialPosition, Storage}; + use pulsar_lite_storage::Storage; + use pulsar_lite_storage_managed_ledger::{CursorInitOptions, InitialPosition}; use std::path::Path; use tokio::sync::{mpsc, Mutex, RwLock}; diff --git a/rust/src/broker/dispatcher/single_active.rs b/rust/src/broker/dispatcher/single_active.rs index c01964d..d4befbc 100644 --- a/rust/src/broker/dispatcher/single_active.rs +++ b/rust/src/broker/dispatcher/single_active.rs @@ -1,5 +1,5 @@ use crate::broker::service::SharedStorage; -use crate::storage::ManagedLedgerPosition; +use pulsar_lite_storage_managed_ledger::ManagedLedgerPosition; /// Compute the read position a SingleActive dispatcher should rewind to when /// the active consumer disconnects. diff --git a/rust/src/broker/dispatcher/sticky_key.rs b/rust/src/broker/dispatcher/sticky_key.rs index 15fe42d..edc53a7 100644 --- a/rust/src/broker/dispatcher/sticky_key.rs +++ b/rust/src/broker/dispatcher/sticky_key.rs @@ -1,5 +1,5 @@ -use crate::protocol::codec::proto::pulsar::MessageMetadata; use prost::Message; +use pulsar_lite_proto::codec::proto::pulsar::MessageMetadata; const RANGE_SIZE: u32 = 2 << 15; diff --git a/rust/src/broker/dispatcher/traits.rs b/rust/src/broker/dispatcher/traits.rs index 827baa5..e4badf0 100644 --- a/rust/src/broker/dispatcher/traits.rs +++ b/rust/src/broker/dispatcher/traits.rs @@ -6,7 +6,7 @@ use crate::broker::service::topic::SubscriptionType; use crate::broker::service::{Consumer, SharedStorage}; -use crate::storage::ManagedLedgerPosition; +use pulsar_lite_storage_managed_ledger::ManagedLedgerPosition; use std::future::Future; use std::sync::Arc; diff --git a/rust/src/broker/handler/connection_handler.rs b/rust/src/broker/handler/connection_handler.rs index 8387b45..f603192 100644 --- a/rust/src/broker/handler/connection_handler.rs +++ b/rust/src/broker/handler/connection_handler.rs @@ -3,12 +3,12 @@ * Handles connection-level commands: Connect, Ping/Pong */ -use crate::protocol::codec::{ +use futures::SinkExt; +use pulsar_lite_proto::codec::{ proto::pulsar::{BaseCommand, CommandPong}, PulsarFrameCodec, }; -use crate::protocol::ServerCommand; -use futures::SinkExt; +use pulsar_lite_proto::ServerCommand; use tokio_util::codec::Framed; /// Handle Connect command diff --git a/rust/src/broker/handler/consumer_handler.rs b/rust/src/broker/handler/consumer_handler.rs index cb6542c..849510c 100644 --- a/rust/src/broker/handler/consumer_handler.rs +++ b/rust/src/broker/handler/consumer_handler.rs @@ -10,10 +10,10 @@ use crate::broker::service::topic::{ }; use crate::broker::service::ConnectionWriteState; use crate::broker::service::Consumer; -use crate::protocol::codec::{proto::pulsar::BaseCommand, PulsarFrameCodec}; -use crate::protocol::ServerCommand; -use crate::storage::{CursorInitOptions, InitialPosition, MessageId}; use futures::SinkExt; +use pulsar_lite_proto::codec::{proto::pulsar::BaseCommand, PulsarFrameCodec}; +use pulsar_lite_proto::ServerCommand; +use pulsar_lite_storage_managed_ledger::{CursorInitOptions, InitialPosition, MessageId}; use std::collections::HashMap; use std::sync::Arc; use tokio::sync::mpsc; @@ -585,13 +585,13 @@ mod tests { use crate::broker::broker_service::{BrokerService, TopicRef}; use crate::broker::service::topic::{KeySharedMode, TopicRuntimeMode}; use crate::broker::service::SharedStorage; - use crate::protocol::codec::proto::pulsar::{ + use futures::StreamExt; + use prost::Message; + use pulsar_lite_proto::codec::proto::pulsar::{ base_command, CommandAck, CommandSubscribe, IntRange, KeySharedMeta, KeyValue, MessageIdData, }; - use crate::storage::Storage; - use futures::StreamExt; - use prost::Message; + use pulsar_lite_storage::Storage; use std::time::{SystemTime, UNIX_EPOCH}; use tokio::io::duplex; use tokio::sync::{Mutex, RwLock}; diff --git a/rust/src/broker/handler/lookup_handler.rs b/rust/src/broker/handler/lookup_handler.rs index 92ce922..00ba859 100644 --- a/rust/src/broker/handler/lookup_handler.rs +++ b/rust/src/broker/handler/lookup_handler.rs @@ -4,9 +4,9 @@ */ use crate::broker::SharedBrokerService; -use crate::protocol::codec::{proto::pulsar::BaseCommand, PulsarFrameCodec}; -use crate::protocol::ServerCommand; use futures::SinkExt; +use pulsar_lite_proto::codec::{proto::pulsar::BaseCommand, PulsarFrameCodec}; +use pulsar_lite_proto::ServerCommand; use tokio_util::codec::Framed; /// Handle PartitionMetadata command @@ -85,10 +85,10 @@ where mod tests { use super::*; use crate::broker::broker_service::BrokerService; - use crate::protocol::codec::proto::pulsar::{ + use pulsar_lite_proto::codec::proto::pulsar::{ base_command, CommandLookupTopic, CommandPartitionedTopicMetadata, }; - use crate::storage::Storage; + use pulsar_lite_storage::Storage; use std::path::Path; use std::sync::Arc; use std::time::Instant; diff --git a/rust/src/broker/handler/producer_handler.rs b/rust/src/broker/handler/producer_handler.rs index 13d5bb5..c02cd89 100644 --- a/rust/src/broker/handler/producer_handler.rs +++ b/rust/src/broker/handler/producer_handler.rs @@ -6,9 +6,9 @@ use crate::broker::broker_service::{SharedBrokerService, TopicRef}; use crate::broker::service::topic::TopicRuntimeMode; use crate::broker::service::Producer; -use crate::protocol::codec::{proto::pulsar::BaseCommand, PulsarFrame, PulsarFrameCodec}; -use crate::protocol::ServerCommand; use futures::SinkExt; +use pulsar_lite_proto::codec::{proto::pulsar::BaseCommand, PulsarFrame, PulsarFrameCodec}; +use pulsar_lite_proto::ServerCommand; use std::collections::HashMap; use std::sync::Arc; use tokio_util::codec::Framed; @@ -97,7 +97,7 @@ where pub struct PublishedSend { pub producer_id: u64, pub sequence_id: u64, - pub message_id: crate::storage::MessageId, + pub message_id: pulsar_lite_storage_managed_ledger::MessageId, } /// Publish a Send command (storage + dispatch) without writing SendReceipt. diff --git a/rust/src/broker/non_persistent/dispatcher/mod.rs b/rust/src/broker/non_persistent/dispatcher/mod.rs index 624b4d6..f2e4a5f 100644 --- a/rust/src/broker/non_persistent/dispatcher/mod.rs +++ b/rust/src/broker/non_persistent/dispatcher/mod.rs @@ -11,7 +11,7 @@ mod sticky_key; use crate::broker::service::topic::{KeySharedPolicy, SubscriptionType}; use crate::broker::service::Consumer; -use crate::storage::NonPersistentEntry; +use pulsar_lite_storage_managed_ledger::NonPersistentEntry; use std::sync::Arc; pub use multiple_consumers::NonPersistentDispatcherMultipleConsumers; diff --git a/rust/src/broker/non_persistent/dispatcher/multiple_consumers.rs b/rust/src/broker/non_persistent/dispatcher/multiple_consumers.rs index 6175e62..46ab633 100644 --- a/rust/src/broker/non_persistent/dispatcher/multiple_consumers.rs +++ b/rust/src/broker/non_persistent/dispatcher/multiple_consumers.rs @@ -1,6 +1,6 @@ use crate::broker::service::topic::SubscriptionType; use crate::broker::service::Consumer; -use crate::storage::{MessageId, NonPersistentEntry}; +use pulsar_lite_storage_managed_ledger::{MessageId, NonPersistentEntry}; use std::collections::HashMap; use std::sync::{ atomic::{AtomicU32, AtomicU64, AtomicUsize, Ordering}, @@ -184,8 +184,9 @@ mod tests { use super::*; use crate::broker::service::topic::Subscription; use crate::broker::service::SharedStorage; - use crate::storage::{NonPersistentEntry, Storage}; use bytes::Bytes; + use pulsar_lite_storage::Storage; + use pulsar_lite_storage_managed_ledger::NonPersistentEntry; use std::path::Path; use std::sync::Arc; use std::time::Instant; diff --git a/rust/src/broker/non_persistent/dispatcher/single_active.rs b/rust/src/broker/non_persistent/dispatcher/single_active.rs index 452cd88..0ce8094 100644 --- a/rust/src/broker/non_persistent/dispatcher/single_active.rs +++ b/rust/src/broker/non_persistent/dispatcher/single_active.rs @@ -1,6 +1,6 @@ use crate::broker::service::topic::SubscriptionType; use crate::broker::service::Consumer; -use crate::storage::{MessageId, NonPersistentEntry}; +use pulsar_lite_storage_managed_ledger::{MessageId, NonPersistentEntry}; use std::sync::{ atomic::{AtomicU64, Ordering}, Arc, diff --git a/rust/src/broker/non_persistent/dispatcher/sticky_key.rs b/rust/src/broker/non_persistent/dispatcher/sticky_key.rs index 002191a..220f1da 100644 --- a/rust/src/broker/non_persistent/dispatcher/sticky_key.rs +++ b/rust/src/broker/non_persistent/dispatcher/sticky_key.rs @@ -5,7 +5,7 @@ use crate::broker::service::topic::{ KeySharedHashRange, KeySharedMode, KeySharedPolicy, SubscriptionType, }; use crate::broker::service::Consumer; -use crate::storage::{MessageId, NonPersistentEntry}; +use pulsar_lite_storage_managed_ledger::{MessageId, NonPersistentEntry}; use std::collections::HashMap; use std::sync::{ atomic::{AtomicU32, AtomicU64, Ordering}, @@ -263,10 +263,10 @@ impl NonPersistentStickyKeyDispatcher { mod tests { use super::*; use crate::broker::service::topic::{Subscription, SubscriptionRuntimeMode}; - use crate::protocol::codec::proto::pulsar::MessageMetadata; - use crate::storage::Storage; use bytes::Bytes; use prost::Message; + use pulsar_lite_proto::codec::proto::pulsar::MessageMetadata; + use pulsar_lite_storage::Storage; use std::path::Path; use std::time::Instant; use tokio::sync::{mpsc, Mutex, RwLock}; diff --git a/rust/src/broker/non_persistent/runtime.rs b/rust/src/broker/non_persistent/runtime.rs index 927e812..c3a1c8d 100644 --- a/rust/src/broker/non_persistent/runtime.rs +++ b/rust/src/broker/non_persistent/runtime.rs @@ -9,7 +9,7 @@ use crate::broker::non_persistent::NonPersistentDispatcherEnum; use crate::broker::service::topic::{KeySharedPolicy, SubscriptionType}; use crate::broker::service::Consumer; -use crate::storage::NonPersistentEntry; +use pulsar_lite_storage_managed_ledger::NonPersistentEntry; use std::collections::HashMap; use std::sync::Arc; diff --git a/rust/src/broker/service/consumer.rs b/rust/src/broker/service/consumer.rs index 081baea..59f370d 100644 --- a/rust/src/broker/service/consumer.rs +++ b/rust/src/broker/service/consumer.rs @@ -15,7 +15,7 @@ use tokio::sync::{mpsc, RwLock}; use super::topic::KeySharedPolicy; use super::topic::{AckCommandType, Subscription, SubscriptionType}; -use crate::storage::MessageId; +use pulsar_lite_storage_managed_ledger::MessageId; /// Consumer statistics #[derive(Debug, Default, Clone)] @@ -359,7 +359,7 @@ impl Consumer { { let metadata = metadata.into(); let payload = payload.into(); - let wire_size = crate::protocol::codec::estimate_message_parts_size( + let wire_size = pulsar_lite_proto::codec::estimate_message_parts_size( self.consumer_id, message_id.ledger, message_id.entry, @@ -436,7 +436,7 @@ impl Consumer { } // Step 2: Calculate wire size - let wire_size = crate::protocol::codec::estimate_message_parts_size( + let wire_size = pulsar_lite_proto::codec::estimate_message_parts_size( self.consumer_id, message_id.ledger, message_id.entry, @@ -665,7 +665,7 @@ impl Consumer { } /// Record consumer-level ack stats (Exclusive/Failover path). - pub async fn ack_message(&self, message_id: crate::storage::MessageId) { + pub async fn ack_message(&self, message_id: pulsar_lite_storage_managed_ledger::MessageId) { log::debug!( "Consumer {} acking message {}:{}", self.consumer_id, @@ -828,7 +828,7 @@ mod tests { use super::super::topic::{Subscription, SubscriptionRuntimeMode}; use super::super::{ConnectionWriteState, SharedStorage}; use super::*; - use crate::storage::Storage; + use pulsar_lite_storage::Storage; use std::path::Path; use tokio::sync::Mutex; @@ -931,7 +931,7 @@ mod tests { assert_eq!(consumer.get_available_permits().await, 10); // Test ack - let msg_id = crate::storage::MessageId { + let msg_id = pulsar_lite_storage_managed_ledger::MessageId { ledger: 1, entry: 1, partition: -1, @@ -1097,7 +1097,7 @@ mod tests { .await ); - let expected_wire_size = crate::protocol::codec::estimate_message_parts_size( + let expected_wire_size = pulsar_lite_proto::codec::estimate_message_parts_size( consumer.consumer_id, 1, 1, diff --git a/rust/src/broker/service/mod.rs b/rust/src/broker/service/mod.rs index c724e3f..c32cdc7 100644 --- a/rust/src/broker/service/mod.rs +++ b/rust/src/broker/service/mod.rs @@ -4,7 +4,7 @@ * Inspired by Apache Pulsar's service structure */ -use crate::storage::Storage; +use pulsar_lite_storage::Storage; use std::sync::Arc; use tokio::sync::Mutex; diff --git a/rust/src/broker/service/pending_acks.rs b/rust/src/broker/service/pending_acks.rs index 6aff968..6098ae2 100644 --- a/rust/src/broker/service/pending_acks.rs +++ b/rust/src/broker/service/pending_acks.rs @@ -1,4 +1,4 @@ -use crate::storage::MessageId; +use pulsar_lite_storage_managed_ledger::MessageId; use std::collections::BTreeMap; use std::sync::atomic::{AtomicBool, Ordering}; use std::time::Instant; diff --git a/rust/src/broker/service/persistent/persistent_subscription.rs b/rust/src/broker/service/persistent/persistent_subscription.rs index 4b82a7e..42544af 100644 --- a/rust/src/broker/service/persistent/persistent_subscription.rs +++ b/rust/src/broker/service/persistent/persistent_subscription.rs @@ -2,7 +2,7 @@ use crate::broker::dispatcher::redelivery_controller::RedeliveryEntry; use crate::broker::dispatcher::DispatcherEnum; use crate::broker::service::topic::{KeySharedPolicy, SubscriptionType}; use crate::broker::service::{Consumer, SharedStorage}; -use crate::storage::{ManagedLedgerPosition, MessageId}; +use pulsar_lite_storage_managed_ledger::{ManagedLedgerPosition, MessageId}; use std::sync::Arc; #[derive(Debug)] diff --git a/rust/src/broker/service/persistent/persistent_topic.rs b/rust/src/broker/service/persistent/persistent_topic.rs index c39f491..c9c2aa9 100644 --- a/rust/src/broker/service/persistent/persistent_topic.rs +++ b/rust/src/broker/service/persistent/persistent_topic.rs @@ -1,6 +1,7 @@ use crate::broker::service::SharedStorage; -use crate::storage::{CursorInitOptions, CursorOpenResult, MessageId, Storage}; use bytes::Bytes; +use pulsar_lite_storage::Storage; +use pulsar_lite_storage_managed_ledger::{CursorInitOptions, CursorOpenResult, MessageId}; #[derive(Debug, Clone)] pub(crate) struct PersistentTopicRuntime { diff --git a/rust/src/broker/service/producer.rs b/rust/src/broker/service/producer.rs index 8ea79b1..febd48e 100644 --- a/rust/src/broker/service/producer.rs +++ b/rust/src/broker/service/producer.rs @@ -80,7 +80,10 @@ impl Producer { &self, metadata: Option, payload: Bytes, - ) -> Result> { + ) -> Result< + pulsar_lite_storage_managed_ledger::MessageId, + Box, + > { log::debug!( "Producer {} publishing message (metadata={} bytes, payload={} bytes)", self.producer_id, @@ -189,7 +192,7 @@ impl std::hash::Hash for Producer { mod tests { use super::super::topic::Topic; use super::*; - use crate::storage::Storage; + use pulsar_lite_storage::Storage; use std::path::Path; use std::sync::Arc as StdArc; use tokio::sync::Mutex; diff --git a/rust/src/broker/service/topic/partitioned_topic.rs b/rust/src/broker/service/topic/partitioned_topic.rs index def5a32..510dbe4 100644 --- a/rust/src/broker/service/topic/partitioned_topic.rs +++ b/rust/src/broker/service/topic/partitioned_topic.rs @@ -229,7 +229,10 @@ impl PartitionedTopic { partition_index: usize, metadata: Option, payload: Bytes, - ) -> Result> { + ) -> Result< + pulsar_lite_storage_managed_ledger::MessageId, + Box, + > { if partition_index >= self.partition_count { return Err(format!("Partition index {} out of bounds", partition_index).into()); } diff --git a/rust/src/broker/service/topic/subscription.rs b/rust/src/broker/service/topic/subscription.rs index 6df6166..b34bc03 100644 --- a/rust/src/broker/service/topic/subscription.rs +++ b/rust/src/broker/service/topic/subscription.rs @@ -12,7 +12,7 @@ use super::super::{Consumer, SharedStorage}; use crate::broker::dispatcher::redelivery_controller::RedeliveryEntry; use crate::broker::non_persistent::NonPersistentSubscriptionRuntime; use crate::broker::service::persistent::PersistentSubscriptionRuntime; -use crate::storage::{ManagedLedgerPosition, MessageId, NonPersistentEntry, StorageSeekExt}; +use pulsar_lite_storage_managed_ledger::{ManagedLedgerPosition, MessageId, NonPersistentEntry}; /// Subscription type (matches Pulsar protocol) #[derive(Debug, Clone, Copy, PartialEq, Default)] diff --git a/rust/src/broker/service/topic/tests/partitioned_topic.rs b/rust/src/broker/service/topic/tests/partitioned_topic.rs index e80a66b..0774676 100644 --- a/rust/src/broker/service/topic/tests/partitioned_topic.rs +++ b/rust/src/broker/service/topic/tests/partitioned_topic.rs @@ -1,6 +1,6 @@ use crate::broker::service::topic::{PartitionedTopic, Topic}; use crate::broker::service::{Producer, SharedStorage}; -use crate::storage::Storage; +use pulsar_lite_storage::Storage; use std::path::Path; use std::sync::Arc; use tokio::sync::{Mutex, RwLock}; diff --git a/rust/src/broker/service/topic/tests/subscription_non_persistence.rs b/rust/src/broker/service/topic/tests/subscription_non_persistence.rs index 309f34b..27b15ab 100644 --- a/rust/src/broker/service/topic/tests/subscription_non_persistence.rs +++ b/rust/src/broker/service/topic/tests/subscription_non_persistence.rs @@ -2,9 +2,9 @@ use crate::broker::service::topic::{ KeySharedMode, KeySharedPolicy, Subscription, SubscriptionRuntimeMode, SubscriptionType, }; use crate::broker::service::{Consumer, PendingMessage, SharedStorage}; -use crate::storage::MessageId; -use crate::storage::NonPersistentEntry; -use crate::storage::Storage; +use pulsar_lite_storage::Storage; +use pulsar_lite_storage_managed_ledger::MessageId; +use pulsar_lite_storage_managed_ledger::NonPersistentEntry; use std::collections::HashMap; use std::path::Path; use std::sync::Arc; diff --git a/rust/src/broker/service/topic/tests/subscription_persistence.rs b/rust/src/broker/service/topic/tests/subscription_persistence.rs index f65a428..5db499c 100644 --- a/rust/src/broker/service/topic/tests/subscription_persistence.rs +++ b/rust/src/broker/service/topic/tests/subscription_persistence.rs @@ -1,6 +1,7 @@ use crate::broker::service::topic::{Subscription, SubscriptionType}; use crate::broker::service::{Consumer, PendingMessage, SharedStorage}; -use crate::storage::{CursorInitOptions, InitialPosition, Storage}; +use pulsar_lite_storage::Storage; +use pulsar_lite_storage_managed_ledger::{CursorInitOptions, InitialPosition}; use std::path::Path; use std::sync::Arc; use tempfile::tempdir; @@ -371,8 +372,8 @@ async fn persistent_seek_rewinds_dispatch_to_target_message() { } fn metadata_with_publish_time(publish_time: u64) -> Vec { - use crate::protocol::codec::proto::pulsar::MessageMetadata; use prost::Message; + use pulsar_lite_proto::codec::proto::pulsar::MessageMetadata; MessageMetadata { publish_time, ..Default::default() diff --git a/rust/src/broker/service/topic/tests/topic.rs b/rust/src/broker/service/topic/tests/topic.rs index 9dee6f1..cc1990e 100644 --- a/rust/src/broker/service/topic/tests/topic.rs +++ b/rust/src/broker/service/topic/tests/topic.rs @@ -3,12 +3,12 @@ use crate::broker::service::topic::{ TopicRuntimeMode, }; use crate::broker::service::{Consumer, Producer, SharedStorage}; -use crate::protocol::codec::proto::pulsar::MessageMetadata; -use crate::storage::Storage; -#[cfg(feature = "rocksdb-storage")] -use crate::storage::{CursorInitOptions, InitialPosition}; use bytes::Bytes; use prost::Message; +use pulsar_lite_proto::codec::proto::pulsar::MessageMetadata; +use pulsar_lite_storage::Storage; +#[cfg(feature = "rocksdb-storage")] +use pulsar_lite_storage_managed_ledger::{CursorInitOptions, InitialPosition}; use std::path::Path; use std::sync::{Arc, Arc as StdArc}; use std::time::Instant; diff --git a/rust/src/broker/service/topic/topic.rs b/rust/src/broker/service/topic/topic.rs index 81b83d5..b70b702 100644 --- a/rust/src/broker/service/topic/topic.rs +++ b/rust/src/broker/service/topic/topic.rs @@ -9,7 +9,9 @@ use super::{ use crate::broker::service::persistent::PersistentTopicRuntime; use crate::broker::service::{Consumer, Producer, SharedStorage}; -use crate::storage::{parse_topic_name, CursorInitOptions, MessageId, NonPersistentEntry}; +use pulsar_lite_storage_managed_ledger::{CursorInitOptions, MessageId, NonPersistentEntry}; +use pulsar_lite_storage_metadata::parse_topic_name; + use bytes::Bytes; use std::collections::HashMap; use std::fmt; @@ -502,7 +504,10 @@ impl Topic { &mut self, metadata: Option, payload: Bytes, - ) -> Result> { + ) -> Result< + pulsar_lite_storage_managed_ledger::MessageId, + Box, + > { log::debug!( "Publishing message to topic '{}' partition '{}' (metadata={} bytes, payload={} bytes)", self.name, @@ -613,7 +618,9 @@ impl Topic { self.producers.is_empty() && self.get_total_consumer_count().await == 0 } - pub async fn get_last_message_id(&self) -> Result, String> { + pub async fn get_last_message_id( + &self, + ) -> Result, String> { match self.runtime_mode { TopicRuntimeMode::Persistent => { self.persistent_runtime diff --git a/rust/src/main.rs b/rust/src/main.rs index 478832d..71f6bea 100644 --- a/rust/src/main.rs +++ b/rust/src/main.rs @@ -9,9 +9,9 @@ use pulsar_lite::broker::handle_connection; use pulsar_lite::broker::service::topic::TopicPublishRate; use pulsar_lite::broker::{BrokerService, ConnectionLimiter}; use pulsar_lite::config::Config; -use pulsar_lite::protocol::codec::PulsarFrameCodec; -use pulsar_lite::protocol::ServerCommand; -use pulsar_lite::storage::Storage; +use pulsar_lite_proto::codec::PulsarFrameCodec; +use pulsar_lite_proto::ServerCommand; +use pulsar_lite_storage::Storage; use std::net::{IpAddr, Ipv4Addr, SocketAddr}; use std::path::PathBuf; use std::sync::Arc; diff --git a/rust/storage/core/src/backend.rs b/rust/storage/core/src/backend.rs index b56f361..51a8b1e 100644 --- a/rust/storage/core/src/backend.rs +++ b/rust/storage/core/src/backend.rs @@ -4,9 +4,7 @@ use pulsar_lite_storage_managed_ledger::{ ManagedLedgerStorage, MessageId, StoredMessage, }; #[cfg(feature = "rocksdb-storage")] -use pulsar_lite_storage_managed_ledger_rocksdb::{ - ConcurrentAppender, RocksDbManagedLedgerStorage, -}; +use pulsar_lite_storage_managed_ledger_rocksdb::{ConcurrentAppender, RocksDbManagedLedgerStorage}; #[cfg(feature = "rocksdb-storage")] use std::path::Path; From e5213af83ca533e2ee2a75c124fc2310816bce0f Mon Sep 17 00:00:00 2001 From: XTLine <2668543450@qq.com> Date: Sat, 8 Aug 2026 22:38:22 +0800 Subject: [PATCH 20/44] style: cargo fmt workspace formatting --- .../managed-ledger-rocksdb/src/entrylog.rs | 10 +++------- .../managed-ledger-rocksdb/src/ledger.rs | 18 +++++++++--------- rust/storage/managed-ledger-rocksdb/src/lib.rs | 7 +++++-- .../tests/rocksdb_cursor.rs | 11 +++++++---- .../tests/rocksdb_entrylog.rs | 2 -- .../tests/rocksdb_keys.rs | 3 +-- .../tests/rocksdb_ledger.rs | 14 +++++++++----- .../tests/rocksdb_storage.rs | 8 ++++---- 8 files changed, 38 insertions(+), 35 deletions(-) diff --git a/rust/storage/managed-ledger-rocksdb/src/entrylog.rs b/rust/storage/managed-ledger-rocksdb/src/entrylog.rs index dc8d41c..071f40c 100644 --- a/rust/storage/managed-ledger-rocksdb/src/entrylog.rs +++ b/rust/storage/managed-ledger-rocksdb/src/entrylog.rs @@ -164,9 +164,8 @@ impl WriteState { let checksum = EntryLogStore::checksum(&[&entry.metadata, &entry.payload]); let metadata_len = entry.metadata.len() as u32; let payload_len = entry.payload.len() as u32; - let len = ENTRY_HEADER_LEN as u64 - + entry.metadata.len() as u64 - + entry.payload.len() as u64; + let len = + ENTRY_HEADER_LEN as u64 + entry.metadata.len() as u64 + entry.payload.len() as u64; pending.push(EntryIndex { ledger_id: entry.ledger_id, @@ -386,10 +385,7 @@ impl EntryLogStore { .as_ref() .ok_or_else(|| anyhow!("entrylog writer is closed"))?; sender - .send(WriterMsg::Batch { - entries, - reply: tx, - }) + .send(WriterMsg::Batch { entries, reply: tx }) .map_err(|_| anyhow!("entrylog writer disconnected"))?; rx.recv() .map_err(|_| anyhow!("entrylog writer disconnected"))? diff --git a/rust/storage/managed-ledger-rocksdb/src/ledger.rs b/rust/storage/managed-ledger-rocksdb/src/ledger.rs index 49f2d1f..4f80921 100644 --- a/rust/storage/managed-ledger-rocksdb/src/ledger.rs +++ b/rust/storage/managed-ledger-rocksdb/src/ledger.rs @@ -397,15 +397,15 @@ impl RocksDBManagedLedger { pub fn message_entries(&self) -> Result> { let info = self.info.load(); - let Some(from) = info - .ledgers - .iter() - .find(|l| l.entries > 0) - .map(|l| ManagedLedgerPosition { - ledger_id: l.ledger_id, - entry_id: 0, - partition: -1, - }) + let Some(from) = + info.ledgers + .iter() + .find(|l| l.entries > 0) + .map(|l| ManagedLedgerPosition { + ledger_id: l.ledger_id, + entry_id: 0, + partition: -1, + }) else { return Ok(Vec::new()); }; diff --git a/rust/storage/managed-ledger-rocksdb/src/lib.rs b/rust/storage/managed-ledger-rocksdb/src/lib.rs index e00575f..6a6668d 100644 --- a/rust/storage/managed-ledger-rocksdb/src/lib.rs +++ b/rust/storage/managed-ledger-rocksdb/src/lib.rs @@ -46,8 +46,11 @@ pub mod test_support { partition: i32, payload: &[u8], ) -> Result { - let mut positions = - ledger.add_entries_with_partition_and_metadata(&[(partition, &[] as &[u8], payload)])?; + let mut positions = ledger.add_entries_with_partition_and_metadata(&[( + partition, + &[] as &[u8], + payload, + )])?; positions .pop() .ok_or_else(|| anyhow::anyhow!("add_entries returned empty positions")) diff --git a/rust/storage/managed-ledger-rocksdb/tests/rocksdb_cursor.rs b/rust/storage/managed-ledger-rocksdb/tests/rocksdb_cursor.rs index dcd69d1..e5e66ed 100644 --- a/rust/storage/managed-ledger-rocksdb/tests/rocksdb_cursor.rs +++ b/rust/storage/managed-ledger-rocksdb/tests/rocksdb_cursor.rs @@ -8,8 +8,9 @@ use pulsar_lite_storage_managed_ledger::{ ManagedLedgerConfig, ManagedLedgerFactory, ManagedLedgerPosition, }; use pulsar_lite_storage_managed_ledger_rocksdb::test_support::{ - ack_managed_cursor_shared, is_managed_position_acknowledged, RocksDBManagedCursor, - append_payload, append_with_partition, RocksDBManagedLedger, RocksDBManagedLedgerFactory, + ack_managed_cursor_shared, append_payload, append_with_partition, + is_managed_position_acknowledged, RocksDBManagedCursor, RocksDBManagedLedger, + RocksDBManagedLedgerFactory, }; use std::sync::Arc; use tempfile::tempdir; @@ -97,7 +98,8 @@ fn shared_ack_advances_contiguously_across_rolled_ledgers() { let third = append_payload(&ledger, b"third").unwrap(); let mut cursor = ledger.open_cursor("sub-a").unwrap(); - ack_managed_cursor_shared(&mut cursor, third.clone(), &ledger.info_snapshot().as_ref()).unwrap(); + ack_managed_cursor_shared(&mut cursor, third.clone(), &ledger.info_snapshot().as_ref()) + .unwrap(); assert_eq!(cursor.state().mark_delete, None); assert!(cursor.state().individually_deleted_entries.contains(&third)); @@ -268,7 +270,8 @@ fn shared_ack_normalizes_partition_when_advancing_mark_delete() { let mut cursor = ledger.open_cursor("sub-a").unwrap(); // First, reorder ACK item 'second'. It should enter the individual collection - ack_managed_cursor_shared(&mut cursor, second.clone(), ledger.info_snapshot().as_ref()).unwrap(); + ack_managed_cursor_shared(&mut cursor, second.clone(), ledger.info_snapshot().as_ref()) + .unwrap(); assert_eq!(cursor.state().mark_delete, None); assert!(cursor diff --git a/rust/storage/managed-ledger-rocksdb/tests/rocksdb_entrylog.rs b/rust/storage/managed-ledger-rocksdb/tests/rocksdb_entrylog.rs index 86726d4..f43d3e9 100644 --- a/rust/storage/managed-ledger-rocksdb/tests/rocksdb_entrylog.rs +++ b/rust/storage/managed-ledger-rocksdb/tests/rocksdb_entrylog.rs @@ -4,7 +4,6 @@ use pulsar_lite_storage_managed_ledger_rocksdb::test_support::{EntryLogStore, En use std::fs; use tempfile::tempdir; - #[test] fn entrylog_appends_and_reads_entry_payload() { let dir = tempdir().unwrap(); @@ -296,4 +295,3 @@ fn entrylog_append_batch_then_single_append_continues_offsets() { assert_eq!(single.offset, batch[1].offset + batch[1].len); assert_eq!(store.read(&single).unwrap().payload, b"c"); } - diff --git a/rust/storage/managed-ledger-rocksdb/tests/rocksdb_keys.rs b/rust/storage/managed-ledger-rocksdb/tests/rocksdb_keys.rs index e627170..fe747c0 100644 --- a/rust/storage/managed-ledger-rocksdb/tests/rocksdb_keys.rs +++ b/rust/storage/managed-ledger-rocksdb/tests/rocksdb_keys.rs @@ -36,8 +36,7 @@ fn managed_ledger_info_value_is_protobuf_encoded() { { let entry_log = open_test_entry_log(&db_path); - let ledger = - RocksDBManagedLedger::open("ledger-a", Arc::clone(&db), entry_log).unwrap(); + let ledger = RocksDBManagedLedger::open("ledger-a", Arc::clone(&db), entry_log).unwrap(); append_payload(&ledger, b"first").unwrap(); } diff --git a/rust/storage/managed-ledger-rocksdb/tests/rocksdb_ledger.rs b/rust/storage/managed-ledger-rocksdb/tests/rocksdb_ledger.rs index 2ea93b2..9350eab 100644 --- a/rust/storage/managed-ledger-rocksdb/tests/rocksdb_ledger.rs +++ b/rust/storage/managed-ledger-rocksdb/tests/rocksdb_ledger.rs @@ -3,10 +3,12 @@ mod common; use common::*; -use pulsar_lite_storage_managed_ledger::{ManagedLedger, ManagedLedgerConfig, ManagedLedgerFactory}; +use pulsar_lite_storage_managed_ledger::{ + ManagedLedger, ManagedLedgerConfig, ManagedLedgerFactory, +}; use pulsar_lite_storage_managed_ledger_rocksdb::test_support::{ - append_payload, append_with_partition, keys, RocksDBManagedLedger, - RocksDBManagedLedgerFactory, StoredEntryLocation, StoredManagedLedgerInfo, + append_payload, append_with_partition, keys, RocksDBManagedLedger, RocksDBManagedLedgerFactory, + StoredEntryLocation, StoredManagedLedgerInfo, }; use std::sync::Arc; use tempfile::tempdir; @@ -432,7 +434,8 @@ fn message_ledger_append_updates_runtime_last_position() { let db = open_test_db(&db_path); let entry_log = open_test_entry_log(&db_path); - let ledger = RocksDBManagedLedger::open("ledger-a", Arc::clone(&db), Arc::clone(&entry_log)).unwrap(); + let ledger = + RocksDBManagedLedger::open("ledger-a", Arc::clone(&db), Arc::clone(&entry_log)).unwrap(); let first = append_with_partition(&ledger, 7, b"first").unwrap(); @@ -442,7 +445,8 @@ fn message_ledger_append_updates_runtime_last_position() { assert_eq!(ledger.last_position().unwrap(), Some(first)); // Consecutive append - advancing to the last position in the runtime. - let ledger = RocksDBManagedLedger::open("ledger-b", Arc::clone(&db), Arc::clone(&entry_log)).unwrap(); + let ledger = + RocksDBManagedLedger::open("ledger-b", Arc::clone(&db), Arc::clone(&entry_log)).unwrap(); let first = append_with_partition(&ledger, 7, b"first").unwrap(); let second = append_with_partition(&ledger, 7, b"second").unwrap(); diff --git a/rust/storage/managed-ledger-rocksdb/tests/rocksdb_storage.rs b/rust/storage/managed-ledger-rocksdb/tests/rocksdb_storage.rs index 946dd8d..3f22302 100644 --- a/rust/storage/managed-ledger-rocksdb/tests/rocksdb_storage.rs +++ b/rust/storage/managed-ledger-rocksdb/tests/rocksdb_storage.rs @@ -137,7 +137,9 @@ fn append_after_latest_cursor_is_visible_to_reads() { "empty topic with Latest cursor should have no backlog" ); - let message_id = storage.append_message(topic, -1, b"hello-after-subscribe").unwrap(); + let message_id = storage + .append_message(topic, -1, b"hello-after-subscribe") + .unwrap(); let first = storage .first_unacked_position(topic, subscription) @@ -146,9 +148,7 @@ fn append_after_latest_cursor_is_visible_to_reads() { assert_eq!(first.ledger_id, message_id.ledger); assert_eq!(first.entry_id, message_id.entry); - let entries = storage - .read_entries_from(topic, &first, 1) - .unwrap(); + let entries = storage.read_entries_from(topic, &first, 1).unwrap(); assert_eq!(entries.len(), 1); assert_eq!(entries[0].message_id, message_id); assert_eq!(entries[0].payload, b"hello-after-subscribe"); From 5da74d66aebf563c4dc54c373fc5f591da55848c Mon Sep 17 00:00:00 2001 From: XTLine <2668543450@qq.com> Date: Wed, 12 Aug 2026 11:17:23 +0800 Subject: [PATCH 21/44] perf(broker): decouple dispatch from send-receipt path --- rust/src/broker/handler/producer_handler.rs | 9 +++------ rust/src/broker/service/server_cnx.rs | 6 ++---- rust/src/broker/service/topic/topic.rs | 13 ++++++++++++- 3 files changed, 17 insertions(+), 11 deletions(-) diff --git a/rust/src/broker/handler/producer_handler.rs b/rust/src/broker/handler/producer_handler.rs index c02cd89..536ad39 100644 --- a/rust/src/broker/handler/producer_handler.rs +++ b/rust/src/broker/handler/producer_handler.rs @@ -3,6 +3,7 @@ * Handles producer-related commands: Producer, Send, CloseProducer */ +use crate::broker::Topic; use crate::broker::broker_service::{SharedBrokerService, TopicRef}; use crate::broker::service::topic::TopicRuntimeMode; use crate::broker::service::Producer; @@ -150,9 +151,7 @@ pub async fn publish_send( ); { - let topic = producer.get_topic(); - let mut topic_guard = topic.write().await; - topic_guard.dispatch_to_subscriptions().await; + Topic::spawn_dispatcher(producer.get_topic()); } message_id }; @@ -181,9 +180,7 @@ pub async fn publish_persistent_send( producer.get_topic_name() ); { - let topic = producer.get_topic(); - let mut topic_guard = topic.write().await; - topic_guard.dispatch_to_subscriptions().await; + Topic::spawn_dispatcher(producer.get_topic()); } Ok(PublishedSend { producer_id, diff --git a/rust/src/broker/service/server_cnx.rs b/rust/src/broker/service/server_cnx.rs index 662091e..80599f2 100644 --- a/rust/src/broker/service/server_cnx.rs +++ b/rust/src/broker/service/server_cnx.rs @@ -16,7 +16,7 @@ use tokio_util::codec::Framed; use super::consumer::PendingMessage; use super::{ConnectionWriteState, Consumer, Producer, SharedStorage}; use crate::broker::broker_service::SharedBrokerService; -use crate::broker::handler; +use crate::broker::{Topic, handler}; use crate::broker::service::topic::TopicPublishRateExceeded; use pulsar_lite_proto::codec::{ proto::pulsar::{base_command, BaseCommand, ProtocolVersion, ServerError}, @@ -762,9 +762,7 @@ where .await .map_err(to_cnx_error)?; if let Some(producer) = self.producers.get(&outcome.producer_id) { - let topic = producer.get_topic(); - let mut topic_guard = topic.write().await; - topic_guard.dispatch_to_subscriptions().await; + Topic::spawn_dispatcher(producer.get_topic()); } } Err(message) => { diff --git a/rust/src/broker/service/topic/topic.rs b/rust/src/broker/service/topic/topic.rs index b70b702..1881127 100644 --- a/rust/src/broker/service/topic/topic.rs +++ b/rust/src/broker/service/topic/topic.rs @@ -543,11 +543,22 @@ impl Topic { Ok(message_id) } + /// Fire-and-forget dispatch: never blocks the caller's connection task. + /// The dispatcher's merge-trigger (dispatch_in_progress + should_reschedule + /// in dispatcher/shared.rs) coalesces concurrent triggers, so spawning one + /// task per completion is safe: no lost wakeups, no duplicate dispatch. + pub fn spawn_dispatcher(topic: Arc>) { + tokio::spawn(async move { + let topic_guard = topic.read().await; + topic_guard.dispatch_to_subscriptions().await; + }); + } + /// Dispatch messages to all subscriptions (Push mode - Apache Pulsar style) /// /// This should be called after publish_message() to push messages to consumers. /// It triggers the dispatcher for each subscription to deliver pending messages. - pub async fn dispatch_to_subscriptions(&mut self) { + pub async fn dispatch_to_subscriptions(&self) { let subscription_count = self.subscriptions.len(); match self.runtime_mode { TopicRuntimeMode::Persistent => { From 87bdd4c8fb02d86055d33c0db5008a1821868eff Mon Sep 17 00:00:00 2001 From: XTLine <2668543450@qq.com> Date: Wed, 12 Aug 2026 17:12:45 +0800 Subject: [PATCH 22/44] perf(broker): remove locks from persistent enqueue hot path The per-Send hot path took Mutex to clone the write-queue appender and held the topic write lock for the rate-limit check. Under 8-subscription fanout the storage mutex is saturated by dispatch reads and acks, so enqueueing producers queued behind it (P stuck at ~124k). - cache ConcurrentAppender once per connection at setup; Send path enqueues without locking storage - TopicPublishRateLimiter becomes interior-mutable (std Mutex behind &self) so validate_publish_rate runs under the topic read lock - producer publish_message rate check downgraded write->read lock --- rust/src/broker/service/producer.rs | 2 +- rust/src/broker/service/server_cnx.rs | 39 ++++++++++------ rust/src/broker/service/topic/topic.rs | 65 +++++++++++++++----------- 3 files changed, 62 insertions(+), 44 deletions(-) diff --git a/rust/src/broker/service/producer.rs b/rust/src/broker/service/producer.rs index febd48e..0d4f911 100644 --- a/rust/src/broker/service/producer.rs +++ b/rust/src/broker/service/producer.rs @@ -113,7 +113,7 @@ impl Producer { if is_persistent { let (name, partition, persistent_runtime) = { - let mut topic = self.topic.write().await; + let topic = self.topic.read().await; topic.validate_publish_rate_public(metadata.as_ref(), payload.len())?; ( topic.name.clone(), diff --git a/rust/src/broker/service/server_cnx.rs b/rust/src/broker/service/server_cnx.rs index 80599f2..291f82f 100644 --- a/rust/src/broker/service/server_cnx.rs +++ b/rust/src/broker/service/server_cnx.rs @@ -56,7 +56,7 @@ type CnxError = Box; type CnxResult = Result; #[cfg(feature = "rocksdb-storage")] -use pulsar_lite_storage_managed_ledger_rocksdb::ConnAppendResult; +use pulsar_lite_storage_managed_ledger_rocksdb::{ConnAppendResult, ConcurrentAppender}; fn to_cnx_error(error: impl ToString) -> CnxError { Box::new(std::io::Error::other(error.to_string())) @@ -160,6 +160,10 @@ where /// Cap for pipelined persistent Send tasks on this connection. #[cfg(feature = "rocksdb-storage")] max_persistent_in_flight: usize, + /// Write-queue handle cloned once at connection setup so the per-Send hot + /// path never takes Mutex to obtain an appender. + #[cfg(feature = "rocksdb-storage")] + persistent_appender: Option, /// Resume reading threshold (hysteresis = max_pending_publish_requests / 2). resume_threshold: usize, /// When true, the event loop skips framed.next(), equivalent to Netty auto-read = false. @@ -214,6 +218,8 @@ where broker_service_url: String, channel_write_buffer_high_water_mark_bytes: usize, channel_write_buffer_low_water_mark_bytes: usize, + #[cfg(feature = "rocksdb-storage")] + persistent_appender: Option, ) -> Self { let mut framed = Framed::new(socket, PulsarFrameCodec::new()); framed.set_backpressure_boundary(channel_write_buffer_high_water_mark_bytes); @@ -249,6 +255,8 @@ where max_pending_publish_requests, #[cfg(feature = "rocksdb-storage")] max_persistent_in_flight: DEFAULT_MAX_PERSISTENT_IN_FLIGHT, + #[cfg(feature = "rocksdb-storage")] + persistent_appender, resume_threshold: max_pending_publish_requests / 2, read_paused: false, max_message_size, @@ -878,8 +886,8 @@ where let meta_len = frame.metadata.as_ref().map(|m| m.len()).unwrap_or(0); let payload_len = frame.payload.len(); - let (topic_name, partition, storage) = { - let mut topic_guard = topic.write().await; + let (topic_name, partition) = { + let topic_guard = topic.read().await; if let Err(error) = topic_guard.validate_publish_rate_public(frame.metadata.as_ref(), payload_len) { @@ -897,20 +905,12 @@ where } return Err(to_cnx_error(error)); } - ( - topic_guard.name.clone(), - topic_guard.partition, - topic_guard.shared_storage(), - ) + (topic_guard.name.clone(), topic_guard.partition) }; - let appender = { - let guard = storage.lock().await; - guard - .concurrent_appender() - .map_err(|e| to_cnx_error(e.to_string()))? - }; - if let Some(appender) = appender { + // Write-queue handle was cloned once at connection setup; the hot + // path never locks Mutex. + if let Some(appender) = self.persistent_appender.clone() { let meta_slice = frame.metadata.as_ref().map(|b| b.as_ref()).unwrap_or(&[]); appender .enqueue_for_connection( @@ -1090,6 +1090,13 @@ pub async fn handle_connection( "conn-{}", CONNECTION_COUNTER.fetch_add(1, Ordering::Relaxed) ); + // Clone the write-queue handle once per connection (before storage is + // moved into ServerCnx) so the per-Send path never locks Mutex. + #[cfg(feature = "rocksdb-storage")] + let persistent_appender = { + let guard = storage.lock().await; + guard.concurrent_appender().ok().flatten() + }; let mut server_cnx = ServerCnx::new( socket, storage, @@ -1104,6 +1111,8 @@ pub async fn handle_connection( broker_service_url, channel_write_buffer_high_water_mark_bytes, channel_write_buffer_low_water_mark_bytes, + #[cfg(feature = "rocksdb-storage")] + persistent_appender, ); server_cnx.run().await } diff --git a/rust/src/broker/service/topic/topic.rs b/rust/src/broker/service/topic/topic.rs index 1881127..9369698 100644 --- a/rust/src/broker/service/topic/topic.rs +++ b/rust/src/broker/service/topic/topic.rs @@ -61,6 +61,15 @@ pub struct TopicPublishRate { #[derive(Debug)] struct TopicPublishRateLimiter { + /// Interior mutability: checked from read-locked Topic context on the hot + /// send path, so the window state lives behind a short std Mutex instead of + /// requiring `&mut Topic`. The default unlimited configuration takes the + /// fast path before any window arithmetic. + inner: std::sync::Mutex, +} + +#[derive(Debug)] +struct TopicPublishRateLimiterState { limits: TopicPublishRate, window_started_at: Instant, messages_in_window: u64, @@ -70,40 +79,45 @@ struct TopicPublishRateLimiter { impl TopicPublishRateLimiter { fn new() -> Self { Self { - limits: TopicPublishRate::default(), - window_started_at: Instant::now(), - messages_in_window: 0, - bytes_in_window: 0, + inner: std::sync::Mutex::new(TopicPublishRateLimiterState { + limits: TopicPublishRate::default(), + window_started_at: Instant::now(), + messages_in_window: 0, + bytes_in_window: 0, + }), } } - fn set_limits(&mut self, limits: TopicPublishRate) { - self.limits = limits; - self.window_started_at = Instant::now(); - self.messages_in_window = 0; - self.bytes_in_window = 0; + fn set_limits(&self, limits: TopicPublishRate) { + let mut state = self.inner.lock().unwrap_or_else(|e| e.into_inner()); + state.limits = limits; + state.window_started_at = Instant::now(); + state.messages_in_window = 0; + state.bytes_in_window = 0; } - fn allow_publish(&mut self, message_count: u64, bytes: u64) -> bool { - if self.limits.messages_per_sec == 0 && self.limits.bytes_per_sec == 0 { + fn allow_publish(&self, message_count: u64, bytes: u64) -> bool { + let mut state = self.inner.lock().unwrap_or_else(|e| e.into_inner()); + + if state.limits.messages_per_sec == 0 && state.limits.bytes_per_sec == 0 { return true; } - if self.window_started_at.elapsed() >= Duration::from_secs(1) { - self.window_started_at = Instant::now(); - self.messages_in_window = 0; - self.bytes_in_window = 0; + if state.window_started_at.elapsed() >= Duration::from_secs(1) { + state.window_started_at = Instant::now(); + state.messages_in_window = 0; + state.bytes_in_window = 0; } - let next_messages = self.messages_in_window.saturating_add(message_count); - let next_bytes = self.bytes_in_window.saturating_add(bytes); + let next_messages = state.messages_in_window.saturating_add(message_count); + let next_bytes = state.bytes_in_window.saturating_add(bytes); let messages_ok = - self.limits.messages_per_sec == 0 || next_messages <= self.limits.messages_per_sec; - let bytes_ok = self.limits.bytes_per_sec == 0 || next_bytes <= self.limits.bytes_per_sec; + state.limits.messages_per_sec == 0 || next_messages <= state.limits.messages_per_sec; + let bytes_ok = state.limits.bytes_per_sec == 0 || next_bytes <= state.limits.bytes_per_sec; if messages_ok && bytes_ok { - self.messages_in_window = next_messages; - self.bytes_in_window = next_bytes; + state.messages_in_window = next_messages; + state.bytes_in_window = next_bytes; true } else { false @@ -206,15 +220,10 @@ impl Topic { self.persistent_runtime.clone() } - #[cfg(feature = "rocksdb-storage")] - pub(crate) fn shared_storage(&self) -> SharedStorage { - self.storage.clone() - } - /// Rate-limit check used by producers that release the topic write lock /// before waiting on storage IO. pub(crate) fn validate_publish_rate_public( - &mut self, + &self, metadata: Option<&Bytes>, payload_len: usize, ) -> Result<(), Box> { @@ -250,7 +259,7 @@ impl Topic { } fn validate_publish_rate( - &mut self, + &self, metadata: Option<&Bytes>, payload_len: usize, ) -> Result<(), Box> { From 6e5af16c39fd0fd896db8f3d96c324eac6e97e1d Mon Sep 17 00:00:00 2001 From: XTLine <2668543450@qq.com> Date: Wed, 12 Aug 2026 17:13:59 +0800 Subject: [PATCH 23/44] perf(storage): coalesce individual acks into ranges Shared-subscription acks past the mark-delete frontier were stored as one BTreeSet entry per position; 8-subscription fanout acking ~1M msg/s grew the set to gigabytes and OOM'd the broker. Store out-of-order acks as merged inclusive ranges (O(ranges) memory), mirroring Pulsar's individualDeletedMessages. - new RangeSet: insert/contains/take_covering with adjacency merging - mark-delete advancement skips whole ranges in one step - persistence now encodes real ranges; decode coalesces legacy points --- .../managed-ledger-rocksdb/src/cursor.rs | 18 +- .../managed-ledger-rocksdb/src/metadata.rs | 46 +++-- rust/storage/managed-ledger/src/cursor.rs | 26 ++- rust/storage/managed-ledger/src/lib.rs | 2 + rust/storage/managed-ledger/src/range_set.rs | 190 ++++++++++++++++++ 5 files changed, 252 insertions(+), 30 deletions(-) create mode 100644 rust/storage/managed-ledger/src/range_set.rs diff --git a/rust/storage/managed-ledger-rocksdb/src/cursor.rs b/rust/storage/managed-ledger-rocksdb/src/cursor.rs index 0f1c8cc..f74fd05 100644 --- a/rust/storage/managed-ledger-rocksdb/src/cursor.rs +++ b/rust/storage/managed-ledger-rocksdb/src/cursor.rs @@ -158,14 +158,22 @@ pub fn ack_managed_cursor_shared( Some(_) => {} } - while let Some(mark_delete) = cursor.state().mark_delete.clone() { - let Some(next) = next_position(&mark_delete, info) else { + // Advance the mark-delete frontier across contiguous acknowledged ranges. + // `take_covering` consumes a whole coalesced range in one step. + loop { + let Some(mark_delete) = cursor.state().mark_delete.clone() else { break; }; - if cursor.state.individually_deleted_entries.remove(&next) { - cursor.mark_delete(next)?; - } else { + let Some(next) = next_position(&mark_delete, info) else { break; + }; + match cursor + .state + .individually_deleted_entries + .take_covering(&next) + { + Some(end) => cursor.mark_delete(end)?, + None => break, } } diff --git a/rust/storage/managed-ledger-rocksdb/src/metadata.rs b/rust/storage/managed-ledger-rocksdb/src/metadata.rs index 08a4e45..e078711 100644 --- a/rust/storage/managed-ledger-rocksdb/src/metadata.rs +++ b/rust/storage/managed-ledger-rocksdb/src/metadata.rs @@ -1,9 +1,8 @@ use super::entrylog::EntryIndex; use anyhow::{anyhow, Result}; use prost::Message; -use pulsar_lite_storage_managed_ledger::{ManagedCursorState, ManagedLedgerPosition}; +use pulsar_lite_storage_managed_ledger::{ManagedCursorState, ManagedLedgerPosition, RangeSet}; use serde::{Deserialize, Serialize}; -use std::collections::BTreeSet; use std::time::{SystemTime, UNIX_EPOCH}; pub mod proto { @@ -36,7 +35,7 @@ impl From for StoredEntryLocation { #[derive(Debug, Clone, Serialize, Deserialize)] pub struct StoredManagedCursorState { pub mark_delete: Option, - pub individually_deleted_entries: BTreeSet, + pub individually_deleted_entries: RangeSet, } impl From for StoredManagedCursorState { @@ -68,7 +67,7 @@ impl StoredManagedCursorState { individual_deleted_messages: self .individually_deleted_entries .iter() - .map(position_range) + .map(|(start, end)| position_range(start, end)) .collect(), properties: Vec::new(), last_active: None, @@ -88,21 +87,28 @@ impl StoredManagedCursorState { }), _ => None, }; - let mut individually_deleted_entries = BTreeSet::new(); + let mut individually_deleted_entries = RangeSet::new(); for range in info.individual_deleted_messages { let lower = range.lower_endpoint; let upper = range.upper_endpoint; - if lower.ledger_id != upper.ledger_id || lower.entry_id != upper.entry_id { + if lower.ledger_id != upper.ledger_id { return Err(anyhow!( - "cursor delete range spans multiple positions, which pulsar-lite does not support yet" + "cursor delete range spans multiple ledgers, which is unsupported" )); } - individually_deleted_entries.insert(ManagedLedgerPosition { - ledger_id: lower.ledger_id.try_into()?, - entry_id: lower.entry_id.try_into()?, - partition: -1, - }); + individually_deleted_entries.insert_range( + ManagedLedgerPosition { + ledger_id: lower.ledger_id.try_into()?, + entry_id: lower.entry_id.try_into()?, + partition: -1, + }, + ManagedLedgerPosition { + ledger_id: upper.ledger_id.try_into()?, + entry_id: upper.entry_id.try_into()?, + partition: -1, + }, + ); } Ok(Self { @@ -214,13 +220,17 @@ fn current_time_millis() -> u64 { .unwrap_or_default() } -fn position_range(position: &ManagedLedgerPosition) -> proto::MessageRange { - let endpoint = proto::NestedPositionInfo { - ledger_id: position.ledger_id as i64, - entry_id: position.entry_id as i64, +fn position_range(start: &ManagedLedgerPosition, end: &ManagedLedgerPosition) -> proto::MessageRange { + let lower_endpoint = proto::NestedPositionInfo { + ledger_id: start.ledger_id as i64, + entry_id: start.entry_id as i64, + }; + let upper_endpoint = proto::NestedPositionInfo { + ledger_id: end.ledger_id as i64, + entry_id: end.entry_id as i64, }; proto::MessageRange { - lower_endpoint: endpoint, - upper_endpoint: endpoint, + lower_endpoint, + upper_endpoint, } } diff --git a/rust/storage/managed-ledger/src/cursor.rs b/rust/storage/managed-ledger/src/cursor.rs index 1aede05..593726c 100644 --- a/rust/storage/managed-ledger/src/cursor.rs +++ b/rust/storage/managed-ledger/src/cursor.rs @@ -1,6 +1,6 @@ use crate::position::ManagedLedgerPosition; +use crate::range_set::RangeSet; use anyhow::Result; -use std::collections::BTreeSet; /// Managed-cursor state skeleton. /// @@ -9,7 +9,10 @@ use std::collections::BTreeSet; #[derive(Debug, Clone, Default, PartialEq, Eq)] pub struct ManagedCursorState { pub mark_delete: Option, - pub individually_deleted_entries: BTreeSet, + /// Out-of-order acknowledgements past the mark-delete frontier, stored as + /// coalesced ranges (see `RangeSet`) so memory stays O(ranges) even when + /// millions of individual acks arrive out of order. + pub individually_deleted_entries: RangeSet, } /// Cursor abstraction for managed-ledger style persistence. @@ -29,7 +32,7 @@ pub trait ManagedCursor: Send + Sync { #[derive(Debug, Clone, Default, PartialEq, Eq)] pub struct SubscriptionCursor { pub mark_delete: Option, - pub acked_holes: BTreeSet, + pub acked_holes: RangeSet, } pub fn is_message_acknowledged(cursor: Option<&SubscriptionCursor>, entry: u64) -> bool { @@ -43,16 +46,24 @@ pub fn is_message_acknowledged(cursor: Option<&SubscriptionCursor>, entry: u64) .unwrap_or(false) } +/// Advances the mark-delete frontier across contiguous acknowledged ranges. +/// +/// `take_covering` removes the whole covering range at once, so a long +/// sequential ack streak is consumed in one step per stored range instead of +/// one position at a time. pub fn advance_mark_delete(cursor: &mut SubscriptionCursor) { - let mut next_expected = cursor.mark_delete.map_or(0, |mark_delete| mark_delete + 1); - while cursor.acked_holes.remove(&next_expected) { - cursor.mark_delete = Some(next_expected); - next_expected += 1; + loop { + let next_expected = cursor.mark_delete.map_or(0, |mark_delete| mark_delete + 1); + match cursor.acked_holes.take_covering(&next_expected) { + Some(end) => cursor.mark_delete = Some(end), + None => break, + } } } pub fn ack_shared(cursor: &mut SubscriptionCursor, entry: u64) -> (Option, usize) { if is_message_acknowledged(Some(cursor), entry) { + // Second value is the number of stored ranges (not individual holes). return (cursor.mark_delete, cursor.acked_holes.len()); } @@ -75,5 +86,6 @@ pub fn ack_shared(cursor: &mut SubscriptionCursor, entry: u64) -> (Option, } } + // Second value is the number of stored ranges (not individual holes). (cursor.mark_delete, cursor.acked_holes.len()) } diff --git a/rust/storage/managed-ledger/src/lib.rs b/rust/storage/managed-ledger/src/lib.rs index f909c76..aa57aaa 100644 --- a/rust/storage/managed-ledger/src/lib.rs +++ b/rust/storage/managed-ledger/src/lib.rs @@ -8,6 +8,7 @@ mod ledger; mod legacy_storage; mod memory; mod position; +mod range_set; pub use config::ManagedLedgerConfig; pub use cursor::{ @@ -27,3 +28,4 @@ pub use memory::{ InMemoryManagedLedgerStorage, }; pub use position::{ManagedLedgerPosition, MessageId, NonPersistentEntry, StoredMessage}; +pub use range_set::{RangeSet, Succ}; diff --git a/rust/storage/managed-ledger/src/range_set.rs b/rust/storage/managed-ledger/src/range_set.rs new file mode 100644 index 0000000..f7ffb6e --- /dev/null +++ b/rust/storage/managed-ledger/src/range_set.rs @@ -0,0 +1,190 @@ +use crate::position::ManagedLedgerPosition; +use serde::Deserialize; +use std::collections::BTreeSet; + +/// A sorted, non-overlapping set of deleted-position ranges (both endpoints +/// inclusive). +/// +/// Consecutive acknowledgements are coalesced into a single range, so memory +/// usage is O(number of ranges) instead of O(number of acknowledged +/// positions). A Shared-subscription cursor acknowledging millions of messages +/// in near-sequential order keeps only a handful of ranges in memory. +/// +/// The representation mirrors Apache Pulsar's `individualDeletedMessages` +/// (a `LongPairRangeSet`): the ack frontier (`mark_delete`) can only express a +/// contiguous prefix, so out-of-order acknowledgements past the frontier are +/// tracked here as compact ranges instead of one entry per position. +#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize)] +pub struct RangeSet { + /// (start, end) pairs; ranges are kept disjoint and non-adjacent + /// (adjacent ranges are merged on insert). + ranges: BTreeSet<(K, K)>, +} + +impl<'de, K: Ord + Deserialize<'de>> serde::Deserialize<'de> for RangeSet { + fn deserialize>(deserializer: D) -> Result { + Ok(Self { + ranges: BTreeSet::deserialize(deserializer)?, + }) + } +} + +impl Default for RangeSet { + fn default() -> Self { + Self { + ranges: BTreeSet::new(), + } + } +} + +/// Successor operation on a position key, used for adjacency checks during +/// range merging. Returns `None` on overflow (e.g. `u64::MAX`), which simply +/// disables adjacency merging at that boundary. +pub trait Succ: Ord + Sized { + fn succ(&self) -> Option; +} + +impl Succ for u64 { + fn succ(&self) -> Option { + self.checked_add(1) + } +} + +impl Succ for ManagedLedgerPosition { + fn succ(&self) -> Option { + self.entry_id.checked_add(1).map(|entry_id| ManagedLedgerPosition { + ledger_id: self.ledger_id, + entry_id, + partition: self.partition, + }) + } +} + +impl RangeSet { + pub fn new() -> Self { + Self::default() + } + + pub fn clear(&mut self) { + self.ranges.clear(); + } + + /// Number of stored ranges (the memory-relevant metric). + pub fn len(&self) -> usize { + self.ranges.len() + } + + pub fn is_empty(&self) -> bool { + self.ranges.is_empty() + } + + pub fn iter(&self) -> impl Iterator { + self.ranges.iter() + } + + /// Returns `true` if `pos` falls inside any stored range. + /// + /// Two lookups are needed because a range starting exactly at `pos` sorts + /// after the `(pos, pos)` probe when its end is greater than `pos`. + pub fn contains(&self, pos: &K) -> bool { + // 1) The largest range whose start is <= pos. + if let Some((start, end)) = self.ranges.range(..=(pos.clone(), pos.clone())).next_back() { + if start <= pos && pos <= end { + return true; + } + } + // 2) A range starting exactly at pos. + if let Some((start, _)) = self.ranges.range((pos.clone(), pos.clone())..).next() { + if start == pos { + return true; + } + } + false + } + + /// Inserts the inclusive range `[start, end]`, merging every stored range + /// that intersects or is adjacent to it. + pub fn insert_range(&mut self, start: K, end: K) { + if start > end { + return; + } + let mut start = start; + let mut end = end; + + // Merge ranges starting at or after the new start that intersect or + // are adjacent to the growing range. Adjacency is `s <= end + 1`. + loop { + let limit = end.succ().unwrap_or_else(|| end.clone()); + let merge = self + .ranges + .range((start.clone(), start.clone())..) + .next() + .cloned() + .filter(|(s, _)| *s <= limit); + let Some((s, e)) = merge else { + break; + }; + self.ranges.remove(&(s.clone(), e.clone())); + if s < start { + start = s; + } + if end < e { + end = e; + } + } + + // Merge the left neighbour when it is adjacent to the new start. + if let Some((s, e)) = self + .ranges + .range(..=(start.clone(), start.clone())) + .next_back() + .cloned() + { + if e.succ() == Some(start.clone()) { + self.ranges.remove(&(s.clone(), e.clone())); + start = s; + } + } + + self.ranges.insert((start, end)); + } + + /// Inserts a single position (a one-point range). + pub fn insert(&mut self, pos: K) { + if self.contains(&pos) { + return; + } + self.insert_range(pos.clone(), pos); + } + + /// If `pos` falls inside a stored range, removes that range and returns + /// its end. Used by mark-delete advancement: the whole contiguous deleted + /// segment can be skipped in one step instead of one position at a time. + pub fn take_covering(&mut self, pos: &K) -> Option { + // 1) The largest range whose start is <= pos. + if let Some((start, end)) = self + .ranges + .range(..=(pos.clone(), pos.clone())) + .next_back() + .cloned() + { + if start <= *pos { + self.ranges.remove(&(start.clone(), end.clone())); + return Some(end); + } + } + // 2) A range starting exactly at pos. + if let Some((start, end)) = self + .ranges + .range((pos.clone(), pos.clone())..) + .next() + .cloned() + { + if start == *pos { + self.ranges.remove(&(start.clone(), end.clone())); + return Some(end); + } + } + None + } +} From b3f244b020f98a216f33a0bfaf601b65bb635b4e Mon Sep 17 00:00:00 2001 From: XTLine <2668543450@qq.com> Date: Wed, 12 Aug 2026 19:42:23 +0800 Subject: [PATCH 24/44] debug(broker): add TEMP DIAG file-based probes for ack pipeline MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Layered diagnostic counters writing to /data/*_diag.txt to locate where the 16-consumer ack pipeline stalls (2100万 acks expected, storage sees <10k): - dispatch_diag: consumer send_message entry (per 100k) - ack_cmd_diag: handle_ack entry with ids/known/ack_type (per 100) - msg_acked_diag: message_acked entry with sub_type/is_persistent (per 100) - ack_drop_diag: dropped-ack points in message_acked (per 100k) - ack_enter_diag: storage ack_message/_shared entry (per 10k) - pending_acks_diag: PendingAcksMap len growth (per 100k insert) - cursor_diag: persist_state mark_delete/ranges CSV (per 100k) All probes are TEMPORARY and must be removed after root-cause. --- rust/src/broker/handler/consumer_handler.rs | 22 +++++ rust/src/broker/service/consumer.rs | 85 +++++++++++++++++++ rust/src/broker/service/pending_acks.rs | 17 ++++ .../managed-ledger-rocksdb/src/cursor.rs | 32 +++++++ .../managed-ledger-rocksdb/src/store.rs | 32 +++++++ 5 files changed, 188 insertions(+) diff --git a/rust/src/broker/handler/consumer_handler.rs b/rust/src/broker/handler/consumer_handler.rs index 849510c..98b6f9d 100644 --- a/rust/src/broker/handler/consumer_handler.rs +++ b/rust/src/broker/handler/consumer_handler.rs @@ -232,6 +232,28 @@ where T: tokio::io::AsyncRead + tokio::io::AsyncWrite + Unpin, { let ack_cmd = cmd.ack.as_ref().ok_or("Missing ack command")?; + // [TEMP DIAG] count ack command arrivals and parse results. + { + use std::io::Write; + use std::sync::atomic::{AtomicU64, Ordering}; + static C: AtomicU64 = AtomicU64::new(0); + let n = C.fetch_add(1, Ordering::Relaxed); + if n % 100 == 0 { + let mid_count = ack_cmd.message_id.len(); + let known = consumers.contains_key(&ack_cmd.consumer_id); + if let Ok(mut f) = std::fs::OpenOptions::new() + .create(true) + .append(true) + .open("/data/ack_cmd_diag.txt") + { + let _ = writeln!( + f, + "{},\tconsumer={},\tids={},\tknown={},\tack_type={:?}", + n, ack_cmd.consumer_id, mid_count, known, ack_cmd.ack_type + ); + } + } + } log::debug!( "Handling Ack command: consumer_id={}, ack_type={:?}", ack_cmd.consumer_id, diff --git a/rust/src/broker/service/consumer.rs b/rust/src/broker/service/consumer.rs index 59f370d..789ffe9 100644 --- a/rust/src/broker/service/consumer.rs +++ b/rust/src/broker/service/consumer.rs @@ -357,6 +357,30 @@ impl Consumer { M: Into, P: Into, { + // [TEMP DIAG] count dispatches to consumers. + { + use std::io::Write; + use std::sync::atomic::{AtomicU64, Ordering}; + static C: AtomicU64 = AtomicU64::new(0); + let n = C.fetch_add(1, Ordering::Relaxed); + if n % 100_000 == 0 { + if let Ok(mut f) = std::fs::OpenOptions::new() + .create(true) + .append(true) + .open("/data/dispatch_diag.txt") + { + let _ = writeln!( + f, + "{},\tconsumer={},\tmsg={}:{}\tredelivery={}", + n, + self.consumer_id, + message_id.ledger, + message_id.entry, + redelivery_count + ); + } + } + } let metadata = metadata.into(); let payload = payload.into(); let wire_size = pulsar_lite_proto::codec::estimate_message_parts_size( @@ -684,6 +708,35 @@ impl Consumer { ack_type: AckCommandType, message_ids: Vec, ) -> Result<(), String> { + // [TEMP DIAG] count message_acked entries with batch size. + { + use std::io::Write; + use std::sync::atomic::{AtomicU64, Ordering}; + static C: AtomicU64 = AtomicU64::new(0); + let n = C.fetch_add(1, Ordering::Relaxed); + if n % 100 == 0 { + let (diag_sub_type, diag_is_persistent) = { + let sub = self.subscription.read().await; + (sub.get_sub_type(), sub.is_persistent()) + }; + if let Ok(mut f) = std::fs::OpenOptions::new() + .create(true) + .append(true) + .open("/data/msg_acked_diag.txt") + { + let _ = writeln!( + f, + "{},\tconsumer={},\tids={},\tack_type={:?},\tsub_type={:?},\tpersistent={}", + n, + self.consumer_id, + message_ids.len(), + ack_type, + diag_sub_type, + diag_is_persistent + ); + } + } + } if message_ids.is_empty() { return Ok(()); } @@ -708,6 +761,8 @@ impl Consumer { let Some((owner, tracked_message_id)) = self.resolve_ack_owner(message_id).await else { + // [TEMP DIAG] ack dropped: no owner found. + diag_ack_drop("no_owner", message_id); log::warn!( "Consumer {} attempted to ack message {}:{} without ownership; ignoring", self.consumer_id, @@ -718,6 +773,8 @@ impl Consumer { }; if !owner.remove_pending_ack(&tracked_message_id).await { + // [TEMP DIAG] ack dropped: pending-ack removal failed. + diag_ack_drop("remove_failed", &tracked_message_id); log::warn!( "Consumer {} found owner {} for message {}:{} but pending ack removal failed; ignoring", self.consumer_id, @@ -808,6 +865,34 @@ impl Consumer { } } +/// [TEMP DIAG] append one line per 100k dropped acks to /data/ack_drop_diag.txt. +fn diag_ack_drop(reason: &str, message_id: &MessageId) { + use std::io::Write; + use std::sync::atomic::{AtomicU64, Ordering}; + static COUNTER: AtomicU64 = AtomicU64::new(0); + let n = COUNTER.fetch_add(1, Ordering::Relaxed); + if n % 100_000 == 0 { + if let Ok(mut f) = std::fs::OpenOptions::new() + .create(true) + .append(true) + .open("/data/ack_drop_diag.txt") + { + let _ = writeln!( + f, + "{},\t{},\t{}:{}\tpartition={}", + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map(|d| d.as_secs()) + .unwrap_or(0), + reason, + message_id.ledger, + message_id.entry, + message_id.partition + ); + } + } +} + impl PartialEq for Consumer { fn eq(&self, other: &Self) -> bool { self.consumer_id == other.consumer_id && self.connection_id == other.connection_id diff --git a/rust/src/broker/service/pending_acks.rs b/rust/src/broker/service/pending_acks.rs index 6098ae2..6c8e029 100644 --- a/rust/src/broker/service/pending_acks.rs +++ b/rust/src/broker/service/pending_acks.rs @@ -50,6 +50,23 @@ impl PendingAcksMap { sticky_key_hash, }, ); + // [TEMP DIAG] log pending-acks size every 100k inserts. + { + use std::io::Write; + use std::sync::atomic::{AtomicU64, Ordering}; + static C: AtomicU64 = AtomicU64::new(0); + let n = C.fetch_add(1, Ordering::Relaxed); + if n % 100_000 == 0 { + let len = inner.len(); + if let Ok(mut f) = std::fs::OpenOptions::new() + .create(true) + .append(true) + .open("/data/pending_acks_diag.txt") + { + let _ = writeln!(f, "{},\tinserts={},\tlen={}", n, n, len); + } + } + } true } diff --git a/rust/storage/managed-ledger-rocksdb/src/cursor.rs b/rust/storage/managed-ledger-rocksdb/src/cursor.rs index f74fd05..990c219 100644 --- a/rust/storage/managed-ledger-rocksdb/src/cursor.rs +++ b/rust/storage/managed-ledger-rocksdb/src/cursor.rs @@ -43,6 +43,38 @@ impl RocksDBManagedCursor { } pub fn persist_state(&self) -> Result<()> { + // [TEMP DIAG] every 100k persists, append cursor state to a CSV file + // (bypasses the logging system entirely) to verify whether the + // individual-delete range set accumulates while mark_delete stalls. + use std::sync::atomic::{AtomicU64, Ordering}; + static DIAG_COUNT: AtomicU64 = AtomicU64::new(0); + let n = DIAG_COUNT.fetch_add(1, Ordering::Relaxed); + if n % 100_000 == 0 { + use std::io::Write; + let mark_delete = self + .state + .mark_delete + .as_ref() + .map(|p| format!("{}:{}", p.ledger_id, p.entry_id)) + .unwrap_or_else(|| "None".to_string()); + let line = format!( + "{},\t{},\t{},\t{}\n", + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map(|d| d.as_secs()) + .unwrap_or(0), + self.name, + mark_delete, + self.state.individually_deleted_entries.len() + ); + if let Ok(mut f) = std::fs::OpenOptions::new() + .create(true) + .append(true) + .open("/data/cursor_diag.csv") + { + let _ = f.write_all(line.as_bytes()); + } + } let key = keys::managed_cursor_key(&self.managedledger_name, &self.name); let stored = StoredManagedCursorState::from(self.state.clone()); self.db.put(key, stored.encode_to_vec())?; diff --git a/rust/storage/managed-ledger-rocksdb/src/store.rs b/rust/storage/managed-ledger-rocksdb/src/store.rs index 9aed3f7..872b3a4 100644 --- a/rust/storage/managed-ledger-rocksdb/src/store.rs +++ b/rust/storage/managed-ledger-rocksdb/src/store.rs @@ -362,6 +362,22 @@ impl ManagedLedgerStorage for RocksDbManagedLedgerStorage { subscription: &str, message_id: MessageId, ) -> Result<()> { + // [TEMP DIAG] count ack entries reaching storage layer (inline). + { + use std::io::Write; + use std::sync::atomic::{AtomicU64, Ordering}; + static C: AtomicU64 = AtomicU64::new(0); + let n = C.fetch_add(1, Ordering::Relaxed); + if n % 100_000 == 0 { + if let Ok(mut f) = std::fs::OpenOptions::new() + .create(true) + .append(true) + .open("/data/ack_enter_diag.txt") + { + let _ = writeln!(f, "{},\tack_message,\t{}", n, n); + } + } + } self.with_cursor_lock(|| { let cursor_name = keys::encode_cursor_name(subscription); let ledger = self.topic_ledger(topic)?; @@ -376,6 +392,22 @@ impl ManagedLedgerStorage for RocksDbManagedLedgerStorage { subscription: &str, message_id: MessageId, ) -> Result<()> { + // [TEMP DIAG] count ack entries reaching storage layer (inline). + { + use std::io::Write; + use std::sync::atomic::{AtomicU64, Ordering}; + static C: AtomicU64 = AtomicU64::new(0); + let n = C.fetch_add(1, Ordering::Relaxed); + if n % 10_000 == 0 { + if let Ok(mut f) = std::fs::OpenOptions::new() + .create(true) + .append(true) + .open("/data/ack_enter_diag.txt") + { + let _ = writeln!(f, "{},\tack_message_shared,\t{}", n, n); + } + } + } self.with_cursor_lock(|| { let cursor_name = keys::encode_cursor_name(subscription); let ledger = self.topic_ledger(topic)?; From aa8b6d8fa1c6b9addee7419ab3d6911f21c89721 Mon Sep 17 00:00:00 2001 From: XTLine <2668543450@qq.com> Date: Wed, 12 Aug 2026 20:11:40 +0800 Subject: [PATCH 25/44] fix(broker): pass persistent_appender in test connection builder ServerCnx::new gained a rocksdb-storage persistent_appender param in the enqueue-lock-removal change; build_test_connection was not updated, so the test profile failed to compile with rocksdb-storage. --- rust/src/broker/service/server_cnx.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/rust/src/broker/service/server_cnx.rs b/rust/src/broker/service/server_cnx.rs index 291f82f..22958cf 100644 --- a/rust/src/broker/service/server_cnx.rs +++ b/rust/src/broker/service/server_cnx.rs @@ -1156,6 +1156,8 @@ mod tests { "pulsar://127.0.0.1:6650".to_string(), 64 * 1024, 32 * 1024, + #[cfg(feature = "rocksdb-storage")] + None, ); let client_framed = Framed::new(client, PulsarFrameCodec::new()); (server_cnx, client_framed, test_dir) From c5c16e15e145f86e430a5c8b3fb4b620be4d4b8c Mon Sep 17 00:00:00 2001 From: XTLine <2668543450@qq.com> Date: Thu, 13 Aug 2026 16:33:02 +0800 Subject: [PATCH 26/44] Fix permit accounting and backpressure in broker dispatch --- rust/src/broker/dispatcher/key_shared.rs | 39 ++--- rust/src/broker/dispatcher/mod.rs | 16 ++ rust/src/broker/dispatcher/shared.rs | 73 ++++++---- rust/src/broker/handler/consumer_handler.rs | 37 ++--- .../broker/service/connection_write_state.rs | 41 ++++++ rust/src/broker/service/consumer.rs | 137 ++++++------------ rust/src/broker/service/pending_acks.rs | 2 +- rust/src/broker/service/server_cnx.rs | 71 ++++++--- rust/src/broker/service/topic/subscription.rs | 8 +- 9 files changed, 235 insertions(+), 189 deletions(-) diff --git a/rust/src/broker/dispatcher/key_shared.rs b/rust/src/broker/dispatcher/key_shared.rs index 80d7851..030ce34 100644 --- a/rust/src/broker/dispatcher/key_shared.rs +++ b/rust/src/broker/dispatcher/key_shared.rs @@ -13,7 +13,7 @@ use crate::broker::service::topic::{ use crate::broker::service::{Consumer, SharedStorage}; use pulsar_lite_storage_managed_ledger::{ManagedLedgerPosition, MessageId}; use std::collections::HashMap; -use std::sync::atomic::{AtomicU32, Ordering}; +use std::sync::atomic::{AtomicI32, Ordering}; use std::sync::{Arc, RwLock}; const DISPATCHER_MAX_ROUND_ROBIN_BATCH_SIZE: u32 = 20; @@ -29,7 +29,7 @@ pub struct KeySharedDispatcher { auto_split_assignments: Vec<(KeySharedHashRange, Arc)>, sticky_assignments: Vec<(KeySharedHashRange, Arc)>, key_shared_policy: KeySharedPolicy, - total_available_permits: AtomicU32, + total_available_permits: AtomicI32, read_position: RwLock>, redelivery_controller: RwLock, } @@ -47,7 +47,7 @@ impl KeySharedDispatcher { auto_split_assignments: Vec::new(), sticky_assignments: Vec::new(), key_shared_policy, - total_available_permits: AtomicU32::new(0), + total_available_permits: AtomicI32::new(0), read_position: RwLock::new(None), redelivery_controller: RwLock::new(RedeliveryController::new(block_hashes)), } @@ -64,7 +64,7 @@ impl KeySharedDispatcher { let _ = self.total_available_permits.fetch_update( Ordering::Relaxed, Ordering::Relaxed, - |current| Some(current.saturating_sub(permits)), + |current| Some(current.saturating_sub(permits as i32)), ); } @@ -326,12 +326,12 @@ impl KeySharedDispatcher { let max_batch = self .total_available_permits .load(Ordering::Relaxed) - .min(DISPATCHER_MAX_ROUND_ROBIN_BATCH_SIZE); - if max_batch == 0 { + .min(DISPATCHER_MAX_ROUND_ROBIN_BATCH_SIZE as i32); + if max_batch <= 0 { return Ok(0); } - let mut remaining_dispatches = max_batch; + let mut remaining_dispatches = max_batch as u32; let mut progress = 0u32; while remaining_dispatches > 0 { @@ -344,7 +344,8 @@ impl KeySharedDispatcher { let sticky_key_hash = redelivery .sticky_key_hash .unwrap_or_else(|| sticky_key_hash_from_metadata(&entry.metadata)); - if !consumer.use_permit().await { + let batch_count = crate::broker::dispatcher::messages_in_batch(&entry.metadata); + if !consumer.try_use_permits(batch_count).await { self.restore_redelivery_message(RedeliveryEntry { sticky_key_hash: Some(sticky_key_hash), ..redelivery @@ -352,7 +353,7 @@ impl KeySharedDispatcher { break; } remaining_dispatches -= 1; - self.subtract_total_permits(1); + self.subtract_total_permits(batch_count as u32); if consumer .send_message_with_sticky_hash( @@ -374,11 +375,11 @@ impl KeySharedDispatcher { redelivery_count: redelivery.redelivery_count + 1, sticky_key_hash: Some(sticky_key_hash), }); - consumer.add_permits(1).await; - self.total_available_permits.fetch_add(1, Ordering::Relaxed); + consumer.add_permits(batch_count).await; + self.total_available_permits + .fetch_add(batch_count as i32, Ordering::Relaxed); break; } - continue; } let Some(candidate) = self @@ -419,11 +420,12 @@ impl KeySharedDispatcher { let Some(consumer) = self.select_consumer_for_hash(sticky_key_hash) else { break; }; - if !consumer.use_permit().await { + let batch_count = crate::broker::dispatcher::messages_in_batch(&candidate.metadata); + if !consumer.try_use_permits(batch_count).await { break; } remaining_dispatches -= 1; - self.subtract_total_permits(1); + self.subtract_total_permits(batch_count as u32); if consumer .send_message_with_sticky_hash( @@ -447,8 +449,9 @@ impl KeySharedDispatcher { sticky_key_hash: Some(sticky_key_hash), }); commit_read_position(&self.read_position, candidate.next_position); - consumer.add_permits(1).await; - self.total_available_permits.fetch_add(1, Ordering::Relaxed); + consumer.add_permits(batch_count).await; + self.total_available_permits + .fetch_add(batch_count as i32, Ordering::Relaxed); break; } } @@ -508,7 +511,7 @@ impl Dispatcher for KeySharedDispatcher { fn consumer_flow(&self, consumer_id: u64, additional_permits: u32) { if self.consumers_by_id.contains_key(&consumer_id) { self.total_available_permits - .fetch_add(additional_permits, Ordering::Relaxed); + .fetch_add(additional_permits as i32, Ordering::Relaxed); } } @@ -527,7 +530,7 @@ impl Dispatcher for KeySharedDispatcher { if progress == 0 { break; } - if self.total_available_permits.load(Ordering::Relaxed) == 0 { + if self.total_available_permits.load(Ordering::Relaxed) <= 0 { break; } diff --git a/rust/src/broker/dispatcher/mod.rs b/rust/src/broker/dispatcher/mod.rs index e09794c..1b7c38d 100644 --- a/rust/src/broker/dispatcher/mod.rs +++ b/rust/src/broker/dispatcher/mod.rs @@ -21,3 +21,19 @@ pub use key_shared::KeySharedDispatcher; pub use shared::SharedDispatcher; pub use single_active::rewind_read_position; pub use traits::Dispatcher; + +/// Number of client-visible messages carried by one entry's metadata batch. +/// Permit accounting is per client-visible message (Apache Pulsar semantics): +/// a batch entry of N messages consumes N permits. Entries without batch +/// metadata count as a single message. +pub fn messages_in_batch(metadata: &[u8]) -> u32 { + use prost::Message; + use pulsar_lite_proto::codec::proto::pulsar::MessageMetadata; + + MessageMetadata::decode(metadata) + .ok() + .and_then(|m| m.num_messages_in_batch) + .filter(|n| *n > 0) + .map(|n| n as u32) + .unwrap_or(1) +} diff --git a/rust/src/broker/dispatcher/shared.rs b/rust/src/broker/dispatcher/shared.rs index 776e556..92c8fb2 100644 --- a/rust/src/broker/dispatcher/shared.rs +++ b/rust/src/broker/dispatcher/shared.rs @@ -11,7 +11,7 @@ use crate::broker::service::topic::SubscriptionType; use crate::broker::service::{Consumer, SharedStorage}; use pulsar_lite_storage_managed_ledger::{ManagedLedgerPosition, MessageId}; use std::collections::HashMap; -use std::sync::atomic::{AtomicBool, AtomicU32, AtomicUsize, Ordering}; +use std::sync::atomic::{AtomicBool, AtomicI32, AtomicUsize, Ordering}; use std::sync::{Arc, RwLock}; /// Consistent with Apache Pulsar: dispatcherMaxRoundRobinBatchSize = 20 @@ -30,8 +30,10 @@ pub struct SharedDispatcher { /// Round-Robin index for consumer selection (atomic for thread safety) round_robin_index: AtomicUsize, - /// Total available permits across all consumers (atomic for thread safety) - total_available_permits: AtomicU32, + /// Total available permits across all consumers (atomic for thread safety). + /// i32: the balance may briefly go negative when a batch entry is larger + /// than the remaining permits (Apache Pulsar semantics). + total_available_permits: AtomicI32, /// Flag to prevent reentrant dispatching (Pulsar readMoreEntriesInProgress). dispatch_in_progress: AtomicBool, @@ -63,7 +65,7 @@ impl SharedDispatcher { consumers: HashMap::new(), consumer_order: Vec::new(), round_robin_index: AtomicUsize::new(0), - total_available_permits: AtomicU32::new(0), + total_available_permits: AtomicI32::new(0), dispatch_in_progress: AtomicBool::new(false), should_reschedule: AtomicBool::new(false), max_unacked_messages: DEFAULT_MAX_UNACKED_MESSAGES_PER_CONSUMER, @@ -183,7 +185,7 @@ impl SharedDispatcher { let _ = self.total_available_permits.fetch_update( Ordering::Relaxed, Ordering::Relaxed, - |current| Some(current.saturating_sub(permits)), + |current| Some(current.saturating_sub(permits as i32)), ); } @@ -247,7 +249,7 @@ impl SharedDispatcher { ) -> Result> { // Check if we have permits let total_permits = self.total_available_permits.load(Ordering::Relaxed); - if total_permits == 0 { + if total_permits <= 0 { log::debug!("No permits available, skipping dispatch"); return Ok(0); } @@ -255,7 +257,8 @@ impl SharedDispatcher { // Dispatch up to DISPATCHER_MAX_ROUND_ROBIN_BATCH_SIZE messages let mut dispatched = 0u32; let mut redelivered = 0u32; - let max_batch = std::cmp::min(total_permits, DISPATCHER_MAX_ROUND_ROBIN_BATCH_SIZE); + let max_batch = + std::cmp::min(total_permits, DISPATCHER_MAX_ROUND_ROBIN_BATCH_SIZE as i32) as u32; log::debug!( "Starting batch dispatch: max_batch={}, total_permits={}, consumers={}, redelivery_queue={}", @@ -294,14 +297,9 @@ impl SharedDispatcher { break; } - // Use one permit - if !consumer.use_permit().await { - log::warn!("Consumer {} permits exhausted during dispatch", consumer_id); - break; - } - - // Decrease total permits - self.total_available_permits.fetch_sub(1, Ordering::Relaxed); + // Permits are consumed per client-visible message after the entry + // is fetched (batch entries carry N messages), matching Apache + // Pulsar. Nothing is consumed here at the top of the loop. // 1. Priority: get message from redelivery queue if let Some(redelivery) = self.pop_redelivery_message() { @@ -312,8 +310,6 @@ impl SharedDispatcher { guard.is_acknowledged(&topic, &subscription, &msg_id)? }; if already_acked { - consumer.add_permits(1).await; - self.total_available_permits.fetch_add(1, Ordering::Relaxed); log::debug!( "Skipping replay for already-acked message {}:{}", msg_id.ledger, @@ -329,6 +325,21 @@ impl SharedDispatcher { }; if let Some(entry) = message_opt { + let batch_count = crate::broker::dispatcher::messages_in_batch(&entry.metadata); + if !consumer.try_use_permits(batch_count).await { + self.restore_redelivery_message(RedeliveryEntry { + message_id: entry.message_id.clone(), + redelivery_count, + sticky_key_hash: redelivery.sticky_key_hash, + }); + log::warn!( + "Consumer {} permits insufficient for batch of {} messages, waiting for flow", + consumer_id, batch_count + ); + break; + } + self.total_available_permits + .fetch_sub(batch_count as i32, Ordering::Relaxed); if consumer .send_message( entry.message_id.clone(), @@ -357,17 +368,15 @@ impl SharedDispatcher { redelivery_count: redelivery_count + 1, sticky_key_hash: None, }); - consumer.add_permits(1).await; - self.total_available_permits.fetch_add(1, Ordering::Relaxed); + consumer.add_permits(batch_count).await; + self.total_available_permits + .fetch_add(batch_count as i32, Ordering::Relaxed); // Send failed: connection/write path is saturated. // Stop this pass instead of immediately re-reading the // same entry (previous `continue` caused the spin). break; } } else { - // Message no longer exists (may have been deleted), restore permit - consumer.add_permits(1).await; - self.total_available_permits.fetch_add(1, Ordering::Relaxed); log::warn!( "Redelivery message {}:{} not found in storage", msg_id.ledger, @@ -383,6 +392,16 @@ impl SharedDispatcher { .await?; if let Some(candidate) = message_opt { + let batch_count = crate::broker::dispatcher::messages_in_batch(&candidate.metadata); + if !consumer.try_use_permits(batch_count).await { + log::warn!( + "Consumer {} permits insufficient for batch of {} messages, waiting for flow", + consumer_id, batch_count + ); + break; + } + self.total_available_permits + .fetch_sub(batch_count as i32, Ordering::Relaxed); if consumer .send_message( candidate.message_id.clone(), @@ -406,17 +425,15 @@ impl SharedDispatcher { } else { self.add_to_redelivery_queue(vec![(candidate.message_id, 1)]); commit_read_position(&self.read_position, candidate.next_position); - consumer.add_permits(1).await; - self.total_available_permits.fetch_add(1, Ordering::Relaxed); + consumer.add_permits(batch_count).await; + self.total_available_permits + .fetch_add(batch_count as i32, Ordering::Relaxed); // Send failed: connection/write path is saturated. Stop // this pass rather than queueing the whole batch for // redelivery and re-reading in a tight loop. break; } } else { - // No more messages, restore permit - consumer.add_permits(1).await; - self.total_available_permits.fetch_add(1, Ordering::Relaxed); log::debug!("No more dispatchable messages"); break; } @@ -652,7 +669,7 @@ impl Dispatcher for SharedDispatcher { // Consumer-local permit state is updated by the flow handler before it // triggers dispatch. The dispatcher only tracks the aggregate count. self.total_available_permits - .fetch_add(additional_permits, Ordering::Relaxed); + .fetch_add(additional_permits as i32, Ordering::Relaxed); log::info!( "Consumer {} flowed {} permits, total={}", diff --git a/rust/src/broker/handler/consumer_handler.rs b/rust/src/broker/handler/consumer_handler.rs index 98b6f9d..f379ef8 100644 --- a/rust/src/broker/handler/consumer_handler.rs +++ b/rust/src/broker/handler/consumer_handler.rs @@ -212,13 +212,18 @@ pub async fn handle_flow( // dispatcher-level aggregates. consumer.add_permits(flow_cmd.message_permits).await; - // Flow permits to dispatcher and trigger dispatch via Subscription + // Flow permits to dispatcher and trigger dispatch via Subscription. + // Dispatch must run in its own task: a dispatch loop can keep running + // while triggers keep arriving (should_reschedule), so awaiting it here + // would block this connection's event loop and starve reads/acks + // (observed as channel growth + OOM under 8-subscription fanout). let consumer_id = consumer.consumer_id; + let permits = flow_cmd.message_permits; let subscription = consumer.get_subscription(); - let sub_guard = subscription.read().await; - sub_guard - .consumer_flow(consumer_id, flow_cmd.message_permits) - .await; + tokio::spawn(async move { + let sub_guard = subscription.read().await; + sub_guard.consumer_flow(consumer_id, permits).await; + }); Ok(()) } @@ -232,28 +237,6 @@ where T: tokio::io::AsyncRead + tokio::io::AsyncWrite + Unpin, { let ack_cmd = cmd.ack.as_ref().ok_or("Missing ack command")?; - // [TEMP DIAG] count ack command arrivals and parse results. - { - use std::io::Write; - use std::sync::atomic::{AtomicU64, Ordering}; - static C: AtomicU64 = AtomicU64::new(0); - let n = C.fetch_add(1, Ordering::Relaxed); - if n % 100 == 0 { - let mid_count = ack_cmd.message_id.len(); - let known = consumers.contains_key(&ack_cmd.consumer_id); - if let Ok(mut f) = std::fs::OpenOptions::new() - .create(true) - .append(true) - .open("/data/ack_cmd_diag.txt") - { - let _ = writeln!( - f, - "{},\tconsumer={},\tids={},\tknown={},\tack_type={:?}", - n, ack_cmd.consumer_id, mid_count, known, ack_cmd.ack_type - ); - } - } - } log::debug!( "Handling Ack command: consumer_id={}, ack_type={:?}", ack_cmd.consumer_id, diff --git a/rust/src/broker/service/connection_write_state.rs b/rust/src/broker/service/connection_write_state.rs index 6a734d4..5153fe3 100644 --- a/rust/src/broker/service/connection_write_state.rs +++ b/rust/src/broker/service/connection_write_state.rs @@ -1,5 +1,11 @@ use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; +/// Byte budget for the per-connection message channel (dispatcher -> socket). +/// Dispatch pauses when buffered-but-unencoded message bytes exceed this, +/// so fan-out scenarios (e.g. 8 subscriptions) cannot flood the channel +/// faster than the client can drain it. Roughly 64 entries of ~1MB each. +const DEFAULT_MAX_CHANNEL_BYTES: usize = 64 * 1024 * 1024; + /// Shared connection-level outbound write state. /// /// This approximates Pulsar/Netty channel writability semantics: once the @@ -12,6 +18,9 @@ pub struct ConnectionWriteState { writable: AtomicBool, high_watermark_bytes: usize, low_watermark_bytes: usize, + /// Unencoded message bytes sitting in the mpsc channel (dispatcher side). + channel_pending_bytes: AtomicUsize, + max_channel_bytes: usize, } impl ConnectionWriteState { @@ -26,6 +35,8 @@ impl ConnectionWriteState { writable: AtomicBool::new(true), high_watermark_bytes, low_watermark_bytes, + channel_pending_bytes: AtomicUsize::new(0), + max_channel_bytes: DEFAULT_MAX_CHANNEL_BYTES, } } @@ -51,6 +62,36 @@ impl ConnectionWriteState { self.writable.store(next_writable, Ordering::Release); } + /// Try to reserve `bytes` of channel budget. Returns false (and reserves + /// nothing) when the channel would exceed the byte cap. + pub fn try_reserve_channel_bytes(&self, bytes: usize) -> bool { + let current = self.channel_pending_bytes.load(Ordering::Relaxed); + loop { + let next = current.saturating_add(bytes); + if next > self.max_channel_bytes { + return false; + } + match self.channel_pending_bytes.compare_exchange_weak( + current, + next, + Ordering::Relaxed, + Ordering::Relaxed, + ) { + Ok(_) => return true, + Err(actual) => { + let current = actual; + let _ = current; + } + } + } + } + + /// Release channel budget after the run loop dequeues a message. + pub fn release_channel_bytes(&self, bytes: usize) { + self.channel_pending_bytes + .fetch_sub(bytes, Ordering::Relaxed); + } + pub fn high_watermark_bytes(&self) -> usize { self.high_watermark_bytes } diff --git a/rust/src/broker/service/consumer.rs b/rust/src/broker/service/consumer.rs index 789ffe9..fb257c9 100644 --- a/rust/src/broker/service/consumer.rs +++ b/rust/src/broker/service/consumer.rs @@ -5,7 +5,7 @@ use bytes::Bytes; use std::sync::{ - atomic::{AtomicBool, AtomicI64, AtomicU32, Ordering}, + atomic::{AtomicBool, AtomicI32, AtomicI64, Ordering}, Arc, }; @@ -60,7 +60,7 @@ pub struct PendingMessage { /// Call `send()` to commit the dispatch. If dropped without sending, all /// resources are automatically rolled back (dispatch-or-drop semantics). pub struct DispatchReservation { - available_permits: Arc, + available_permits: Arc, pending_acks: Arc, owned_permit: Option>, consumer_id: u64, @@ -119,7 +119,7 @@ pub struct Consumer { /// Statistics stats: Arc>, - available_permits: Arc, + available_permits: Arc, /// Message sender channel - sends messages to ServerCnx for delivery /// Format: (consumer_id, PendingMessage) @@ -192,7 +192,7 @@ impl Consumer { subscription, connection_id, stats: Arc::new(RwLock::new(ConsumerStats::default())), - available_permits: Arc::new(AtomicU32::new(0)), + available_permits: Arc::new(AtomicI32::new(0)), message_tx, connection_write_state, pending_acks: Arc::new(PendingAcksMap::new()), @@ -205,7 +205,8 @@ impl Consumer { /// Update permits (flow control) pub async fn add_permits(&self, permits: u32) { - self.available_permits.fetch_add(permits, Ordering::Relaxed); + self.available_permits + .fetch_add(permits as i32, Ordering::Relaxed); } /// Instrumentation/gate: dispatched-but-unacked message count for this consumer. @@ -230,8 +231,33 @@ impl Consumer { false } + /// Use `n` permits when dispatching a batch entry. + /// + /// Permit accounting is per client-visible message (Apache Pulsar + /// semantics): a batch entry of N messages consumes N permits. Like Pulsar, + /// the balance may briefly go negative when a batch entry is larger than + /// the remaining permits — the entry is still delivered and the deficit is + /// repaid by the client's next Flow. This keeps the message stream flowing + /// (no deadlock where the client waits for messages and the broker waits + /// for Flow). Returns false only when there is no permit at all. + pub async fn try_use_permits(&self, n: u32) -> bool { + let mut current = self.available_permits.load(Ordering::Relaxed); + while current > 0 { + match self.available_permits.compare_exchange( + current, + current - n as i32, + Ordering::Relaxed, + Ordering::Relaxed, + ) { + Ok(_) => return true, + Err(observed) => current = observed, + } + } + false + } + /// Get available permits - pub async fn get_available_permits(&self) -> u32 { + pub async fn get_available_permits(&self) -> i32 { self.available_permits.load(Ordering::Relaxed) } @@ -251,7 +277,7 @@ impl Consumer { /// Get current statistics pub async fn get_stats(&self) -> ConsumerStats { let mut stats = self.stats.read().await.clone(); - stats.available_permits = self.available_permits.load(Ordering::Relaxed); + stats.available_permits = self.available_permits.load(Ordering::Relaxed).max(0) as u32; let active_consumer_id = self.active_consumer_id.load(Ordering::Relaxed); stats.active_consumer_id = (active_consumer_id >= 0).then_some(active_consumer_id as u64); stats.is_active_consumer = self.is_active_consumer.load(Ordering::Relaxed); @@ -357,32 +383,18 @@ impl Consumer { M: Into, P: Into, { - // [TEMP DIAG] count dispatches to consumers. - { - use std::io::Write; - use std::sync::atomic::{AtomicU64, Ordering}; - static C: AtomicU64 = AtomicU64::new(0); - let n = C.fetch_add(1, Ordering::Relaxed); - if n % 100_000 == 0 { - if let Ok(mut f) = std::fs::OpenOptions::new() - .create(true) - .append(true) - .open("/data/dispatch_diag.txt") - { - let _ = writeln!( - f, - "{},\tconsumer={},\tmsg={}:{}\tredelivery={}", - n, - self.consumer_id, - message_id.ledger, - message_id.entry, - redelivery_count - ); - } - } - } let metadata = metadata.into(); let payload = payload.into(); + + // Backpressure: if this connection's outbound write buffer is backed up + // (socket cannot drain fast enough), stop dispatching instead of filling + // the message channel. The dispatcher returns the message to its + // redelivery queue and restores the permit; dispatch resumes once acks + // flow and the buffer drains below the low watermark. + if !self.is_writable() { + return false; + } + let wire_size = pulsar_lite_proto::codec::estimate_message_parts_size( self.consumer_id, message_id.ledger, @@ -655,7 +667,7 @@ impl Consumer { } pub fn available_permits_now(&self) -> u32 { - self.available_permits.load(Ordering::Relaxed) + self.available_permits.load(Ordering::Relaxed).max(0) as u32 } pub fn notify_active_consumer_change(&self, active_consumer_id: u64) { @@ -708,35 +720,6 @@ impl Consumer { ack_type: AckCommandType, message_ids: Vec, ) -> Result<(), String> { - // [TEMP DIAG] count message_acked entries with batch size. - { - use std::io::Write; - use std::sync::atomic::{AtomicU64, Ordering}; - static C: AtomicU64 = AtomicU64::new(0); - let n = C.fetch_add(1, Ordering::Relaxed); - if n % 100 == 0 { - let (diag_sub_type, diag_is_persistent) = { - let sub = self.subscription.read().await; - (sub.get_sub_type(), sub.is_persistent()) - }; - if let Ok(mut f) = std::fs::OpenOptions::new() - .create(true) - .append(true) - .open("/data/msg_acked_diag.txt") - { - let _ = writeln!( - f, - "{},\tconsumer={},\tids={},\tack_type={:?},\tsub_type={:?},\tpersistent={}", - n, - self.consumer_id, - message_ids.len(), - ack_type, - diag_sub_type, - diag_is_persistent - ); - } - } - } if message_ids.is_empty() { return Ok(()); } @@ -761,8 +744,6 @@ impl Consumer { let Some((owner, tracked_message_id)) = self.resolve_ack_owner(message_id).await else { - // [TEMP DIAG] ack dropped: no owner found. - diag_ack_drop("no_owner", message_id); log::warn!( "Consumer {} attempted to ack message {}:{} without ownership; ignoring", self.consumer_id, @@ -773,8 +754,6 @@ impl Consumer { }; if !owner.remove_pending_ack(&tracked_message_id).await { - // [TEMP DIAG] ack dropped: pending-ack removal failed. - diag_ack_drop("remove_failed", &tracked_message_id); log::warn!( "Consumer {} found owner {} for message {}:{} but pending ack removal failed; ignoring", self.consumer_id, @@ -865,34 +844,6 @@ impl Consumer { } } -/// [TEMP DIAG] append one line per 100k dropped acks to /data/ack_drop_diag.txt. -fn diag_ack_drop(reason: &str, message_id: &MessageId) { - use std::io::Write; - use std::sync::atomic::{AtomicU64, Ordering}; - static COUNTER: AtomicU64 = AtomicU64::new(0); - let n = COUNTER.fetch_add(1, Ordering::Relaxed); - if n % 100_000 == 0 { - if let Ok(mut f) = std::fs::OpenOptions::new() - .create(true) - .append(true) - .open("/data/ack_drop_diag.txt") - { - let _ = writeln!( - f, - "{},\t{},\t{}:{}\tpartition={}", - std::time::SystemTime::now() - .duration_since(std::time::UNIX_EPOCH) - .map(|d| d.as_secs()) - .unwrap_or(0), - reason, - message_id.ledger, - message_id.entry, - message_id.partition - ); - } - } -} - impl PartialEq for Consumer { fn eq(&self, other: &Self) -> bool { self.consumer_id == other.consumer_id && self.connection_id == other.connection_id diff --git a/rust/src/broker/service/pending_acks.rs b/rust/src/broker/service/pending_acks.rs index 6c8e029..843f382 100644 --- a/rust/src/broker/service/pending_acks.rs +++ b/rust/src/broker/service/pending_acks.rs @@ -61,7 +61,7 @@ impl PendingAcksMap { if let Ok(mut f) = std::fs::OpenOptions::new() .create(true) .append(true) - .open("/data/pending_acks_diag.txt") + .open("/tmp/pending_acks_diag.txt") { let _ = writeln!(f, "{},\tinserts={},\tlen={}", n, n, len); } diff --git a/rust/src/broker/service/server_cnx.rs b/rust/src/broker/service/server_cnx.rs index 22958cf..f30ff60 100644 --- a/rust/src/broker/service/server_cnx.rs +++ b/rust/src/broker/service/server_cnx.rs @@ -7,6 +7,7 @@ use futures::future::pending; use futures::{SinkExt, StreamExt}; use prost::Message; use std::collections::HashMap; +use std::collections::VecDeque; use std::error::Error; use std::sync::Arc; use tokio::sync::mpsc; @@ -16,8 +17,8 @@ use tokio_util::codec::Framed; use super::consumer::PendingMessage; use super::{ConnectionWriteState, Consumer, Producer, SharedStorage}; use crate::broker::broker_service::SharedBrokerService; -use crate::broker::{Topic, handler}; use crate::broker::service::topic::TopicPublishRateExceeded; +use crate::broker::{handler, Topic}; use pulsar_lite_proto::codec::{ proto::pulsar::{base_command, BaseCommand, ProtocolVersion, ServerError}, PulsarFrame, PulsarFrameCodec, @@ -56,7 +57,7 @@ type CnxError = Box; type CnxResult = Result; #[cfg(feature = "rocksdb-storage")] -use pulsar_lite_storage_managed_ledger_rocksdb::{ConnAppendResult, ConcurrentAppender}; +use pulsar_lite_storage_managed_ledger_rocksdb::{ConcurrentAppender, ConnAppendResult}; fn to_cnx_error(error: impl ToString) -> CnxError { Box::new(std::io::Error::other(error.to_string())) @@ -151,12 +152,21 @@ where /// Number of in-flight publish tasks on this connection /// (non-persistent sync path + persistent pipelined path). pending_send_requests: usize, + /// In-flight publish bytes on this connection (payload + metadata). + /// Byte-level counterpart of pending_send_requests: caps memory for + /// large-message workloads where 1000 requests could hold GBs. + pending_send_bytes: usize, + /// FIFO of in-flight persistent publish sizes, pairing enqueue order with + /// completion order (write-queue is single-worker FIFO) for exact accounting. + pending_send_sizes: VecDeque, /// Drop gate: messages are silently dropped with fake receipt when this is exceeded. /// Maps to Pulsar's maxConcurrentNonPersistentMessagePerConnection. max_concurrent_non_persistent: usize, /// TCP throttle gate: reading from framed stops when pending reaches this limit. /// Maps to Pulsar's maxPendingPublishRequestsPerConnection. max_pending_publish_requests: usize, + /// Byte-level TCP throttle high watermark (hysteresis: resume at 50%). + max_pending_publish_bytes: usize, /// Cap for pipelined persistent Send tasks on this connection. #[cfg(feature = "rocksdb-storage")] max_persistent_in_flight: usize, @@ -166,6 +176,8 @@ where persistent_appender: Option, /// Resume reading threshold (hysteresis = max_pending_publish_requests / 2). resume_threshold: usize, + /// Resume reading threshold for bytes (max_pending_publish_bytes / 2). + resume_bytes_threshold: usize, /// When true, the event loop skips framed.next(), equivalent to Netty auto-read = false. read_paused: bool, /// Maximum message size accepted by the broker. @@ -214,12 +226,12 @@ where connection_liveness_check_timeout: Duration, max_concurrent_non_persistent: usize, max_pending_publish_requests: usize, + max_pending_publish_bytes: usize, max_message_size: usize, broker_service_url: String, channel_write_buffer_high_water_mark_bytes: usize, channel_write_buffer_low_water_mark_bytes: usize, - #[cfg(feature = "rocksdb-storage")] - persistent_appender: Option, + #[cfg(feature = "rocksdb-storage")] persistent_appender: Option, ) -> Self { let mut framed = Framed::new(socket, PulsarFrameCodec::new()); framed.set_backpressure_boundary(channel_write_buffer_high_water_mark_bytes); @@ -251,13 +263,17 @@ where connection_id, topic_manager, pending_send_requests: 0, + pending_send_bytes: 0, + pending_send_sizes: VecDeque::new(), max_concurrent_non_persistent, max_pending_publish_requests, + max_pending_publish_bytes, #[cfg(feature = "rocksdb-storage")] max_persistent_in_flight: DEFAULT_MAX_PERSISTENT_IN_FLIGHT, #[cfg(feature = "rocksdb-storage")] persistent_appender, resume_threshold: max_pending_publish_requests / 2, + resume_bytes_threshold: max_pending_publish_bytes / 2, read_paused: false, max_message_size, #[cfg(feature = "rocksdb-storage")] @@ -298,7 +314,7 @@ where tokio::select! { // Inbound protocol commands — skipped when read_paused (TCP backpressure). - frame_result = self.framed.next(), if !self.read_paused && self.connection_write_state.is_writable() => { + frame_result = self.framed.next(), if !self.read_paused => { let Some(frame) = frame_result else { self.close_reason.get_or_insert(CloseReason::ClientClosed); break Ok(()); @@ -391,7 +407,7 @@ where tokio::select! { // Inbound protocol commands — skipped when read_paused (TCP backpressure). - frame_result = self.framed.next(), if !self.read_paused && self.connection_write_state.is_writable() => { + frame_result = self.framed.next(), if !self.read_paused => { let Some(frame) = frame_result else { self.close_reason.get_or_insert(CloseReason::ClientClosed); break Ok(()); @@ -747,14 +763,28 @@ where .map_err(to_cnx_error) } + fn maybe_pause_read(&mut self) { + if self.pending_send_requests >= self.max_pending_publish_requests + || self.pending_send_bytes >= self.max_pending_publish_bytes + { + self.read_paused = true; + } + } + fn maybe_resume_read_after_send(&mut self) { - if self.read_paused && self.pending_send_requests <= self.resume_threshold { + if self.read_paused + && self.pending_send_requests <= self.resume_threshold + && self.pending_send_bytes <= self.resume_bytes_threshold + { self.read_paused = false; } } async fn complete_persistent_send(&mut self, outcome: PersistentSendOutcome) -> CnxResult<()> { self.pending_send_requests = self.pending_send_requests.saturating_sub(1); + if let Some(size) = self.pending_send_sizes.pop_front() { + self.pending_send_bytes = self.pending_send_bytes.saturating_sub(size); + } self.maybe_resume_read_after_send(); match outcome.result { @@ -838,13 +868,13 @@ where } self.pending_send_requests += 1; - if self.pending_send_requests >= self.max_pending_publish_requests { - self.read_paused = true; - } + self.pending_send_bytes += message_size; + self.maybe_pause_read(); let result = handler::handle_send(&mut self.framed, cmd, frame, &self.producers).await; self.pending_send_requests = self.pending_send_requests.saturating_sub(1); + self.pending_send_bytes = self.pending_send_bytes.saturating_sub(message_size); self.maybe_resume_read_after_send(); return match result { @@ -927,18 +957,19 @@ where producer.record_message_sent(payload_len + meta_len).await; self.pending_send_requests += 1; - if self.pending_send_requests >= self.max_pending_publish_requests { - self.read_paused = true; - } + let send_size = payload_len + meta_len; + self.pending_send_bytes += send_size; + self.pending_send_sizes.push_back(send_size); + self.maybe_pause_read(); return Ok(()); } } // Memory backend fallback (unit tests): await publish on this task. self.pending_send_requests += 1; - if self.pending_send_requests >= self.max_pending_publish_requests { - self.read_paused = true; - } + self.pending_send_bytes += message_size; + self.pending_send_sizes.push_back(message_size); + self.maybe_pause_read(); let metadata = frame.metadata.clone(); let payload = frame.payload.clone(); match handler::publish_persistent_send( @@ -960,6 +991,9 @@ where } Err(error) => { self.pending_send_requests = self.pending_send_requests.saturating_sub(1); + if let Some(size) = self.pending_send_sizes.pop_front() { + self.pending_send_bytes = self.pending_send_bytes.saturating_sub(size); + } self.maybe_resume_read_after_send(); if let Some(rate_error) = error.downcast_ref::() { self.framed @@ -1078,6 +1112,7 @@ pub async fn handle_connection( connection_liveness_check_timeout: Duration, max_non_persistent_pending_messages: usize, max_pending_publish_requests: usize, + max_pending_publish_bytes: usize, max_message_size: usize, broker_service_url: String, channel_write_buffer_high_water_mark_bytes: usize, @@ -1107,6 +1142,7 @@ pub async fn handle_connection( connection_liveness_check_timeout, max_non_persistent_pending_messages, max_pending_publish_requests, + max_pending_publish_bytes, max_message_size, broker_service_url, channel_write_buffer_high_water_mark_bytes, @@ -1152,12 +1188,11 @@ mod tests { Duration::from_secs(10), 1000, 1000, + 256 * 1024 * 1024, 5 * 1024 * 1024, "pulsar://127.0.0.1:6650".to_string(), 64 * 1024, 32 * 1024, - #[cfg(feature = "rocksdb-storage")] - None, ); let client_framed = Framed::new(client, PulsarFrameCodec::new()); (server_cnx, client_framed, test_dir) diff --git a/rust/src/broker/service/topic/subscription.rs b/rust/src/broker/service/topic/subscription.rs index b34bc03..168010e 100644 --- a/rust/src/broker/service/topic/subscription.rs +++ b/rust/src/broker/service/topic/subscription.rs @@ -678,18 +678,18 @@ impl Subscription { /// Get total available permits across all consumers pub async fn get_total_permits(&self) -> u32 { - let mut total = 0; + let mut total = 0i32; for consumer in self.get_consumers() { total += consumer.get_available_permits().await; } - total + total.max(0) as u32 } /// Get subscription statistics pub async fn get_stats(&self) -> SubscriptionStats { let consumers = self.get_consumers(); let consumer_count = consumers.len(); - let mut total_permits = 0; + let mut total_permits = 0i32; for consumer in consumers { total_permits += consumer.get_available_permits().await; } @@ -699,7 +699,7 @@ impl Subscription { topic: self.topic.clone(), sub_type: self.sub_type, consumer_count, - total_permits, + total_permits: total_permits.max(0) as u32, received_messages: match self.runtime_mode { SubscriptionRuntimeMode::Persistent => 0, SubscriptionRuntimeMode::NonPersistent => self From cc4b065fe359429dd0a90fe3662289b9e7f56a95 Mon Sep 17 00:00:00 2001 From: XTLine <2668543450@qq.com> Date: Thu, 13 Aug 2026 16:33:02 +0800 Subject: [PATCH 27/44] Add pending publish byte limits and connection write-state config --- rust/src/config.rs | 16 ++++++++++++++++ rust/src/main.rs | 4 +++- 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/rust/src/config.rs b/rust/src/config.rs index 244535b..e8fdd23 100644 --- a/rust/src/config.rs +++ b/rust/src/config.rs @@ -59,6 +59,14 @@ pub struct Config { #[serde(default = "default_max_pending_publish_requests_per_connection")] pub max_pending_publish_requests_per_connection: usize, + /// Byte-level TCP throttle high watermark per connection. + /// In-flight publish bytes reaching this limit also pause reads (same + /// hysteresis as the request-count limit: resume at 50%). This caps memory + /// for large-message workloads where 1000 requests could hold GBs. + /// Default: 256 MiB + #[serde(default = "default_max_pending_publish_bytes_per_connection")] + pub max_pending_publish_bytes_per_connection: usize, + /// Maximum allowed message size in bytes. #[serde(default = "default_max_message_size_bytes")] pub max_message_size_bytes: usize, @@ -114,6 +122,10 @@ fn default_max_pending_publish_requests_per_connection() -> usize { 1000 } +fn default_max_pending_publish_bytes_per_connection() -> usize { + 256 * 1024 * 1024 +} + fn default_max_message_size_bytes() -> usize { 5 * 1024 * 1024 } @@ -143,6 +155,8 @@ impl Default for Config { default_max_concurrent_non_persistent_messages_per_connection(), max_pending_publish_requests_per_connection: default_max_pending_publish_requests_per_connection(), + max_pending_publish_bytes_per_connection: + default_max_pending_publish_bytes_per_connection(), max_message_size_bytes: default_max_message_size_bytes(), publish_rate_messages_per_sec: 0, publish_rate_bytes_per_sec: 0, @@ -271,6 +285,7 @@ managed_ledger_store = "rocksdb" max_connections_per_ip: 8, max_concurrent_non_persistent_messages_per_connection: 10000, max_pending_publish_requests_per_connection: 2000, + max_pending_publish_bytes_per_connection: 512 * 1024 * 1024, max_message_size_bytes: 2048, publish_rate_messages_per_sec: 123, publish_rate_bytes_per_sec: 456, @@ -338,6 +353,7 @@ managed_ledger_store = "rocksdb" max_connections_per_ip: 8, max_concurrent_non_persistent_messages_per_connection: 10000, max_pending_publish_requests_per_connection: 2000, + max_pending_publish_bytes_per_connection: 512 * 1024 * 1024, max_message_size_bytes: 2048, publish_rate_messages_per_sec: 123, publish_rate_bytes_per_sec: 456, diff --git a/rust/src/main.rs b/rust/src/main.rs index 71f6bea..9909e22 100644 --- a/rust/src/main.rs +++ b/rust/src/main.rs @@ -79,9 +79,10 @@ async fn main() -> Result<(), Box> { config.max_connections_per_ip ); log::info!( - "Non-persistent limits: max_concurrent_per_connection={}, max_pending_publish={}, max_message_size={}B", + "Non-persistent limits: max_concurrent_per_connection={}, max_pending_publish={}, max_pending_publish_bytes={}MiB, max_message_size={}B", config.max_concurrent_non_persistent_messages_per_connection, config.max_pending_publish_requests_per_connection, + config.max_pending_publish_bytes_per_connection / (1024 * 1024), config.max_message_size_bytes ); log::info!( @@ -159,6 +160,7 @@ async fn main() -> Result<(), Box> { connection_liveness_check_timeout, config.max_concurrent_non_persistent_messages_per_connection, config.max_pending_publish_requests_per_connection, + config.max_pending_publish_bytes_per_connection, config.max_message_size_bytes, advertised_broker_url, config.pulsar_channel_write_buffer_high_water_mark_bytes, From 970e76b891625e3f9c5e4c6380447b71fdc857d8 Mon Sep 17 00:00:00 2001 From: XTLine <2668543450@qq.com> Date: Thu, 13 Aug 2026 16:37:05 +0800 Subject: [PATCH 28/44] chore(broker): remove TEMP DIAG instrumentation probes --- rust/src/broker/service/pending_acks.rs | 17 ---------- .../managed-ledger-rocksdb/src/cursor.rs | 32 ------------------- .../managed-ledger-rocksdb/src/store.rs | 32 ------------------- 3 files changed, 81 deletions(-) diff --git a/rust/src/broker/service/pending_acks.rs b/rust/src/broker/service/pending_acks.rs index 843f382..6098ae2 100644 --- a/rust/src/broker/service/pending_acks.rs +++ b/rust/src/broker/service/pending_acks.rs @@ -50,23 +50,6 @@ impl PendingAcksMap { sticky_key_hash, }, ); - // [TEMP DIAG] log pending-acks size every 100k inserts. - { - use std::io::Write; - use std::sync::atomic::{AtomicU64, Ordering}; - static C: AtomicU64 = AtomicU64::new(0); - let n = C.fetch_add(1, Ordering::Relaxed); - if n % 100_000 == 0 { - let len = inner.len(); - if let Ok(mut f) = std::fs::OpenOptions::new() - .create(true) - .append(true) - .open("/tmp/pending_acks_diag.txt") - { - let _ = writeln!(f, "{},\tinserts={},\tlen={}", n, n, len); - } - } - } true } diff --git a/rust/storage/managed-ledger-rocksdb/src/cursor.rs b/rust/storage/managed-ledger-rocksdb/src/cursor.rs index 990c219..f74fd05 100644 --- a/rust/storage/managed-ledger-rocksdb/src/cursor.rs +++ b/rust/storage/managed-ledger-rocksdb/src/cursor.rs @@ -43,38 +43,6 @@ impl RocksDBManagedCursor { } pub fn persist_state(&self) -> Result<()> { - // [TEMP DIAG] every 100k persists, append cursor state to a CSV file - // (bypasses the logging system entirely) to verify whether the - // individual-delete range set accumulates while mark_delete stalls. - use std::sync::atomic::{AtomicU64, Ordering}; - static DIAG_COUNT: AtomicU64 = AtomicU64::new(0); - let n = DIAG_COUNT.fetch_add(1, Ordering::Relaxed); - if n % 100_000 == 0 { - use std::io::Write; - let mark_delete = self - .state - .mark_delete - .as_ref() - .map(|p| format!("{}:{}", p.ledger_id, p.entry_id)) - .unwrap_or_else(|| "None".to_string()); - let line = format!( - "{},\t{},\t{},\t{}\n", - std::time::SystemTime::now() - .duration_since(std::time::UNIX_EPOCH) - .map(|d| d.as_secs()) - .unwrap_or(0), - self.name, - mark_delete, - self.state.individually_deleted_entries.len() - ); - if let Ok(mut f) = std::fs::OpenOptions::new() - .create(true) - .append(true) - .open("/data/cursor_diag.csv") - { - let _ = f.write_all(line.as_bytes()); - } - } let key = keys::managed_cursor_key(&self.managedledger_name, &self.name); let stored = StoredManagedCursorState::from(self.state.clone()); self.db.put(key, stored.encode_to_vec())?; diff --git a/rust/storage/managed-ledger-rocksdb/src/store.rs b/rust/storage/managed-ledger-rocksdb/src/store.rs index 872b3a4..9aed3f7 100644 --- a/rust/storage/managed-ledger-rocksdb/src/store.rs +++ b/rust/storage/managed-ledger-rocksdb/src/store.rs @@ -362,22 +362,6 @@ impl ManagedLedgerStorage for RocksDbManagedLedgerStorage { subscription: &str, message_id: MessageId, ) -> Result<()> { - // [TEMP DIAG] count ack entries reaching storage layer (inline). - { - use std::io::Write; - use std::sync::atomic::{AtomicU64, Ordering}; - static C: AtomicU64 = AtomicU64::new(0); - let n = C.fetch_add(1, Ordering::Relaxed); - if n % 100_000 == 0 { - if let Ok(mut f) = std::fs::OpenOptions::new() - .create(true) - .append(true) - .open("/data/ack_enter_diag.txt") - { - let _ = writeln!(f, "{},\tack_message,\t{}", n, n); - } - } - } self.with_cursor_lock(|| { let cursor_name = keys::encode_cursor_name(subscription); let ledger = self.topic_ledger(topic)?; @@ -392,22 +376,6 @@ impl ManagedLedgerStorage for RocksDbManagedLedgerStorage { subscription: &str, message_id: MessageId, ) -> Result<()> { - // [TEMP DIAG] count ack entries reaching storage layer (inline). - { - use std::io::Write; - use std::sync::atomic::{AtomicU64, Ordering}; - static C: AtomicU64 = AtomicU64::new(0); - let n = C.fetch_add(1, Ordering::Relaxed); - if n % 10_000 == 0 { - if let Ok(mut f) = std::fs::OpenOptions::new() - .create(true) - .append(true) - .open("/data/ack_enter_diag.txt") - { - let _ = writeln!(f, "{},\tack_message_shared,\t{}", n, n); - } - } - } self.with_cursor_lock(|| { let cursor_name = keys::encode_cursor_name(subscription); let ledger = self.topic_ledger(topic)?; From 6acc43808815707a691b6cd2f7b8b5d174d733d7 Mon Sep 17 00:00:00 2001 From: XTLine <2668543450@qq.com> Date: Fri, 21 Aug 2026 16:35:39 +0800 Subject: [PATCH 29/44] style: cargo fmt and clippy 1.98 fixes (while-let loop in cursor) --- rust/src/broker/handler/producer_handler.rs | 2 +- rust/src/broker/service/topic/topic.rs | 2 +- rust/storage/managed-ledger-rocksdb/src/cursor.rs | 5 +---- 3 files changed, 3 insertions(+), 6 deletions(-) diff --git a/rust/src/broker/handler/producer_handler.rs b/rust/src/broker/handler/producer_handler.rs index 536ad39..1ae4df5 100644 --- a/rust/src/broker/handler/producer_handler.rs +++ b/rust/src/broker/handler/producer_handler.rs @@ -3,10 +3,10 @@ * Handles producer-related commands: Producer, Send, CloseProducer */ -use crate::broker::Topic; use crate::broker::broker_service::{SharedBrokerService, TopicRef}; use crate::broker::service::topic::TopicRuntimeMode; use crate::broker::service::Producer; +use crate::broker::Topic; use futures::SinkExt; use pulsar_lite_proto::codec::{proto::pulsar::BaseCommand, PulsarFrame, PulsarFrameCodec}; use pulsar_lite_proto::ServerCommand; diff --git a/rust/src/broker/service/topic/topic.rs b/rust/src/broker/service/topic/topic.rs index 9369698..a78f04a 100644 --- a/rust/src/broker/service/topic/topic.rs +++ b/rust/src/broker/service/topic/topic.rs @@ -562,7 +562,7 @@ impl Topic { topic_guard.dispatch_to_subscriptions().await; }); } - + /// Dispatch messages to all subscriptions (Push mode - Apache Pulsar style) /// /// This should be called after publish_message() to push messages to consumers. diff --git a/rust/storage/managed-ledger-rocksdb/src/cursor.rs b/rust/storage/managed-ledger-rocksdb/src/cursor.rs index f74fd05..28c84a4 100644 --- a/rust/storage/managed-ledger-rocksdb/src/cursor.rs +++ b/rust/storage/managed-ledger-rocksdb/src/cursor.rs @@ -160,10 +160,7 @@ pub fn ack_managed_cursor_shared( // Advance the mark-delete frontier across contiguous acknowledged ranges. // `take_covering` consumes a whole coalesced range in one step. - loop { - let Some(mark_delete) = cursor.state().mark_delete.clone() else { - break; - }; + while let Some(mark_delete) = cursor.state().mark_delete.clone() { let Some(next) = next_position(&mark_delete, info) else { break; }; From b4dc1c2fa38e8b353ff6b0d3d5e3274d9e792c5a Mon Sep 17 00:00:00 2001 From: XTLine <2668543450@qq.com> Date: Fri, 21 Aug 2026 16:36:48 +0800 Subject: [PATCH 30/44] fix(broker): restore persistent_appender arg and drop redundant u32 casts for clippy 1.98 --- rust/src/broker/dispatcher/key_shared.rs | 4 ++-- rust/src/broker/service/server_cnx.rs | 2 ++ 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/rust/src/broker/dispatcher/key_shared.rs b/rust/src/broker/dispatcher/key_shared.rs index 030ce34..32b67c1 100644 --- a/rust/src/broker/dispatcher/key_shared.rs +++ b/rust/src/broker/dispatcher/key_shared.rs @@ -353,7 +353,7 @@ impl KeySharedDispatcher { break; } remaining_dispatches -= 1; - self.subtract_total_permits(batch_count as u32); + self.subtract_total_permits(batch_count); if consumer .send_message_with_sticky_hash( @@ -425,7 +425,7 @@ impl KeySharedDispatcher { break; } remaining_dispatches -= 1; - self.subtract_total_permits(batch_count as u32); + self.subtract_total_permits(batch_count); if consumer .send_message_with_sticky_hash( diff --git a/rust/src/broker/service/server_cnx.rs b/rust/src/broker/service/server_cnx.rs index f30ff60..687665d 100644 --- a/rust/src/broker/service/server_cnx.rs +++ b/rust/src/broker/service/server_cnx.rs @@ -1193,6 +1193,8 @@ mod tests { "pulsar://127.0.0.1:6650".to_string(), 64 * 1024, 32 * 1024, + #[cfg(feature = "rocksdb-storage")] + None, ); let client_framed = Framed::new(client, PulsarFrameCodec::new()); (server_cnx, client_framed, test_dir) From 3acae83b59ea1df966b971abdb69275370b233c2 Mon Sep 17 00:00:00 2001 From: XTLine <2668543450@qq.com> Date: Fri, 21 Aug 2026 17:57:11 +0800 Subject: [PATCH 31/44] fix(broker): apply flow permits synchronously under the subscription lock The backpressure change moved Flow handling into a spawned task, which split permit accounting in time: consumer-local permits were applied at frame processing while the dispatcher aggregate landed only when the task ran. Two races surfaced under CI timing: - key_shared: remove_consumer_with_recovery subtracts the consumer balance from the aggregate. With a refill Flow half-applied, the close zeroed the aggregate, and the flow task then dropped its own increment (contains_key guard), so the post-close redelivery dispatch exited on max_batch <= 0 and the surviving consumer never received the message. - non-persistent shared: send_messages dropped entries on a stale zero aggregate while the consumer actually held permits (the ack refill Flow from another connection had not been applied yet). Apply both permit layers synchronously under the subscription read lock (mutually exclusive with consumer removal writes), keep the dispatch trigger in its own task to preserve the event-loop starvation fix, and let per-consumer reservations instead of the aggregate fast path decide non-persistent drops. Verified locally: both previously-flaky tests now pass 30/30 runs (key_shared unacked redelivery preserves key owner, non-persist shared flow full consumer stops receiving); workspace tests and clippy 1.98 clean. --- rust/src/broker/handler/consumer_handler.rs | 41 +++++++++++++------ .../dispatcher/multiple_consumers.rs | 35 ++++++++-------- .../persistent/persistent_subscription.rs | 35 ++++++++-------- rust/src/broker/service/topic/subscription.rs | 29 +++++++++++++ 4 files changed, 91 insertions(+), 49 deletions(-) diff --git a/rust/src/broker/handler/consumer_handler.rs b/rust/src/broker/handler/consumer_handler.rs index f379ef8..f2c534c 100644 --- a/rust/src/broker/handler/consumer_handler.rs +++ b/rust/src/broker/handler/consumer_handler.rs @@ -207,23 +207,38 @@ pub async fn handle_flow( .ok_or_else(|| format!("Unknown consumer ID: {}", flow_cmd.consumer_id))?; // Native Pulsar updates permits on the consumer and then notifies the - // dispatcher/subscription path. Non-persistent single-active dispatchers - // read consumer-local permits directly, while shared variants also keep - // dispatcher-level aggregates. - consumer.add_permits(flow_cmd.message_permits).await; - - // Flow permits to dispatcher and trigger dispatch via Subscription. - // Dispatch must run in its own task: a dispatch loop can keep running - // while triggers keep arriving (should_reschedule), so awaiting it here - // would block this connection's event loop and starve reads/acks - // (observed as channel growth + OOM under 8-subscription fanout). + // dispatcher/subscription path. Both accounting layers (consumer-local + // and dispatcher aggregate) must be applied synchronously under the + // subscription read lock: remove_consumer_with_recovery subtracts the + // consumer-local balance from the aggregate, and non-persistent dispatch + // gates on permits from other connections, so a half-applied Flow (one + // layer updated, the other still queued in a task) either zeroed the + // aggregate on consumer removal or dropped entries a consumer actually + // had permits for. Readers share this lock; only subscribe/close writers + // can briefly delay it. let consumer_id = consumer.consumer_id; let permits = flow_cmd.message_permits; let subscription = consumer.get_subscription(); - tokio::spawn(async move { + let needs_dispatch_trigger = { let sub_guard = subscription.read().await; - sub_guard.consumer_flow(consumer_id, permits).await; - }); + consumer.add_permits(permits).await; + sub_guard.apply_flow_permits(consumer_id, permits); + sub_guard.is_persistent() + }; + + // Dispatch runs in its own task: a dispatch loop can keep running while + // triggers keep arriving (should_reschedule), so awaiting it here would + // block this connection's event loop and starve reads/acks (observed as + // channel growth + OOM under 8-subscription fanout). + if needs_dispatch_trigger { + let subscription = subscription.clone(); + tokio::spawn(async move { + let sub_guard = subscription.read().await; + if let Err(e) = sub_guard.dispatch_messages().await { + log::error!("Flow-triggered dispatch failed: {}", e); + } + }); + } Ok(()) } diff --git a/rust/src/broker/non_persistent/dispatcher/multiple_consumers.rs b/rust/src/broker/non_persistent/dispatcher/multiple_consumers.rs index 46ab633..6cf4d7d 100644 --- a/rust/src/broker/non_persistent/dispatcher/multiple_consumers.rs +++ b/rust/src/broker/non_persistent/dispatcher/multiple_consumers.rs @@ -123,19 +123,13 @@ impl NonPersistentDispatcherMultipleConsumers { let entry_count = entries.len() as u64; self.received_messages .fetch_add(entry_count, Ordering::Relaxed); - let mut pending_entries = entries.into_iter(); - while let Some(entry) = pending_entries.next() { - if self.total_available_permits.load(Ordering::Relaxed) == 0 { - log::debug!("Dropping non-persistent shared entry due to zero aggregate permits"); - self.record_drop(1); - entry.release(); - for remaining in pending_entries.by_ref() { - self.record_drop(1); - remaining.release(); - } - continue; - } - + for entry in entries { + // No aggregate-permit gate here: the aggregate can momentarily + // read zero while a Flow's permit update is still being applied + // by its spawned task, and dropping on that stale snapshot lost + // messages a consumer actually had permits for. Per-consumer + // reservations are the authority; genuine permit exhaustion + // fails `use_permit` immediately and drops below as before. let Some(start) = self.next_consumer_start_index() else { log::debug!("Dropping non-persistent shared entry due to no connected consumer"); self.record_drop(1); @@ -170,7 +164,9 @@ impl NonPersistentDispatcherMultipleConsumers { } if !delivered { - log::debug!("Dropping non-persistent shared entry due to no writable consumer"); + log::debug!( + "Dropping non-persistent shared entry: no consumer with permits or writability" + ); self.record_drop(1); } entry.release(); @@ -297,12 +293,15 @@ mod tests { } #[tokio::test] - async fn shared_dispatcher_limits_batch_by_aggregate_permits() { + async fn shared_dispatcher_uses_consumer_permits_when_aggregate_stale() { let subscription = create_test_subscription(); let (consumer, mut rx) = create_test_consumer(1, subscription); let mut dispatcher = NonPersistentDispatcherMultipleConsumers::new(); dispatcher.add_consumer(consumer.clone()).unwrap(); + // Reproduce the flow-application skew: consumer-local permits + // applied, dispatcher aggregate still stale. Deliveries must follow + // the consumer-local authority, not the stale aggregate. consumer.add_permits(2).await; dispatcher.consumer_flow(consumer.consumer_id, 1); @@ -316,9 +315,11 @@ mod tests { let first = rx.recv().await.expect("first message should be delivered"); assert_eq!(first.1.payload, b"first".to_vec()); + let second = rx.recv().await.expect("second message should be delivered"); + assert_eq!(second.1.payload, b"second".to_vec()); assert!(rx.try_recv().is_err()); - assert_eq!(dispatcher.dropped_messages(), 1); - assert_eq!(consumer.get_available_permits().await, 1); + assert_eq!(dispatcher.dropped_messages(), 0); + assert_eq!(consumer.get_available_permits().await, 0); } #[tokio::test] diff --git a/rust/src/broker/service/persistent/persistent_subscription.rs b/rust/src/broker/service/persistent/persistent_subscription.rs index 42544af..a48c471 100644 --- a/rust/src/broker/service/persistent/persistent_subscription.rs +++ b/rust/src/broker/service/persistent/persistent_subscription.rs @@ -213,28 +213,25 @@ impl PersistentSubscriptionRuntime { } } - pub(crate) async fn consumer_flow(&self, consumer_id: u64, additional_permits: u32) { + /// Apply a Flow's permits to the dispatcher aggregate without triggering + /// dispatch. Used by the flow handler, which applies both permit layers + /// synchronously under the subscription read lock and triggers dispatch + /// separately off the connection event loop. + pub(crate) fn apply_flow_permits(&self, consumer_id: u64, additional_permits: u32) { if let Some(dispatcher) = self.dispatcher.as_ref() { - log::debug!( - "Subscription '{}' received flow from consumer {}, permits={}", - self.name, - consumer_id, - additional_permits - ); dispatcher.consumer_flow(consumer_id, additional_permits); + } + } - if let Err(e) = dispatcher - .dispatch_messages(self.storage.clone(), self.topic.clone(), self.name.clone()) - .await - { - log::error!( - "Failed to dispatch messages for subscription '{}': {}", - self.name, - e - ); - } - } else { - log::warn!("No dispatcher found for subscription '{}'", self.name); + pub(crate) async fn consumer_flow(&self, consumer_id: u64, additional_permits: u32) { + self.apply_flow_permits(consumer_id, additional_permits); + + if let Err(e) = self.dispatch_messages().await { + log::error!( + "Failed to dispatch messages for subscription '{}': {}", + self.name, + e + ); } } } diff --git a/rust/src/broker/service/topic/subscription.rs b/rust/src/broker/service/topic/subscription.rs index 168010e..72c4698 100644 --- a/rust/src/broker/service/topic/subscription.rs +++ b/rust/src/broker/service/topic/subscription.rs @@ -798,6 +798,35 @@ impl Subscription { } } + /// Apply a Flow's permits to both accounting layers (consumer-local via + /// the handler, dispatcher aggregate here) without triggering dispatch. + /// Runs synchronously under the subscription read lock so a concurrent + /// consumer removal (write lock) never observes a half-applied Flow. + pub fn apply_flow_permits(&self, consumer_id: u64, additional_permits: u32) { + match self.runtime_mode { + SubscriptionRuntimeMode::Persistent => { + if let Some(runtime) = self.persistent_runtime.as_ref() { + runtime.apply_flow_permits(consumer_id, additional_permits); + } else { + log::warn!( + "No persistent runtime available for subscription '{}'", + self.name + ); + } + } + SubscriptionRuntimeMode::NonPersistent => { + if let Some(ref runtime) = self.non_persistent_runtime { + runtime.consumer_flow(consumer_id, additional_permits); + } else { + log::warn!( + "No non-persistent runtime available for subscription '{}'", + self.name + ); + } + } + } + } + /// Handle consumer flow command (Apache Pulsar style) pub async fn consumer_flow(&self, consumer_id: u64, additional_permits: u32) { match self.runtime_mode { From 6a9a8eb5e0e063ea49ee94b2afe0af26d2a05ec4 Mon Sep 17 00:00:00 2001 From: XTLine <2668543450@qq.com> Date: Tue, 18 Aug 2026 15:27:28 +0800 Subject: [PATCH 32/44] fix(broker): enable TCP_NODELAY on accepted sockets Rust keeps the OS default (Nagle on) for accepted sockets. The Pulsar protocol is request-response with small replies (Send -> SendReceipt, Ping -> Pong); with Nagle, each reply waits for the previous segment's ACK, adding ~1.4ms steady RTT. Netty defaults tcpNoDelay=true, so the Java broker never had this problem. Measured (8-sub non-persistent, 4 cores, same client): produce-only 417k -> 1000k msg/s (client cap), feed 397k -> 988k msg/s, p50 2.31ms -> 0.76ms. --- rust/src/main.rs | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/rust/src/main.rs b/rust/src/main.rs index 9909e22..7a317a3 100644 --- a/rust/src/main.rs +++ b/rust/src/main.rs @@ -129,6 +129,13 @@ async fn main() -> Result<(), Box> { } }; + // Request-response protocol (Send -> SendReceipt, Ping -> Pong): Nagle + // delays small replies until the previous segment is ACKed, adding + // millisecond-scale RTT. Disable it like Netty (tcpNoDelay=true default). + if let Err(e) = socket.set_nodelay(true) { + log::warn!("Failed to set TCP_NODELAY for {}: {}", peer_addr, e); + } + log::info!("New connection from {}", peer_addr); let permit = match connection_limiter.try_acquire(peer_addr.ip()) { From 8daa9e9dd3f931cc435197edd1d8bbf241a98679 Mon Sep 17 00:00:00 2001 From: XTLine <2668543450@qq.com> Date: Tue, 18 Aug 2026 15:27:50 +0800 Subject: [PATCH 33/44] perf(broker): ordered batched fan-out worker for non-persistent publish The non-persistent SEND path ran fully inline in the connection task: topic.write() exclusive lock -> prepare -> dispatch_sequential().await (8 subscriptions, one lock + one dispatcher call per message) -> receipt, pipeline depth 1. Mirror the persistent write-queue shape instead: - per-topic bounded mpsc (4096) drained by one ordered worker; the connection task snapshots subscriptions under a short read lock, enqueues, and returns to reading frames (accept-ack receipt, same contract as the drop gate's fake receipts) - prepare_non_persistent_publish takes &self (the write lock was phantom: both callees only need &self) - worker drains up to 128 jobs per wakeup and groups entries per subscription: one lock + one vectorized send_non_persistent_entries per subscription per batch, amortizing wakeup and lock costs; per-subscription entry order still follows enqueue order (FIFO) - publish_message routes through the same queue with a oneshot completion reply so in-process callers and tests stay deterministic; topics built without a runtime keep the inline fallback - new test: fire-and-forget enqueue preserves per-subscription order (no-subscription publishes drain; 300-job FIFO; barrier semantics) Measured together with TCP_NODELAY fix (8-sub non-persistent, 4 cores): fanout feed 701k -> 988k msg/s, aggregate delivery 841k -> 1.18M msg/s, p99 119.5ms -> 17.5ms. --- rust/src/broker/handler/producer_handler.rs | 12 +- rust/src/broker/service/topic/tests/topic.rs | 59 +++++++ rust/src/broker/service/topic/topic.rs | 168 ++++++++++++++++++- 3 files changed, 228 insertions(+), 11 deletions(-) diff --git a/rust/src/broker/handler/producer_handler.rs b/rust/src/broker/handler/producer_handler.rs index 1ae4df5..b8c3b68 100644 --- a/rust/src/broker/handler/producer_handler.rs +++ b/rust/src/broker/handler/producer_handler.rs @@ -130,13 +130,11 @@ pub async fn publish_send( let message_id = if is_non_persistent { producer.record_message_sent(frame.payload.len()).await; - let publish = { - let mut topic_guard = topic.write().await; - topic_guard.prepare_non_persistent_publish(frame.metadata, frame.payload)? - }; - let message_id = publish.message_id(); - publish.dispatch_sequential().await; - message_id + // Lock-free hot path (mirror of the persistent write-queue enqueue): + // short read lock for rate check + subscription snapshot, then hand + // off to the per-topic ordered fan-out worker. The connection task is + // free to read the next frame while fan-out runs. + Topic::publish_non_persistent(&topic, frame.metadata, frame.payload).await? } else { let message_id = producer .publish_message(frame.metadata, frame.payload) diff --git a/rust/src/broker/service/topic/tests/topic.rs b/rust/src/broker/service/topic/tests/topic.rs index cc1990e..9586674 100644 --- a/rust/src/broker/service/topic/tests/topic.rs +++ b/rust/src/broker/service/topic/tests/topic.rs @@ -383,6 +383,65 @@ async fn test_non_persistent_dispatches_entries_per_subscription_in_order() { assert_eq!(sub2_second.1.payload, b"second".to_vec()); } +#[tokio::test] +async fn non_persistent_fire_and_forget_enqueue_preserves_order() { + let storage = create_test_storage(); + let topic = Arc::new(RwLock::new(Topic::new( + "non-persistent://public/default/fanout-order".to_string(), + storage, + ))); + + // Publishes with no subscription enqueue and drain without blocking. + for i in 0..5 { + Topic::publish_non_persistent(&topic, None, Bytes::from(format!("nosub-{i}"))) + .await + .unwrap(); + } + + let subscription = topic + .write() + .await + .get_or_create_subscription("sub1", SubscriptionType::Exclusive) + .await + .unwrap(); + let (consumer, mut rx) = create_test_consumer_with_rx(1, subscription.clone()); + { + let mut sub_guard = subscription.write().await; + sub_guard.add_consumer(consumer).unwrap(); + } + { + let sub_guard = subscription.read().await; + sub_guard.get_consumer(1).unwrap().add_permits(400).await; + } + { + let sub_guard = subscription.read().await; + sub_guard.consumer_flow(1, 400).await; + } + + // Hot-path semantics: enqueue and return without awaiting fan-out. + for i in 0..300 { + Topic::publish_non_persistent(&topic, None, Bytes::from(format!("m{i}"))) + .await + .unwrap(); + } + // Barrier: publish_message queues behind all fire-and-forget jobs and + // awaits its own completion, so every earlier job has been dispatched. + topic + .write() + .await + .publish_message(None, Bytes::from_static(b"barrier")) + .await + .unwrap(); + + let mut received = Vec::with_capacity(301); + while let Ok((_, pending)) = rx.try_recv() { + received.push(pending.payload); + } + let mut expected: Vec> = (0..300).map(|i| format!("m{i}").into_bytes()).collect(); + expected.push(b"barrier".to_vec()); + assert_eq!(received, expected); +} + #[tokio::test] async fn test_non_persistent_topic_immediately_drops_for_blocked_subscription() { let storage = create_test_storage(); diff --git a/rust/src/broker/service/topic/topic.rs b/rust/src/broker/service/topic/topic.rs index a78f04a..2ac29f8 100644 --- a/rust/src/broker/service/topic/topic.rs +++ b/rust/src/broker/service/topic/topic.rs @@ -17,7 +17,7 @@ use std::collections::HashMap; use std::fmt; use std::sync::Arc; use std::time::{Duration, Instant}; -use tokio::sync::RwLock; +use tokio::sync::{mpsc, oneshot, RwLock}; /// Type alias for shared subscription pub type SharedSubscription = Arc>; @@ -53,6 +53,70 @@ impl NonPersistentPublish { } } +/// One queued job for the per-topic non-persistent fan-out worker. +struct NonPersistentFanoutJob { + publish: NonPersistentPublish, + /// When set, the worker replies after dispatching this job (used by + /// `publish_message` for deterministic in-process completion). + done: Option>, +} + +/// Bounded depth of the per-topic non-persistent fan-out queue. The worker +/// applies dispatch-or-drop semantics and never blocks on consumers, so this +/// bounds only the producer-side burst between enqueue and fan-out. +const NON_PERSISTENT_FANOUT_QUEUE_CAPACITY: usize = 4096; + +/// Jobs drained from the fan-out queue per worker pass. Batching amortizes +/// the worker wakeup and (per subscription) the lock acquisition plus the +/// vectorized dispatcher call across many messages. +const NON_PERSISTENT_FANOUT_BATCH_MAX: usize = 128; + +/// Dispatch a drained batch of fan-out jobs. +/// +/// Entries are grouped per subscription across the whole batch (jobs on one +/// topic normally carry identical subscription snapshots), so each +/// subscription pays one lock acquisition and one vectorized +/// `send_non_persistent_entries` call per batch instead of per message. +/// Entry order within a subscription follows job order (FIFO), preserving +/// per-subscription delivery order. `done` replies fire after the whole +/// batch is dispatched, which keeps `publish_message`'s barrier semantics +/// conservative and deterministic. +async fn dispatch_fanout_jobs(jobs: Vec) { + let mut groups: Vec<(SharedSubscription, Vec)> = Vec::new(); + let mut dones: Vec> = Vec::new(); + for job in jobs { + let NonPersistentFanoutJob { publish, done } = job; + if let Some(done) = done { + dones.push(done); + } + let entry = publish.entry; + for subscription in publish.subscriptions { + match groups + .iter_mut() + .find(|(s, _)| Arc::ptr_eq(s, &subscription)) + { + Some((_, entries)) => entries.push(entry.retained_duplicate()), + None => groups.push((subscription, vec![entry.retained_duplicate()])), + } + } + entry.release(); + } + for (subscription, entries) in groups { + let result = { + let sub_guard = subscription.read().await; + sub_guard.send_non_persistent_entries(entries).await + }; + if let Err(e) = result { + log::error!( + "Failed to dispatch non-persistent entries to subscription: {}", + e + ); + } + } + for done in dones { + let _ = done.send(()); + } +} #[derive(Debug, Clone, Copy, Default)] pub struct TopicPublishRate { pub messages_per_sec: u64, @@ -163,6 +227,11 @@ pub struct Topic { /// Storage backend storage: SharedStorage, publish_rate_limiter: TopicPublishRateLimiter, + /// Ordered fan-out queue for non-persistent publishes (lock-free hot path). + /// Mirrors the persistent write-queue shape: connection tasks enqueue under + /// a short read lock and return immediately; a single worker drains FIFO, + /// preserving per-subscription delivery order. + non_persistent_fanout_tx: Option>, } impl Topic { @@ -191,7 +260,7 @@ impl Topic { runtime_mode ); let persistent_runtime = PersistentTopicRuntime::new(storage.clone()); - Self { + let mut topic = Self { name, partition, producers: HashMap::new(), @@ -200,7 +269,12 @@ impl Topic { persistent_runtime, storage, publish_rate_limiter: TopicPublishRateLimiter::new(), + non_persistent_fanout_tx: None, + }; + if topic.runtime_mode == TopicRuntimeMode::NonPersistent { + topic.ensure_non_persistent_fanout_worker(); } + topic } pub fn runtime_mode(&self) -> TopicRuntimeMode { @@ -209,6 +283,9 @@ impl Topic { pub fn set_runtime_mode(&mut self, mode: TopicRuntimeMode) { self.runtime_mode = mode; + if mode == TopicRuntimeMode::NonPersistent { + self.ensure_non_persistent_fanout_worker(); + } } pub fn set_publish_rate(&mut self, limits: TopicPublishRate) { @@ -277,7 +354,7 @@ impl Topic { } pub fn prepare_non_persistent_publish( - &mut self, + &self, metadata: Option, payload: Bytes, ) -> Result> { @@ -301,6 +378,74 @@ impl Topic { Ok(self.build_non_persistent_publish(metadata, payload)) } + /// Spawn the per-topic ordered fan-out worker (idempotent). + /// + /// Topics constructed outside a tokio runtime (sync unit-test contexts) + /// skip this and keep the inline `dispatch_sequential` fallback. + fn ensure_non_persistent_fanout_worker(&mut self) { + if self.non_persistent_fanout_tx.is_some() { + return; + } + if tokio::runtime::Handle::try_current().is_err() { + return; + } + let (tx, mut rx) = + mpsc::channel::(NON_PERSISTENT_FANOUT_QUEUE_CAPACITY); + tokio::spawn(async move { + while let Some(first) = rx.recv().await { + let mut jobs = Vec::with_capacity(NON_PERSISTENT_FANOUT_BATCH_MAX); + jobs.push(first); + while jobs.len() < NON_PERSISTENT_FANOUT_BATCH_MAX { + match rx.try_recv() { + Ok(job) => jobs.push(job), + Err(_) => break, + } + } + dispatch_fanout_jobs(jobs).await; + } + }); + self.non_persistent_fanout_tx = Some(tx); + } + + /// Non-persistent publish hot path — the lock-free mirror of the persistent + /// write-queue enqueue: one short read lock for the rate check and the + /// subscription snapshot, then enqueue to the ordered fan-out worker and + /// return. The SendReceipt is an accept-ack; fan-out completes + /// asynchronously (same contract as the drop gate's fake receipts). + pub(crate) async fn publish_non_persistent( + topic: &Arc>, + metadata: Option, + payload: Bytes, + ) -> Result> { + let (publish, fanout_tx) = { + let topic_guard = topic.read().await; + let publish = topic_guard.prepare_non_persistent_publish(metadata, payload)?; + (publish, topic_guard.non_persistent_fanout_tx.clone()) + }; + let message_id = publish.message_id(); + Self::fan_out_non_persistent(publish, fanout_tx).await?; + Ok(message_id) + } + + async fn fan_out_non_persistent( + publish: NonPersistentPublish, + fanout_tx: Option>, + ) -> Result<(), Box> { + match fanout_tx { + Some(tx) => tx + .send(NonPersistentFanoutJob { + publish, + done: None, + }) + .await + .map_err(|e| -> Box { + format!("non-persistent fan-out worker gone: {e}").into() + })?, + None => publish.dispatch_sequential().await, + } + Ok(()) + } + /// Extract partition ID from topic name /// Returns -1 for non-partitioned topics fn extract_partition_from_name(name: &str) -> i32 { @@ -536,7 +681,22 @@ impl Topic { TopicRuntimeMode::NonPersistent => { let publish = self.build_non_persistent_publish(metadata, payload); let message_id = publish.message_id(); - publish.dispatch_sequential().await; + match self.non_persistent_fanout_tx.clone() { + Some(tx) => { + // Same ordered queue as the network path; wait for this + // job so in-process callers (and tests) observe delivery + // deterministically. The worker never locks the topic, + // so awaiting under the caller's topic write lock is safe. + let (done_tx, done_rx) = oneshot::channel(); + tx.send(NonPersistentFanoutJob { + publish, + done: Some(done_tx), + }) + .await?; + let _ = done_rx.await; + } + None => publish.dispatch_sequential().await, + } message_id } }; From 21238dbdf279db13247dc19449a876ba3c836c8c Mon Sep 17 00:00:00 2001 From: XTLine <2668543450@qq.com> Date: Tue, 18 Aug 2026 15:28:00 +0800 Subject: [PATCH 34/44] chore(broker): demote per-Flow log to debug Clients send thousands of Flow commands per second under load; an INFO line each flooded the journal/PTY (10.5MB per 80s stress run) and the synchronous logger write stalled runtime workers under backpressure, observed as second-scale producer stalls. Ack handling already logs at debug. --- rust/src/broker/handler/consumer_handler.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/rust/src/broker/handler/consumer_handler.rs b/rust/src/broker/handler/consumer_handler.rs index f2c534c..e8b9a37 100644 --- a/rust/src/broker/handler/consumer_handler.rs +++ b/rust/src/broker/handler/consumer_handler.rs @@ -195,7 +195,10 @@ pub async fn handle_flow( consumers: &mut HashMap>, ) -> Result<(), Box> { let flow_cmd = cmd.flow.as_ref().ok_or("Missing flow command")?; - log::info!( + // Per-Flow logging is debug-level: clients send thousands of Flow + // commands per second under load; INFO floods the journal/PTY and the + // synchronous logger write can stall runtime workers under backpressure. + log::debug!( "Handling Flow command: consumer_id={}, permits={}", flow_cmd.consumer_id, flow_cmd.message_permits From 3e84dc53e7eb66b63cdb9418795d996783d46773 Mon Sep 17 00:00:00 2001 From: XTLine <2668543450@qq.com> Date: Fri, 21 Aug 2026 22:09:32 +0800 Subject: [PATCH 35/44] perf(broker): batch persistent send receipts into single flushes complete_persistent_send flushed a SendReceipt per completion, so with TCP_NODELAY every persistent message produced one small TCP segment (~200k pkt/s at max produce rate). That regressed bulk persistent produce ~29% (10GiB fill 204k -> 145k msg/s) when TCP_NODELAY landed. Feed each receipt into the write buffer instead and flush once per drained batch: the connection loop now drains up to 128 completions from conn_append_rx per wakeup, completes them all, and issues a single flush (mirroring write_message_batch on the consumer side). The bounded-drain path in handle_send and the test helper flush accordingly. Recovered from the 08-19 perf session (checkpoint b4c0b517 on the perf server); measured there at 10GiB fill 145k -> 277.9k msg/s (+36% over the pre-NODELAY baseline) with TCP_NODELAY retained. Verified locally: tests/persist 29/29, cargo test 128, clippy 1.98 clean. --- rust/src/broker/service/server_cnx.rs | 59 +++++++++++++++++++++++---- 1 file changed, 51 insertions(+), 8 deletions(-) diff --git a/rust/src/broker/service/server_cnx.rs b/rust/src/broker/service/server_cnx.rs index 687665d..e8c6163 100644 --- a/rust/src/broker/service/server_cnx.rs +++ b/rust/src/broker/service/server_cnx.rs @@ -335,18 +335,49 @@ where } // Durable append completions from write-queue → SendReceipt / SendError. + // Batched: drain all pending completions, feed each receipt into the + // write buffer, then flush once so a batch of receipts leaves as a + // single large TCP write. With TCP_NODELAY enabled, flushing per + // receipt would emit one small segment per message (~200k pkt/s at + // max produce rate), which is what regressed bulk persistent produce. Some(append) = self.conn_append_rx.recv() => { - let outcome = PersistentSendOutcome { - producer_id: append.producer_id, - sequence_id: append.sequence_id, - result: append.result, - }; - if let Err(e) = self.complete_persistent_send(outcome).await { + let mut batch = vec![append]; + while let Ok(next) = self.conn_append_rx.try_recv() { + batch.push(next); + if batch.len() >= 128 { + break; + } + } + + let mut send_error: Option = None; + for append in batch { + let outcome = PersistentSendOutcome { + producer_id: append.producer_id, + sequence_id: append.sequence_id, + result: append.result, + }; + if let Err(e) = self.complete_persistent_send(outcome).await { + send_error = Some(e); + break; + } + } + // Single flush for the whole batch (complete_persistent_send only feeds). + if send_error.is_none() { + if let Err(e) = + futures::sink::SinkExt::::flush(&mut self.framed) + .await + .map_err(to_cnx_error) + { + send_error = Some(e); + } + } + self.sync_connection_writable_from_framed_buffer(); + + if let Some(e) = send_error { self.set_failed(CloseReason::ProtocolError(e.to_string())); log::error!("Error completing persistent send: {}", e); break Err(e); } - self.sync_connection_writable_from_framed_buffer(); } // Outbound broker messages are batched: drain all pending, encode @@ -789,8 +820,10 @@ where match outcome.result { Ok(message_id) => { + // Feed only: the caller batches completions and flushes once per + // batch (see the conn_append_rx arm in the connection loop). self.framed - .send(ServerCommand::SendReceipt { + .feed(ServerCommand::SendReceipt { producer_id: outcome.producer_id, sequence_id: outcome.sequence_id, ledger_id: message_id.ledger, @@ -912,6 +945,11 @@ where }; self.complete_persistent_send(outcome).await?; } + // complete_persistent_send only feeds the write buffer; flush the + // receipts drained above before enqueueing more work. + futures::sink::SinkExt::::flush(&mut self.framed) + .await + .map_err(to_cnx_error)?; let meta_len = frame.metadata.as_ref().map(|m| m.len()).unwrap_or(0); let payload_len = frame.payload.len(); @@ -1304,6 +1342,11 @@ mod tests { break; } } + // complete_persistent_send only feeds the write buffer; flush so + // test clients can observe the receipts. + futures::sink::SinkExt::::flush(&mut server_cnx.framed) + .await + .expect("flush persistent receipts"); } let _ = server_cnx; } From 7f621af4c1b97a4200be0bac7463fb65b98fd0f1 Mon Sep 17 00:00:00 2001 From: XTLine <2668543450@qq.com> Date: Wed, 19 Aug 2026 13:16:24 +0800 Subject: [PATCH 36/44] feat(metrics): add pulsar-lite-metrics crate with storage-side families Extract every Prometheus family into a dedicated workspace crate: - global registry + process collector served via prometheus-hyper - storage families: e2e/ledger write latency, entry size, write-queue batch metrics (observed from the single-writer worker thread) - PublishCommitObserver hook so the RocksDB write queue folds whole committed batches into one atomic update per topic - ledger-aware backlog_entries and stored_bytes on the storage contract with memory and RocksDB implementations plus reopen coverage --- rust/Cargo.lock | 375 ++++++++++- rust/Cargo.toml | 7 + rust/metrics/Cargo.toml | 9 + rust/metrics/src/broker.rs | 581 ++++++++++++++++++ rust/metrics/src/lib.rs | 92 +++ rust/metrics/src/observer.rs | 16 + rust/metrics/src/storage.rs | 228 +++++++ rust/metrics/src/subscription.rs | 214 +++++++ rust/metrics/src/topic.rs | 263 ++++++++ rust/storage/core/Cargo.toml | 1 + rust/storage/core/src/backend.rs | 16 + rust/storage/core/src/lib.rs | 3 +- rust/storage/core/src/service.rs | 10 + .../storage/managed-ledger-rocksdb/Cargo.toml | 1 + .../storage/managed-ledger-rocksdb/src/lib.rs | 1 + .../managed-ledger-rocksdb/src/store.rs | 44 +- .../managed-ledger-rocksdb/src/write_queue.rs | 115 ++-- .../tests/backlog_entries.rs | 63 ++ .../managed-ledger/src/legacy_storage.rs | 15 +- rust/storage/managed-ledger/src/memory.rs | 32 +- 20 files changed, 1983 insertions(+), 103 deletions(-) create mode 100644 rust/metrics/Cargo.toml create mode 100644 rust/metrics/src/broker.rs create mode 100644 rust/metrics/src/lib.rs create mode 100644 rust/metrics/src/observer.rs create mode 100644 rust/metrics/src/storage.rs create mode 100644 rust/metrics/src/subscription.rs create mode 100644 rust/metrics/src/topic.rs create mode 100644 rust/storage/managed-ledger-rocksdb/tests/backlog_entries.rs diff --git a/rust/Cargo.lock b/rust/Cargo.lock index 848eabe..838cedb 100644 --- a/rust/Cargo.lock +++ b/rust/Cargo.lock @@ -71,7 +71,7 @@ version = "1.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" dependencies = [ - "windows-sys", + "windows-sys 0.61.2", ] [[package]] @@ -82,7 +82,7 @@ checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" dependencies = [ "anstyle", "once_cell_polyfill", - "windows-sys", + "windows-sys 0.61.2", ] [[package]] @@ -108,9 +108,15 @@ checksum = "9035ad2d096bed7955a320ee7e2230574d28fd3c3a0f186cbea1ff3c7eed5dbb" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] +[[package]] +name = "atomic-waker" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" + [[package]] name = "bincode" version = "1.3.3" @@ -135,7 +141,7 @@ dependencies = [ "regex", "rustc-hash", "shlex", - "syn", + "syn 2.0.117", ] [[package]] @@ -235,7 +241,7 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -292,7 +298,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys", + "windows-sys 0.61.2", ] [[package]] @@ -313,6 +319,12 @@ version = "0.5.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1d674e81391d1e1ab681a28d99df07927c6d4aa5b027d7da16ba32d1d21ecd99" +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + [[package]] name = "foldhash" version = "0.1.5" @@ -375,7 +387,7 @@ checksum = "e835b70203e41293343137df5c0664546da5745f82ec9b84d40be8336958447b" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -459,6 +471,91 @@ version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" +[[package]] +name = "hex" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" + +[[package]] +name = "http" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "918d3568bebf352712bc2ef3d46a8bcf1a75b373be6539de198e9105cbbf9ce0" +dependencies = [ + "bytes", + "itoa", +] + +[[package]] +name = "http-body" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca2a8f2913ee65f60facd6a5905613afaa448497a0230cc41ce022d93290bc2c" +dependencies = [ + "bytes", + "http", +] + +[[package]] +name = "http-body-util" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23169fe34a5fbcdd3f3862e78fb9b6fccd5f02a6dc6f732547005d45631ce71c" +dependencies = [ + "bytes", + "futures-core", + "http", + "http-body", + "pin-project-lite", +] + +[[package]] +name = "httparse" +version = "1.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87" + +[[package]] +name = "httpdate" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" + +[[package]] +name = "hyper" +version = "1.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d22053281f852e11534f5198498373cbb59295120a20771d90f7ed1897490a72" +dependencies = [ + "atomic-waker", + "bytes", + "futures-channel", + "futures-core", + "http", + "http-body", + "httparse", + "httpdate", + "itoa", + "pin-project-lite", + "smallvec", + "tokio", +] + +[[package]] +name = "hyper-util" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96547c2556ec9d12fb1578c4eaf448b04993e7fb79cbaad930a656880a6bdfa0" +dependencies = [ + "bytes", + "http", + "http-body", + "hyper", + "pin-project-lite", + "tokio", +] + [[package]] name = "id-arena" version = "2.3.0" @@ -528,7 +625,7 @@ checksum = "2a8c8b344124222efd714b73bb41f8b5120b27a7cc1c75593a6ff768d9d05aa4" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -551,6 +648,12 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "lazy_static" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" + [[package]] name = "leb128fmt" version = "0.1.0" @@ -599,6 +702,12 @@ dependencies = [ "vcpkg", ] +[[package]] +name = "linux-raw-sys" +version = "0.4.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d26c52dbd32dccf2d10cac7725f8eae5296885fb5703b261f7d0a0739ec807ab" + [[package]] name = "linux-raw-sys" version = "0.12.1" @@ -650,7 +759,7 @@ checksum = "02bd0af71c67b473010cbbc60715ee815645a4dc942899111f494b4b737d6fda" dependencies = [ "libc", "wasi", - "windows-sys", + "windows-sys 0.61.2", ] [[package]] @@ -748,7 +857,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b" dependencies = [ "proc-macro2", - "syn", + "syn 2.0.117", ] [[package]] @@ -760,6 +869,59 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "procfs" +version = "0.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc5b72d8145275d844d4b5f6d4e1eef00c8cd889edb6035c21675d1bb1f45c9f" +dependencies = [ + "bitflags", + "hex", + "procfs-core", + "rustix 0.38.44", +] + +[[package]] +name = "procfs-core" +version = "0.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "239df02d8349b06fc07398a3a1697b06418223b1c7725085e801e7c0fc6a12ec" +dependencies = [ + "bitflags", + "hex", +] + +[[package]] +name = "prometheus" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ca5326d8d0b950a9acd87e6a3f94745394f62e4dae1b1ee22b2bc0c394af43a" +dependencies = [ + "cfg-if", + "fnv", + "lazy_static", + "libc", + "memchr", + "parking_lot", + "procfs", + "thiserror 2.0.20", +] + +[[package]] +name = "prometheus-hyper" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "60d6eeac44b972d6f552e8aaec7f869200aef42f14ddffdddde308b94d0c066e" +dependencies = [ + "bytes", + "http-body-util", + "hyper", + "hyper-util", + "prometheus", + "tokio", + "tracing", +] + [[package]] name = "prost" version = "0.13.5" @@ -786,7 +948,7 @@ dependencies = [ "prost", "prost-types", "regex", - "syn", + "syn 2.0.117", "tempfile", ] @@ -800,7 +962,7 @@ dependencies = [ "itertools 0.14.0", "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -824,9 +986,12 @@ dependencies = [ "env_logger", "futures", "log", + "prometheus", + "prometheus-hyper", "prost", "prost-build", "prost-types", + "pulsar-lite-metrics", "pulsar-lite-proto", "pulsar-lite-storage", "pulsar-lite-storage-managed-ledger", @@ -836,7 +1001,7 @@ dependencies = [ "serde", "serde_json", "tempfile", - "thiserror", + "thiserror 1.0.69", "tikv-jemalloc-sys", "tikv-jemallocator", "tokio", @@ -846,6 +1011,14 @@ dependencies = [ "uuid", ] +[[package]] +name = "pulsar-lite-metrics" +version = "0.1.0" +dependencies = [ + "log", + "prometheus", +] + [[package]] name = "pulsar-lite-proto" version = "0.1.0" @@ -863,6 +1036,7 @@ version = "0.1.0" dependencies = [ "anyhow", "log", + "prometheus", "prost", "pulsar-lite-proto", "pulsar-lite-storage-managed-ledger", @@ -895,6 +1069,7 @@ dependencies = [ "log", "prost", "prost-build", + "pulsar-lite-metrics", "pulsar-lite-storage-managed-ledger", "rocksdb", "serde", @@ -1000,6 +1175,19 @@ version = "2.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "94300abf3f1ae2e2b8ffb7b58043de3d399c73fa6f4b73826402a5c457614dbe" +[[package]] +name = "rustix" +version = "0.38.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fdb5bc1ae2baa591800df16c9ca78619bf65c0488b41b96ccec5d11220d8c154" +dependencies = [ + "bitflags", + "errno", + "libc", + "linux-raw-sys 0.4.15", + "windows-sys 0.59.0", +] + [[package]] name = "rustix" version = "1.1.4" @@ -1009,8 +1197,8 @@ dependencies = [ "bitflags", "errno", "libc", - "linux-raw-sys", - "windows-sys", + "linux-raw-sys 0.12.1", + "windows-sys 0.61.2", ] [[package]] @@ -1058,7 +1246,7 @@ checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -1118,7 +1306,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3a766e1110788c36f4fa1c2b71b387a7815aa65f88ce0229841826633d93723e" dependencies = [ "libc", - "windows-sys", + "windows-sys 0.61.2", ] [[package]] @@ -1138,6 +1326,17 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "syn" +version = "3.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53e9bae58849f64dfa4f5d5ae372c8341f7305f82a3868709269343628b659a3" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + [[package]] name = "tempfile" version = "3.26.0" @@ -1147,8 +1346,8 @@ dependencies = [ "fastrand", "getrandom 0.4.2", "once_cell", - "rustix", - "windows-sys", + "rustix 1.1.4", + "windows-sys 0.61.2", ] [[package]] @@ -1157,7 +1356,16 @@ version = "1.0.69" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6aaf5339b578ea85b50e080feb250a3e8ae8cfcdff9a461c9ec2904bc923f52" dependencies = [ - "thiserror-impl", + "thiserror-impl 1.0.69", +] + +[[package]] +name = "thiserror" +version = "2.0.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec86235f5fcc2a73650310756d2ac5b138a5780bbbdfae3eeccec992c435ba4f" +dependencies = [ + "thiserror-impl 2.0.20", ] [[package]] @@ -1168,7 +1376,18 @@ checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", +] + +[[package]] +name = "thiserror-impl" +version = "2.0.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc04cd3e1236dd4a98afca4569f2deb3f120e5422a4023be2cb683f8486292af" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.3", ] [[package]] @@ -1205,7 +1424,7 @@ dependencies = [ "signal-hook-registry", "socket2", "tokio-macros", - "windows-sys", + "windows-sys 0.61.2", ] [[package]] @@ -1216,7 +1435,7 @@ checksum = "385a6cb71ab9ab790c5fe8d67f1645e6c450a7ce006a33de03daa956cf70a496" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -1284,6 +1503,37 @@ version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5d99f8c9a7727884afe522e9bd5edbfc91a3312b36a77b5fb8926e4c31a41801" +[[package]] +name = "tracing" +version = "0.1.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63e71662fa4b2a2c3a26f570f037eb95bb1f85397f3cd8076caed2f026a6d100" +dependencies = [ + "pin-project-lite", + "tracing-attributes", + "tracing-core", +] + +[[package]] +name = "tracing-attributes" +version = "0.1.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "tracing-core" +version = "0.1.36" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db97caf9d906fbde555dd62fa95ddba9eecfd14cb388e4f491a66d74cd5fb79a" +dependencies = [ + "once_cell", +] + [[package]] name = "unicode-ident" version = "1.0.24" @@ -1375,7 +1625,7 @@ dependencies = [ "bumpalo", "proc-macro2", "quote", - "syn", + "syn 2.0.117", "wasm-bindgen-shared", ] @@ -1428,6 +1678,15 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" +[[package]] +name = "windows-sys" +version = "0.59.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b" +dependencies = [ + "windows-targets", +] + [[package]] name = "windows-sys" version = "0.61.2" @@ -1437,6 +1696,70 @@ dependencies = [ "windows-link", ] +[[package]] +name = "windows-targets" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" +dependencies = [ + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_gnullvm", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" + +[[package]] +name = "windows_i686_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" + +[[package]] +name = "windows_i686_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" + [[package]] name = "winnow" version = "0.7.15" @@ -1476,7 +1799,7 @@ dependencies = [ "heck", "indexmap", "prettyplease", - "syn", + "syn 2.0.117", "wasm-metadata", "wit-bindgen-core", "wit-component", @@ -1492,7 +1815,7 @@ dependencies = [ "prettyplease", "proc-macro2", "quote", - "syn", + "syn 2.0.117", "wit-bindgen-core", "wit-bindgen-rust", ] diff --git a/rust/Cargo.toml b/rust/Cargo.toml index 379b921..06d8f6a 100644 --- a/rust/Cargo.toml +++ b/rust/Cargo.toml @@ -1,6 +1,7 @@ [workspace] members = [ ".", + "metrics", "storage/core", "storage/metadata", "storage/resources", @@ -24,6 +25,8 @@ anyhow = "1.0" log = "0.4" tikv-jemallocator = { version = "0.6", features = ["unprefixed_malloc_on_supported_platforms"] } tikv-jemalloc-sys = { version = "0.6", features = ["profiling", "unprefixed_malloc_on_supported_platforms"] } +prometheus = { version = "0.14", default-features = false, features = ["process"] } +prometheus-hyper = "0.2" [package] name = "pulsar-lite" @@ -85,6 +88,10 @@ uuid = { version = "1.6", features = ["v4"] } # 流处理 tokio-stream = "0.1" futures = "0.3" +# Metrics: Prometheus families and HTTP export (GET /metrics) +prometheus.workspace = true +prometheus-hyper.workspace = true +pulsar-lite-metrics = { path = "metrics" } # 内存分配器：jemalloc（缓解 glibc arena 内存滞留导致的 RSS 高水位） tikv-jemallocator.workspace = true diff --git a/rust/metrics/Cargo.toml b/rust/metrics/Cargo.toml new file mode 100644 index 0000000..beaac8f --- /dev/null +++ b/rust/metrics/Cargo.toml @@ -0,0 +1,9 @@ +[package] +name = "pulsar-lite-metrics" +version = "0.1.0" +edition.workspace = true +license.workspace = true + +[dependencies] +prometheus.workspace = true +log.workspace = true diff --git a/rust/metrics/src/broker.rs b/rust/metrics/src/broker.rs new file mode 100644 index 0000000..f3f29d5 --- /dev/null +++ b/rust/metrics/src/broker.rs @@ -0,0 +1,581 @@ +/* + * Broker Prometheus Metric Families + * + * Families are created once at startup and registered into the shared + * registry from `pulsar_lite_storage::metrics`. Hot paths only ever touch + * pre-resolved handles (plain atomic adds); label resolution happens once + * at entity creation, never per message. + * + * Naming contract: families that reproduce native Pulsar semantics keep + * the exact `pulsar_*` names and label sets; extensions with no native + * counterpart use the `pulsar_lite_*` prefix. + */ + +use std::sync::{Arc, OnceLock}; + +use prometheus::{Gauge, GaugeVec, IntCounter, IntCounterVec, IntGauge, IntGaugeVec, Opts}; +use crate::global_registry; + +/// Topic-labeled families shared by every `TopicMetrics` handle. +#[derive(Debug, Clone)] +pub struct TopicFamilies { + /// pulsar_in_messages_total{cluster, namespace, topic, partition} + pub in_messages: IntCounterVec, + /// pulsar_in_bytes_total{...} + pub in_bytes: IntCounterVec, + /// pulsar_publish_rate_limit_times{...} + pub publish_rate_limit: IntCounterVec, + /// pulsar_rate_in{...} (scrape-derived gauge) + pub rate_in: GaugeVec, + /// pulsar_throughput_in{...} + pub throughput_in: GaugeVec, + /// pulsar_average_msg_size{...} + pub average_msg_size: IntGaugeVec, + /// pulsar_storage_size{...} + pub storage_size: IntGaugeVec, + /// pulsar_subscriptions_count{...} + pub subscriptions_count: IntGaugeVec, + /// pulsar_producers_count{...} + pub producers_count: IntGaugeVec, + /// pulsar_consumers_count{...} + pub consumers_count: IntGaugeVec, +} + +/// Subscription-labeled families shared by every `SubscriptionMetrics`. +#[derive(Debug, Clone)] +pub struct SubscriptionFamilies { + /// pulsar_out_messages_total{cluster, namespace, topic, partition, subscription} + pub out_messages: IntCounterVec, + /// pulsar_out_bytes_total{...} + pub out_bytes: IntCounterVec, + /// pulsar_lite_subscription_redelivered_total{...} + pub redelivered: IntCounterVec, + /// pulsar_lite_subscription_dropped_messages_total{...} + pub dropped: IntCounterVec, + /// pulsar_lite_subscription_acked_messages_total{...} + pub acked: IntCounterVec, + /// pulsar_subscription_msg_rate_out{...} + pub msg_rate_out: GaugeVec, + /// pulsar_subscription_msg_throughput_out{...} + pub msg_throughput_out: GaugeVec, + /// pulsar_subscription_msg_ack_rate{...} + pub msg_ack_rate: GaugeVec, + /// pulsar_subscription_msg_rate_redeliver{...} + pub msg_rate_redeliver: GaugeVec, + /// pulsar_subscription_msg_drop_rate{...} + pub msg_drop_rate: GaugeVec, + /// pulsar_subscription_back_log{...} + pub back_log: IntGaugeVec, + /// pulsar_subscription_unacked_messages{...} + pub unacked_messages: IntGaugeVec, + /// pulsar_subscription_blocked_on_unacked_messages{...} + pub blocked_on_unacked: IntGaugeVec, + /// pulsar_subscription_consumers_count{...} + pub consumers_count: IntGaugeVec, + /// pulsar_subscription_last_acked_timestamp{...} (unix seconds) + pub last_acked_timestamp: GaugeVec, + /// pulsar_subscription_last_consumed_timestamp{...} (unix seconds) + pub last_consumed_timestamp: GaugeVec, +} + +impl SubscriptionFamilies { + fn new(registry: &prometheus::Registry) -> Result { + let out_messages = IntCounterVec::new( + Opts::new( + "pulsar_out_messages_total", + "Total messages delivered to consumers", + ), + SUBSCRIPTION_LABELS, + )?; + let out_bytes = IntCounterVec::new( + Opts::new( + "pulsar_out_bytes_total", + "Total bytes delivered to consumers", + ), + SUBSCRIPTION_LABELS, + )?; + let redelivered = IntCounterVec::new( + Opts::new( + "pulsar_lite_subscription_redelivered_total", + "Messages queued for redelivery", + ), + SUBSCRIPTION_LABELS, + )?; + let dropped = IntCounterVec::new( + Opts::new( + "pulsar_lite_subscription_dropped_messages_total", + "Non-persistent messages dropped with no writable consumer", + ), + SUBSCRIPTION_LABELS, + )?; + let acked = IntCounterVec::new( + Opts::new( + "pulsar_lite_subscription_acked_messages_total", + "Messages acknowledged by consumers", + ), + SUBSCRIPTION_LABELS, + )?; + let msg_rate_out = GaugeVec::new( + Opts::new( + "pulsar_subscription_msg_rate_out", + "Delivered message rate (window average)", + ), + SUBSCRIPTION_LABELS, + )?; + let msg_throughput_out = GaugeVec::new( + Opts::new( + "pulsar_subscription_msg_throughput_out", + "Delivered byte rate (window average)", + ), + SUBSCRIPTION_LABELS, + )?; + let msg_ack_rate = GaugeVec::new( + Opts::new( + "pulsar_subscription_msg_ack_rate", + "Acknowledge rate (window average)", + ), + SUBSCRIPTION_LABELS, + )?; + let msg_rate_redeliver = GaugeVec::new( + Opts::new( + "pulsar_subscription_msg_rate_redeliver", + "Redelivery rate (window average)", + ), + SUBSCRIPTION_LABELS, + )?; + let msg_drop_rate = GaugeVec::new( + Opts::new( + "pulsar_subscription_msg_drop_rate", + "Non-persistent drop rate (window average)", + ), + SUBSCRIPTION_LABELS, + )?; + let back_log = IntGaugeVec::new( + Opts::new( + "pulsar_subscription_back_log", + "Unacknowledged stored entries", + ), + SUBSCRIPTION_LABELS, + )?; + let unacked_messages = IntGaugeVec::new( + Opts::new( + "pulsar_subscription_unacked_messages", + "Dispatched-but-unacknowledged messages", + ), + SUBSCRIPTION_LABELS, + )?; + let blocked_on_unacked = IntGaugeVec::new( + Opts::new( + "pulsar_subscription_blocked_on_unacked_messages", + "1 while dispatch is blocked by the unacked gate", + ), + SUBSCRIPTION_LABELS, + )?; + let consumers_count = IntGaugeVec::new( + Opts::new("pulsar_subscription_consumers_count", "Connected consumers"), + SUBSCRIPTION_LABELS, + )?; + let last_acked_timestamp = GaugeVec::new( + Opts::new( + "pulsar_subscription_last_acked_timestamp", + "Last acknowledge time (unix seconds)", + ), + SUBSCRIPTION_LABELS, + )?; + let last_consumed_timestamp = GaugeVec::new( + Opts::new( + "pulsar_subscription_last_consumed_timestamp", + "Last dispatch time (unix seconds)", + ), + SUBSCRIPTION_LABELS, + )?; + + registry.register(Box::new(out_messages.clone()))?; + registry.register(Box::new(out_bytes.clone()))?; + registry.register(Box::new(redelivered.clone()))?; + registry.register(Box::new(dropped.clone()))?; + registry.register(Box::new(acked.clone()))?; + registry.register(Box::new(msg_rate_out.clone()))?; + registry.register(Box::new(msg_throughput_out.clone()))?; + registry.register(Box::new(msg_ack_rate.clone()))?; + registry.register(Box::new(msg_rate_redeliver.clone()))?; + registry.register(Box::new(msg_drop_rate.clone()))?; + registry.register(Box::new(back_log.clone()))?; + registry.register(Box::new(unacked_messages.clone()))?; + registry.register(Box::new(blocked_on_unacked.clone()))?; + registry.register(Box::new(consumers_count.clone()))?; + registry.register(Box::new(last_acked_timestamp.clone()))?; + registry.register(Box::new(last_consumed_timestamp.clone()))?; + + Ok(Self { + out_messages, + out_bytes, + redelivered, + dropped, + acked, + msg_rate_out, + msg_throughput_out, + msg_ack_rate, + msg_rate_redeliver, + msg_drop_rate, + back_log, + unacked_messages, + blocked_on_unacked, + consumers_count, + last_acked_timestamp, + last_consumed_timestamp, + }) + } +} + +const SUBSCRIPTION_LABELS: &[&str] = + &["cluster", "namespace", "topic", "partition", "subscription"]; +const TOPIC_LABELS: &[&str] = &["cluster", "namespace", "topic", "partition"]; + +impl TopicFamilies { + fn new(registry: &prometheus::Registry) -> Result { + let in_messages = IntCounterVec::new( + Opts::new( + "pulsar_in_messages_total", + "Total messages accepted for publish", + ), + TOPIC_LABELS, + )?; + let in_bytes = IntCounterVec::new( + Opts::new("pulsar_in_bytes_total", "Total bytes accepted for publish"), + TOPIC_LABELS, + )?; + let publish_rate_limit = IntCounterVec::new( + Opts::new( + "pulsar_publish_rate_limit_times", + "Publishes rejected by the topic rate limiter", + ), + TOPIC_LABELS, + )?; + let rate_in = GaugeVec::new( + Opts::new("pulsar_rate_in", "Accepted message rate (window average)"), + TOPIC_LABELS, + )?; + let throughput_in = GaugeVec::new( + Opts::new( + "pulsar_throughput_in", + "Accepted byte rate (window average)", + ), + TOPIC_LABELS, + )?; + let average_msg_size = IntGaugeVec::new( + Opts::new( + "pulsar_average_msg_size", + "Average accepted message size in bytes", + ), + TOPIC_LABELS, + )?; + let storage_size = IntGaugeVec::new( + Opts::new("pulsar_storage_size", "Bytes stored for the topic"), + TOPIC_LABELS, + )?; + let subscriptions_count = IntGaugeVec::new( + Opts::new( + "pulsar_subscriptions_count", + "Active subscriptions on the topic", + ), + TOPIC_LABELS, + )?; + let producers_count = IntGaugeVec::new( + Opts::new("pulsar_producers_count", "Connected producers on the topic"), + TOPIC_LABELS, + )?; + let consumers_count = IntGaugeVec::new( + Opts::new("pulsar_consumers_count", "Connected consumers on the topic"), + TOPIC_LABELS, + )?; + + registry.register(Box::new(in_messages.clone()))?; + registry.register(Box::new(in_bytes.clone()))?; + registry.register(Box::new(publish_rate_limit.clone()))?; + registry.register(Box::new(rate_in.clone()))?; + registry.register(Box::new(throughput_in.clone()))?; + registry.register(Box::new(average_msg_size.clone()))?; + registry.register(Box::new(storage_size.clone()))?; + registry.register(Box::new(subscriptions_count.clone()))?; + registry.register(Box::new(producers_count.clone()))?; + registry.register(Box::new(consumers_count.clone()))?; + + Ok(Self { + in_messages, + in_bytes, + publish_rate_limit, + rate_in, + throughput_in, + average_msg_size, + storage_size, + subscriptions_count, + producers_count, + consumers_count, + }) + } +} + +/// Broker-scoped metric families. +#[derive(Debug)] +pub struct BrokerMetrics { + /// `cluster` label value applied to every family (config-injected). + cluster: String, + + /// Topic-labeled families (per-`TopicMetrics` handles resolve from these). + pub topics: TopicFamilies, + /// Subscription-labeled families (per-`SubscriptionMetrics` handles). + pub subscriptions: SubscriptionFamilies, + + /// pulsar_active_connections{cluster} + pub active_connections: IntGauge, + /// pulsar_connection_created_total_count{cluster} + pub connection_created: IntCounter, + /// pulsar_connection_closed_total_count{cluster} + pub connection_closed: IntCounter, + /// pulsar_lite_broker_errors_total{cluster, reason} + pub errors: IntCounterVec, + /// pulsar_broker_in_messages_total{cluster} + pub broker_in_messages: IntCounter, + /// pulsar_broker_in_bytes_total{cluster} + pub broker_in_bytes: IntCounter, + /// pulsar_broker_out_messages_total{cluster} + pub broker_out_messages: IntCounter, + /// pulsar_broker_out_bytes_total{cluster} + pub broker_out_bytes: IntCounter, + /// pulsar_broker_topics_count{cluster} (scrape-set) + pub broker_topics_count: IntGauge, + /// pulsar_broker_subscriptions_count{cluster} (scrape-set) + pub broker_subscriptions_count: IntGauge, + /// pulsar_broker_producers_count{cluster} (scrape-set) + pub broker_producers_count: IntGauge, + /// pulsar_broker_consumers_count{cluster} (scrape-set) + pub broker_consumers_count: IntGauge, + /// pulsar_broker_rate_in{cluster} (scrape-derived) + pub broker_rate_in: Gauge, + /// pulsar_broker_throughput_in{cluster} + pub broker_throughput_in: Gauge, + /// pulsar_broker_rate_out{cluster} + pub broker_rate_out: Gauge, + /// pulsar_broker_throughput_out{cluster} + pub broker_throughput_out: Gauge, + /// pulsar_broker_msg_backlog{cluster} + pub broker_msg_backlog: IntGauge, + /// pulsar_broker_storage_size{cluster} + pub broker_storage_size: IntGauge, +} + +impl BrokerMetrics { + fn new(cluster: &str, registry: &prometheus::Registry) -> Result { + let topics = TopicFamilies::new(registry)?; + let subscriptions = SubscriptionFamilies::new(registry)?; + + let active_connections = IntGauge::new( + "pulsar_active_connections", + "Currently open client connections", + )?; + let connection_created = IntCounter::new( + "pulsar_connection_created_total_count", + "Total accepted connections", + )?; + let connection_closed = IntCounter::new( + "pulsar_connection_closed_total_count", + "Total closed connections", + )?; + let errors = IntCounterVec::new( + Opts::new( + "pulsar_lite_broker_errors_total", + "Rejected or failed operations by reason", + ), + &["cluster", "reason"], + )?; + let version_info = IntGaugeVec::new( + Opts::new("pulsar_version_info", "Broker version constant"), + &["cluster", "version"], + )?; + let broker_in_messages = IntCounter::new( + "pulsar_broker_in_messages_total", + "Total messages accepted for publish", + )?; + let broker_in_bytes = IntCounter::new( + "pulsar_broker_in_bytes_total", + "Total bytes accepted for publish", + )?; + let broker_out_messages = IntCounter::new( + "pulsar_broker_out_messages_total", + "Total messages delivered to consumers", + )?; + let broker_out_bytes = IntCounter::new( + "pulsar_broker_out_bytes_total", + "Total bytes delivered to consumers", + )?; + let broker_topics_count = + IntGauge::new("pulsar_broker_topics_count", "Topics hosted by the broker")?; + let broker_subscriptions_count = IntGauge::new( + "pulsar_broker_subscriptions_count", + "Subscriptions hosted by the broker", + )?; + let broker_producers_count = IntGauge::new( + "pulsar_broker_producers_count", + "Connected producers on the broker", + )?; + let broker_consumers_count = IntGauge::new( + "pulsar_broker_consumers_count", + "Connected consumers on the broker", + )?; + let broker_rate_in = Gauge::new( + "pulsar_broker_rate_in", + "Accepted message rate across all topics (window average)", + )?; + let broker_throughput_in = Gauge::new( + "pulsar_broker_throughput_in", + "Accepted byte rate across all topics (window average)", + )?; + let broker_rate_out = Gauge::new( + "pulsar_broker_rate_out", + "Delivered message rate across all topics (window average)", + )?; + let broker_throughput_out = Gauge::new( + "pulsar_broker_throughput_out", + "Delivered byte rate across all topics (window average)", + )?; + let broker_msg_backlog = + IntGauge::new("pulsar_broker_msg_backlog", "Total backlog entries")?; + let broker_storage_size = + IntGauge::new("pulsar_broker_storage_size", "Total stored bytes")?; + + registry.register(Box::new(active_connections.clone()))?; + registry.register(Box::new(connection_created.clone()))?; + registry.register(Box::new(connection_closed.clone()))?; + registry.register(Box::new(errors.clone()))?; + registry.register(Box::new(version_info.clone()))?; + registry.register(Box::new(broker_in_messages.clone()))?; + registry.register(Box::new(broker_in_bytes.clone()))?; + registry.register(Box::new(broker_out_messages.clone()))?; + registry.register(Box::new(broker_out_bytes.clone()))?; + registry.register(Box::new(broker_topics_count.clone()))?; + registry.register(Box::new(broker_subscriptions_count.clone()))?; + registry.register(Box::new(broker_producers_count.clone()))?; + registry.register(Box::new(broker_consumers_count.clone()))?; + registry.register(Box::new(broker_rate_in.clone()))?; + registry.register(Box::new(broker_throughput_in.clone()))?; + registry.register(Box::new(broker_rate_out.clone()))?; + registry.register(Box::new(broker_throughput_out.clone()))?; + registry.register(Box::new(broker_msg_backlog.clone()))?; + registry.register(Box::new(broker_storage_size.clone()))?; + // The registry keeps the gauge alive; we only pin the value once. + version_info + .with_label_values(&[cluster, env!("CARGO_PKG_VERSION")]) + .set(1); + Ok(Self { + cluster: cluster.to_string(), + topics, + subscriptions, + active_connections, + connection_created, + connection_closed, + errors, + broker_in_messages, + broker_in_bytes, + broker_out_messages, + broker_out_bytes, + broker_topics_count, + broker_subscriptions_count, + broker_producers_count, + broker_consumers_count, + broker_rate_in, + broker_throughput_in, + broker_rate_out, + broker_throughput_out, + broker_msg_backlog, + broker_storage_size, + }) + } + + /// The configured `cluster` label value. + pub fn cluster(&self) -> &str { + &self.cluster + } + + /// Pre-resolves an error counter for `reason` (one label lookup, then + /// stored by the caller; `inc()` is a single atomic add). + pub fn error_counter(&self, reason: &str) -> IntCounter { + self.errors + .with_label_values(&[self.cluster.as_str(), reason]) + } + + /// Fallback constructor used only when registration failed: identical + /// handles, not attached to the served registry. + fn unregistered(cluster: &str) -> Self { + // Static family names are proven valid by tests; registration into a + // fresh registry cannot fail for any other reason. + Self::new(cluster, &prometheus::Registry::new()) + .expect("static broker metric definitions are valid") + } +} + +static BROKER_METRICS: OnceLock> = OnceLock::new(); + +/// Initializes broker metric families into the shared global registry. +/// +/// Idempotent: the first call wins, so unit tests and production startup +/// share one code path. Construction only fails on invalid static names +/// (a programming error caught by tests); in that case we log once and +/// fall back to unregistered families so counters stay callable. +/// Initializes broker metric families into the shared global registry. +/// +/// Idempotent: the first call wins, so unit tests and production startup +/// share one code path. +pub fn init(cluster: &str) -> Arc { + BROKER_METRICS + .get_or_init(|| match BrokerMetrics::new(cluster, &global_registry()) { + Ok(metrics) => Arc::new(metrics), + Err(error) => { + log::error!("Failed to register broker metrics: {}", error); + Arc::new(BrokerMetrics::unregistered(cluster)) + } + }) + .clone() +} + +/// Returns the initialized metrics, auto-initializing with the default +/// cluster label for callers (tests, library embedders) that skipped +/// explicit startup wiring. +pub fn get() -> Arc { + init("pulsar-lite") +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn init_is_idempotent_and_counters_tick() { + let metrics = init("test-cluster"); + assert!(Arc::ptr_eq(&metrics, &init("other-cluster"))); + + let before = metrics.connection_created.get(); + metrics.connection_created.inc(); + // Unique label: the served registry is process-global and other + // tests (oversized-message handling) also bump error counters. + let rejected = metrics.error_counter("registry_unit_test"); + rejected.inc(); + assert!(metrics.connection_created.get() > before); + assert_eq!(rejected.get(), 1); + // Cluster label itself is racy under parallel tests (first init + // wins process-wide); ptr_eq above already proves idempotency. + } + + #[test] + fn topic_families_resolve_same_cell_for_same_labels() { + let families = &get().topics; + let a = families + .in_messages + .with_label_values(&["c", "public/default", "t", "-1"]); + let b = families + .in_messages + .with_label_values(&["c", "public/default", "t", "-1"]); + a.inc(); + assert_eq!(a.get(), b.get()); + } +} diff --git a/rust/metrics/src/lib.rs b/rust/metrics/src/lib.rs new file mode 100644 index 0000000..6adb1c5 --- /dev/null +++ b/rust/metrics/src/lib.rs @@ -0,0 +1,92 @@ +//! Pulsar Lite metrics crate. +//! +//! One home for every Prometheus family this broker exports: +//! +//! - [`global_registry`] — the process-wide registry served on +//! `GET /metrics` (via `prometheus-hyper` in the broker binary); +//! - [`storage`] — families observed from the managed-ledger write path +//! (`pulsar_storage_write_latency`, `pulsar_entry_size`, +//! write-queue batch metrics); +//! - [`broker`] — broker-scoped families and the topic/subscription +//! label families (`pulsar_broker_*`, `pulsar_in/out_*`, +//! `pulsar_subscription_*`); +//! - [`topic`] / [`subscription`] — pre-resolved per-entity handles whose +//! label lookup happens exactly once at entity creation; +//! - [`observer`] — the `PublishCommitObserver` hook the RocksDB +//! write-queue worker invokes per committed batch. +//! +//! Naming contract: families reproducing native Pulsar semantics keep the +//! exact `pulsar_*` names and label sets; extensions use `pulsar_lite_*`. +//! +//! Hot paths only ever touch pre-resolved handles (plain atomic adds). +//! [`init`] registers everything once; before it runs, accessors return +//! no-op handles so unit tests never panic. + +use std::sync::{Arc, LazyLock}; + +use prometheus::Registry; + +pub mod broker; +pub mod observer; +pub mod storage; +pub mod subscription; +pub mod topic; + +pub use broker::{get, BrokerMetrics}; +pub use observer::PublishCommitObserver; +pub use storage::{storage_metrics, StorageMetrics, StorageMetricsHandle}; +pub use subscription::SubscriptionMetrics; +pub use topic::{parse_topic_labels, TopicLabels, TopicMetrics}; + +static GLOBAL: LazyLock> = LazyLock::new(|| { + let registry = Registry::new(); + // The process collector (RSS / CPU gauges) is registered best-effort: a + // missing /proc mount only drops those gauges, never breaks collection. + match registry.register(Box::new( + prometheus::process_collector::ProcessCollector::for_self(), + )) { + Ok(()) => {} + Err(error) => log::warn!("Failed to register process collector: {}", error), + } + Arc::new(registry) +}); + +/// Returns the process-wide metrics registry, creating it on first use. +pub fn global_registry() -> Arc { + GLOBAL.clone() +} + +/// Registers broker families (idempotent) and storage families. +/// +/// `broker::init` is separately idempotent so it can also serve as the +/// lazy entry point for embedders that never call this function. +pub fn init(cluster: &str) -> Arc { + let metrics = broker::init(cluster); + let _ = storage::init(cluster); + metrics +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn global_registry_is_stable_across_calls() { + assert!(Arc::ptr_eq(&global_registry(), &global_registry())); + } + + #[test] + fn init_registers_all_families_once() { + let metrics = init("crate-test-cluster"); + // Unique labels: the registry is process-global and sibling tests + // also bump shared counters. + let cell = metrics + .topics + .in_messages + .with_label_values(&["c", "ns", "lib-init-check", "-1"]); + cell.inc(); + let storage = storage::storage_metrics(); + storage.observe_entry_size(10.0); + assert_eq!(cell.get(), 1); + } +} diff --git a/rust/metrics/src/observer.rs b/rust/metrics/src/observer.rs new file mode 100644 index 0000000..84fe76f --- /dev/null +++ b/rust/metrics/src/observer.rs @@ -0,0 +1,16 @@ +//! Publish accounting hook implemented by the broker. + +use std::sync::Arc; + +/// Per-topic publish accounting hook implemented by the broker. +/// +/// The write-queue worker calls it once per successfully committed group +/// with the group's message and byte totals, so broker counters fold whole +/// batches into single atomic updates while storage crates stay free of +/// any metrics-family knowledge. +pub trait PublishCommitObserver: Send + Sync { + fn on_commit(&self, messages: u64, bytes: u64); +} + +/// Convenience alias for pre-resolved observer handles carried per request. +pub type SharedObserver = Arc; diff --git a/rust/metrics/src/storage.rs b/rust/metrics/src/storage.rs new file mode 100644 index 0000000..169d512 --- /dev/null +++ b/rust/metrics/src/storage.rs @@ -0,0 +1,228 @@ +//! Storage-side metric families observed from the managed-ledger write path. + +use std::sync::{Arc, LazyLock, OnceLock}; + +use prometheus::{Histogram, HistogramOpts, IntCounter, Registry}; + +use crate::global_registry; + +/// Native Pulsar's addEntry latency bucket layout, in seconds +/// ({0.5, 1, 5, 10, 20, 50, 100, 200, 1000} ms). +const WRITE_LATENCY_BUCKETS: &[f64] = &[ + 0.0005, 0.001, 0.005, 0.01, 0.02, 0.05, 0.1, 0.2, 1.0, +]; + +/// Native Pulsar's entry-size bucket layout, in bytes +/// ({128, 512, 1K, 2K, 4K, 16K, 100K, 1M}). +const ENTRY_SIZE_BUCKETS: &[f64] = &[ + 128.0, 512.0, 1024.0, 2048.0, 4096.0, 16384.0, 102400.0, 1048576.0, +]; + +/// Write-queue batch sizes (MAX_BATCH today is 64). +const BATCH_SIZE_BUCKETS: &[f64] = &[1.0, 2.0, 4.0, 8.0, 16.0, 32.0, 64.0]; + +/// Storage-side histogram/counter families. +/// +/// `pulsar_storage_write_latency` is the end-to-end durable-publish view +/// (enqueue → committed batch), the equivalent of native addEntry latency +/// including queue wait. `pulsar_storage_ledger_write_latency` covers only +/// the ledger append (native bookie-write view). Write-queue families +/// replace the removed 1 Hz summary log. +#[derive(Debug)] +pub struct StorageMetrics { + write_latency: Histogram, + ledger_write_latency: Histogram, + entry_size: Histogram, + wq_batches: IntCounter, + wq_batch_messages: IntCounter, + wq_batch_size: Histogram, +} + +static STORAGE_METRICS: OnceLock> = OnceLock::new(); + +impl StorageMetrics { + fn new(cluster: &str, registry: &Registry) -> Result { + let write_latency = Histogram::with_opts( + HistogramOpts::new( + "pulsar_storage_write_latency", + "End-to-end durable publish latency (enqueue to committed batch)", + ) + .buckets(WRITE_LATENCY_BUCKETS.to_vec()), + )?; + let ledger_write_latency = Histogram::with_opts( + HistogramOpts::new( + "pulsar_storage_ledger_write_latency", + "Managed-ledger append latency per committed group", + ) + .buckets(WRITE_LATENCY_BUCKETS.to_vec()), + )?; + let entry_size = Histogram::with_opts( + HistogramOpts::new("pulsar_entry_size", "Accepted entry size in bytes") + .buckets(ENTRY_SIZE_BUCKETS.to_vec()), + )?; + let wq_batches = IntCounter::new( + "pulsar_lite_write_queue_batches_total", + "Write-queue batches drained", + )?; + let wq_batch_messages = IntCounter::new( + "pulsar_lite_write_queue_batch_messages_total", + "Messages passed through the write queue", + )?; + let wq_batch_size = Histogram::with_opts( + HistogramOpts::new( + "pulsar_lite_write_queue_batch_size", + "Write-queue batch size in messages", + ) + .buckets(BATCH_SIZE_BUCKETS.to_vec()), + )?; + + registry.register(Box::new(write_latency.clone()))?; + registry.register(Box::new(ledger_write_latency.clone()))?; + registry.register(Box::new(entry_size.clone()))?; + registry.register(Box::new(wq_batches.clone()))?; + registry.register(Box::new(wq_batch_messages.clone()))?; + registry.register(Box::new(wq_batch_size.clone()))?; + let _ = cluster; // families are broker-global; reserved for future labels + + Ok(Self { + write_latency, + ledger_write_latency, + entry_size, + wq_batches, + wq_batch_messages, + wq_batch_size, + }) + } +} + +impl StorageMetrics { + /// End-to-end durable-publish latency (seconds). + pub fn observe_write_latency(&self, seconds: f64) { + self.write_latency.observe(seconds); + } + + /// Ledger-append latency (seconds). + pub fn observe_ledger_write_latency(&self, seconds: f64) { + self.ledger_write_latency.observe(seconds); + } + + /// Accepted entry size (metadata + payload bytes). + pub fn observe_entry_size(&self, bytes: f64) { + self.entry_size.observe(bytes); + } + + /// One drained write-queue batch of `messages` entries. + pub fn observe_batch(&self, messages: u64) { + self.wq_batches.inc(); + self.wq_batch_messages.inc_by(messages); + self.wq_batch_size.observe(messages as f64); + } +} + +/// A no-op stand-in used before [`init`]; keeps call sites branch-free. +#[derive(Debug)] +pub struct DisabledStorageMetrics; + +impl DisabledStorageMetrics { + fn observe_write_latency(&self, _seconds: f64) {} + fn observe_ledger_write_latency(&self, _seconds: f64) {} + fn observe_entry_size(&self, _bytes: f64) {} + fn observe_batch(&self, _messages: u64) {} +} + +/// Unified accessor so workers can record without checking initialization. +#[derive(Debug, Clone)] +pub enum StorageMetricsHandle { + Enabled(Arc), + Disabled(Arc), +} + +impl StorageMetricsHandle { + pub fn observe_write_latency(&self, seconds: f64) { + match self { + Self::Enabled(metrics) => metrics.observe_write_latency(seconds), + Self::Disabled(metrics) => metrics.observe_write_latency(seconds), + } + } + + pub fn observe_ledger_write_latency(&self, seconds: f64) { + match self { + Self::Enabled(metrics) => metrics.observe_ledger_write_latency(seconds), + Self::Disabled(metrics) => metrics.observe_ledger_write_latency(seconds), + } + } + + pub fn observe_entry_size(&self, bytes: f64) { + match self { + Self::Enabled(metrics) => metrics.observe_entry_size(bytes), + Self::Disabled(metrics) => metrics.observe_entry_size(bytes), + } + } + + pub fn observe_batch(&self, messages: u64) { + match self { + Self::Enabled(metrics) => metrics.observe_batch(messages), + Self::Disabled(metrics) => metrics.observe_batch(messages), + } + } +} + +static HANDLE: LazyLock = LazyLock::new(|| { + match STORAGE_METRICS.get() { + Some(metrics) => StorageMetricsHandle::Enabled(Arc::clone(metrics)), + None => StorageMetricsHandle::Disabled(Arc::new(DisabledStorageMetrics)), + } +}); + +/// Returns the storage metrics handle (no-op before [`init`]). +pub fn storage_metrics() -> StorageMetricsHandle { + HANDLE.clone() +} + +/// Registers storage metric families into the shared global registry. +/// +/// Idempotent: the first call wins. Called by broker startup; unit tests +/// that never call it get no-op handles. +pub fn init(cluster: &str) -> Arc { + STORAGE_METRICS + .get_or_init(|| match StorageMetrics::new(cluster, &global_registry()) { + Ok(metrics) => Arc::new(metrics), + Err(error) => { + log::error!("Failed to register storage metrics: {}", error); + // Names are static and test-proven; this branch is unreachable + // unless the definitions themselves are invalid. + Arc::new( + StorageMetrics::new(cluster, &Registry::new()) + .expect("static storage metric definitions are valid"), + ) + } + }) + .clone() +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn global_registry_is_stable_across_calls() { + assert!(Arc::ptr_eq(&global_registry(), &global_registry())); + } + + #[test] + fn handle_before_init_is_noop() { + // Do not call init(): the disabled handle must swallow observations. + let handle = storage_metrics(); + handle.observe_write_latency(0.1); + handle.observe_ledger_write_latency(0.2); + handle.observe_entry_size(42.0); + handle.observe_batch(7); + } + + #[test] + fn init_registers_enabled_handle() { + let metrics = init("test-cluster"); + metrics.observe_write_latency(0.001); + assert_eq!(metrics.write_latency.get_sample_count(), 1); + } +} diff --git a/rust/metrics/src/subscription.rs b/rust/metrics/src/subscription.rs new file mode 100644 index 0000000..50e00a6 --- /dev/null +++ b/rust/metrics/src/subscription.rs @@ -0,0 +1,214 @@ +/* + * Subscription-level metric handles + * + * One `SubscriptionMetrics` per Subscription entity, resolved at creation + * from the global families (labels `{cluster, namespace, topic, partition, + * subscription}`). Hot paths touch pre-resolved atomic handles only; + * gauges are set by the scrape aggregation loop. + */ + +use std::collections::VecDeque; +use std::sync::{Arc, Mutex}; +use std::time::{Instant, SystemTime, UNIX_EPOCH}; + +use prometheus::{Gauge, IntCounter, IntGauge}; + +use crate::broker::BrokerMetrics; +use crate::topic::parse_topic_labels; + +/// (timestamp, out_messages, out_bytes, acked, redelivered, dropped). +type RateSample = (Instant, u64, u64, u64, u64, u64); +type RateSampleWindow = VecDeque; + +/// Pre-resolved per-subscription handles. +pub struct SubscriptionMetrics { + out_messages: IntCounter, + out_bytes: IntCounter, + redelivered: IntCounter, + dropped: IntCounter, + acked: IntCounter, + msg_rate_out: Gauge, + msg_throughput_out: Gauge, + msg_ack_rate: Gauge, + msg_rate_redeliver: Gauge, + msg_drop_rate: Gauge, + back_log: IntGauge, + consumers_count: IntGauge, + unacked_messages: IntGauge, + blocked_on_unacked: IntGauge, + last_acked_timestamp: Gauge, + last_consumed_timestamp: Gauge, + broker: Arc, + /// Samples for derived rates; touched only by the scrape task. Seeded + /// with a zero baseline. + rate_samples: Mutex, +} + +impl SubscriptionMetrics { + /// Resolves all handles for `(topic_name, subscription_name)`. + pub fn new(topic_name: &str, subscription_name: &str) -> Self { + let broker = crate::get(); + let labels = parse_topic_labels(topic_name); + let cluster = broker.cluster(); + let partition = labels.partition.to_string(); + let values: Vec<&str> = vec![ + cluster, + labels.namespace.as_str(), + labels.topic.as_str(), + partition.as_str(), + subscription_name, + ]; + + let families = &broker.subscriptions; + let mut samples: RateSampleWindow = VecDeque::new(); + samples.push_back((Instant::now(), 0, 0, 0, 0, 0)); + Self { + out_messages: families.out_messages.with_label_values(&values), + out_bytes: families.out_bytes.with_label_values(&values), + redelivered: families.redelivered.with_label_values(&values), + dropped: families.dropped.with_label_values(&values), + acked: families.acked.with_label_values(&values), + msg_rate_out: families.msg_rate_out.with_label_values(&values), + msg_throughput_out: families.msg_throughput_out.with_label_values(&values), + msg_ack_rate: families.msg_ack_rate.with_label_values(&values), + msg_rate_redeliver: families.msg_rate_redeliver.with_label_values(&values), + msg_drop_rate: families.msg_drop_rate.with_label_values(&values), + back_log: families.back_log.with_label_values(&values), + consumers_count: families.consumers_count.with_label_values(&values), + unacked_messages: families.unacked_messages.with_label_values(&values), + blocked_on_unacked: families.blocked_on_unacked.with_label_values(&values), + last_acked_timestamp: families.last_acked_timestamp.with_label_values(&values), + last_consumed_timestamp: families.last_consumed_timestamp.with_label_values(&values), + broker, + rate_samples: Mutex::new(samples), + } + } + + /// Records a message dispatched to a consumer of this subscription. + pub fn record_dispatched(&self, bytes: u64) { + self.out_messages.inc(); + self.out_bytes.inc_by(bytes); + self.broker.broker_out_messages.inc(); + self.broker.broker_out_bytes.inc_by(bytes); + self.last_consumed_timestamp.set(unix_epoch_seconds()); + } + + /// Records an acknowledged message. + pub fn record_acked(&self) { + self.acked.inc(); + self.last_acked_timestamp.set(unix_epoch_seconds()); + } + + /// Records messages queued for redelivery. + pub fn record_redelivered(&self, messages: u64) { + self.redelivered.inc_by(messages); + } + + /// Records a non-persistent message dropped (no writable consumer). + pub fn record_dropped(&self) { + self.dropped.inc(); + } + + /// Records `count` non-persistent drops (batch drop accounting). + pub fn record_dropped_n(&self, count: u64) { + self.dropped.inc_by(count); + } + + /// Scrape task: refreshes derived rate gauges over `window_secs`. + /// Returns (out_messages, out_bytes) deltas for broker-level folding. + pub fn update_rates(&self, window_secs: u64) -> (u64, u64) { + let now = Instant::now(); + let window = std::time::Duration::from_secs(window_secs.max(1)); + let current = ( + self.out_messages.get(), + self.out_bytes.get(), + self.acked.get(), + self.redelivered.get(), + self.dropped.get(), + ); + + let mut samples = match self.rate_samples.lock() { + Ok(guard) => guard, + Err(poisoned) => poisoned.into_inner(), + }; + samples.push_back((now, current.0, current.1, current.2, current.3, current.4)); + while let Some((t, ..)) = samples.front() { + if now.duration_since(*t) > window { + samples.pop_front(); + } else { + break; + } + } + + let deltas = match (samples.front(), samples.back()) { + (Some((t0, m0, b0, a0, r0, d0)), Some((t1, m1, b1, a1, r1, d1))) => { + let elapsed = t1.duration_since(*t0).as_secs_f64(); + ( + m1.saturating_sub(*m0), + b1.saturating_sub(*b0), + a1.saturating_sub(*a0), + r1.saturating_sub(*r0), + d1.saturating_sub(*d0), + elapsed, + ) + } + _ => (0, 0, 0, 0, 0, 0.0), + }; + let (d_out, d_bytes, d_acked, d_redelivered, d_dropped, elapsed) = deltas; + + if elapsed >= 1.0 { + self.msg_rate_out.set(d_out as f64 / elapsed); + self.msg_throughput_out.set(d_bytes as f64 / elapsed); + self.msg_ack_rate.set(d_acked as f64 / elapsed); + self.msg_rate_redeliver.set(d_redelivered as f64 / elapsed); + self.msg_drop_rate.set(d_dropped as f64 / elapsed); + } + (d_out, d_bytes) + } + + /// Scrape task: state gauges for this subscription. + pub fn set_state(&self, back_log: i64, unacked: i64, blocked: bool, consumers: i64) { + self.back_log.set(back_log); + self.unacked_messages.set(unacked); + self.blocked_on_unacked.set(blocked as i64); + self.consumers_count.set(consumers); + } +} + +impl std::fmt::Debug for SubscriptionMetrics { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("SubscriptionMetrics") + .finish_non_exhaustive() + } +} + +fn unix_epoch_seconds() -> f64 { + SystemTime::now() + .duration_since(UNIX_EPOCH) + .map(|d| d.as_secs_f64()) + .unwrap_or(0.0) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn dispatched_acked_and_liveness_tick() { + let metrics = + SubscriptionMetrics::new("persistent://public/subns/sub-topic", "sub-unit-test"); + metrics.record_dispatched(120); + metrics.record_dispatched(30); + metrics.record_acked(); + metrics.record_redelivered(2); + metrics.record_dropped(); + + let (d_out, d_bytes) = metrics.update_rates(60); + assert_eq!(d_out, 2); + assert_eq!(d_bytes, 150); + + metrics.set_state(3, 1, true, 2); + // State gauges are read through the registry in integration checks; + // here we only assert the counters did not panic and rates reported. + } +} diff --git a/rust/metrics/src/topic.rs b/rust/metrics/src/topic.rs new file mode 100644 index 0000000..1520bbf --- /dev/null +++ b/rust/metrics/src/topic.rs @@ -0,0 +1,263 @@ +/* + * Topic-level metric handles + * + * One `TopicMetrics` per Topic entity, created at `Topic::new` time. All + * label resolution happens exactly once here; hot paths afterwards only + * touch the resolved `IntCounter`/`IntGauge` handles (plain atomic ops). + * + * Label conventions mirror native Pulsar: `{cluster, namespace, topic, + * partition}`, with `partition="-1"` for non-partitioned topics and the + * `-partition-N` suffix stripped from the exported topic label. + */ + +use std::collections::VecDeque; +use std::sync::{Arc, Mutex}; +use std::time::Instant; + +use prometheus::{Gauge, IntCounter, IntGauge}; + +use crate::broker::BrokerMetrics; + +/// Native-style decomposition of a full topic URI. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct TopicLabels { + /// `tenant/namespace` (native namespace label format). + pub namespace: String, + /// Local topic name without domain or partition suffix. + pub topic: String, + /// Partition index, `-1` for non-partitioned topics. + pub partition: i32, +} + +/// Splits `persistent://tenant/ns/local[-partition-N]` into label values. +/// +/// Malformed names degrade to `namespace="unknown"` instead of failing: +/// metrics must never reject traffic the broker already accepted. +pub fn parse_topic_labels(name: &str) -> TopicLabels { + let body = match name.split_once("://") { + Some((_, rest)) => rest, + None => name, + }; + + let mut parts = body.splitn(3, '/'); + let tenant = parts.next().unwrap_or_default(); + let namespace_part = parts.next().unwrap_or_default(); + let local = parts.next().unwrap_or(body); + + let namespace = if tenant.is_empty() || namespace_part.is_empty() { + "unknown".to_string() + } else { + format!("{}/{}", tenant, namespace_part) + }; + + let (topic, partition) = match local.rsplit_once("-partition-") { + Some((base, digits)) + if !base.is_empty() + && !digits.is_empty() + && digits.bytes().all(|b| b.is_ascii_digit()) => + { + (base.to_string(), digits.parse::().unwrap_or(-1)) + } + _ => (local.to_string(), -1), + }; + + TopicLabels { + namespace, + topic, + partition, + } +} + +/// Pre-resolved per-topic handles (counters) + scrape-set gauges. +pub struct TopicMetrics { + in_messages: IntCounter, + in_bytes: IntCounter, + publish_rate_limit: IntCounter, + rate_in: Gauge, + throughput_in: Gauge, + average_msg_size: IntGauge, + storage_size: IntGauge, + subscriptions_count: IntGauge, + producers_count: IntGauge, + consumers_count: IntGauge, + broker: Arc, + /// (timestamp, cumulative messages, cumulative bytes) samples for the + /// scrape-derived rate gauges; only the scrape task touches this. + /// Seeded with a zero baseline so the first update already reports + /// the full window delta. + rate_samples: Mutex>, +} + +impl TopicMetrics { + /// Resolves all handles for `topic_name` from the global families. + pub fn new(topic_name: &str) -> Self { + let broker = crate::get(); + let labels = parse_topic_labels(topic_name); + let cluster = broker.cluster(); + let values = [cluster, labels.namespace.as_str(), labels.topic.as_str()]; + let partition = labels.partition.to_string(); + let mut with_partition = values.to_vec(); + with_partition.push(partition.as_str()); + let vals: Vec<&str> = with_partition; + + let families = &broker.topics; + Self { + in_messages: families.in_messages.with_label_values(&vals), + in_bytes: families.in_bytes.with_label_values(&vals), + publish_rate_limit: families.publish_rate_limit.with_label_values(&vals), + rate_in: families.rate_in.with_label_values(&vals), + throughput_in: families.throughput_in.with_label_values(&vals), + average_msg_size: families.average_msg_size.with_label_values(&vals), + storage_size: families.storage_size.with_label_values(&vals), + subscriptions_count: families.subscriptions_count.with_label_values(&vals), + producers_count: families.producers_count.with_label_values(&vals), + consumers_count: families.consumers_count.with_label_values(&vals), + broker, + rate_samples: { + let mut samples = VecDeque::new(); + samples.push_back((Instant::now(), 0, 0)); + Mutex::new(samples) + }, + } + } + + /// Records accepted publishes (messages and payload+metadata bytes). + /// + /// Called from single-writer contexts (non-persistent fan-out worker, + /// in-process publish); the durable write-queue path goes through the + /// `PublishCommitObserver` impl below instead. + pub fn record_publish(&self, messages: u64, bytes: u64) { + self.in_messages.inc_by(messages); + self.in_bytes.inc_by(bytes); + self.broker.broker_in_messages.inc_by(messages); + self.broker.broker_in_bytes.inc_by(bytes); + } + + /// Records a publish-rate-limit rejection. + pub fn record_rate_limit_reject(&self) { + self.publish_rate_limit.inc(); + } + + /// Scrape task: refreshes derived rate gauges over `window_secs`. + /// + /// Returns the window delta (messages, bytes) so the caller can fold + /// broker-level rate gauges in the same pass. + pub fn update_rates(&self, window_secs: u64) -> (u64, u64) { + let (messages, bytes) = (self.in_messages.get(), self.in_bytes.get()); + let now = Instant::now(); + let window = std::time::Duration::from_secs(window_secs.max(1)); + + let mut samples = match self.rate_samples.lock() { + Ok(guard) => guard, + Err(poisoned) => poisoned.into_inner(), + }; + samples.push_back((now, messages, bytes)); + while let Some((t, _, _)) = samples.front() { + if now.duration_since(*t) > window { + samples.pop_front(); + } else { + break; + } + } + + let (d_messages, d_bytes, elapsed) = match (samples.front(), samples.back()) { + (Some((t0, m0, b0)), Some((t1, m1, b1))) => { + let elapsed = t1.duration_since(*t0).as_secs_f64(); + (m1.saturating_sub(*m0), b1.saturating_sub(*b0), elapsed) + } + _ => (0, 0, 0.0), + }; + + if elapsed >= 1.0 { + self.rate_in.set(d_messages as f64 / elapsed); + self.throughput_in.set(d_bytes as f64 / elapsed); + self.average_msg_size.set(if d_messages > 0 { + (d_bytes / d_messages) as i64 + } else { + 0 + }); + } + (d_messages, d_bytes) + } + + /// Scrape task: entity count gauges for this topic. + pub fn set_entity_counts(&self, subscriptions: i64, producers: i64, consumers: i64) { + self.subscriptions_count.set(subscriptions); + self.producers_count.set(producers); + self.consumers_count.set(consumers); + } + + /// Scrape task: stored bytes for this topic. + pub fn set_storage_size(&self, bytes: i64) { + self.storage_size.set(bytes); + } +} + +impl std::fmt::Debug for TopicMetrics { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("TopicMetrics").finish_non_exhaustive() + } +} +/// Durable-batch accounting for the write-queue worker (single writer). +impl crate::observer::PublishCommitObserver for TopicMetrics { + fn on_commit(&self, messages: u64, bytes: u64) { + self.in_messages.inc_by(messages); + self.in_bytes.inc_by(bytes); + self.broker.broker_in_messages.inc_by(messages); + self.broker.broker_in_bytes.inc_by(bytes); + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn parses_persistent_topic_uri() { + let labels = parse_topic_labels("persistent://public/default/orders"); + assert_eq!(labels.namespace, "public/default"); + assert_eq!(labels.topic, "orders"); + assert_eq!(labels.partition, -1); + } + + #[test] + fn parses_partition_suffix() { + let labels = parse_topic_labels("non-persistent://tenant/ns/queue-partition-7"); + assert_eq!(labels.namespace, "tenant/ns"); + assert_eq!(labels.topic, "queue"); + assert_eq!(labels.partition, 7); + } + + #[test] + fn malformed_names_degrade_to_unknown_namespace() { + let labels = parse_topic_labels("bare-topic"); + assert_eq!(labels.namespace, "unknown"); + assert_eq!(labels.topic, "bare-topic"); + assert_eq!(labels.partition, -1); + } + + #[test] + fn non_numeric_partition_suffix_is_kept_in_topic() { + let labels = parse_topic_labels("persistent://t/n/v2-partition-x"); + assert_eq!(labels.topic, "v2-partition-x"); + assert_eq!(labels.partition, -1); + } + + #[test] + fn topic_metrics_counters_and_rates_tick() { + let metrics = TopicMetrics::new("persistent://public/tickns/metrics-tick"); + metrics.record_publish(3, 300); + metrics.record_rate_limit_reject(); + + // First update reports the full delta from the zero baseline + // (the elapsed guard only suppresses the rate gauges, not the + // returned delta used for broker-level folding). + let (d_msgs, d_bytes) = metrics.update_rates(60); + assert_eq!(d_msgs, 3); + assert_eq!(d_bytes, 300); + + let labels = parse_topic_labels("persistent://public/tickns/metrics-tick"); + assert_eq!(labels.namespace, "public/tickns"); + assert_eq!(labels.topic, "metrics-tick"); + } +} diff --git a/rust/storage/core/Cargo.toml b/rust/storage/core/Cargo.toml index 5147e53..a52ab76 100644 --- a/rust/storage/core/Cargo.toml +++ b/rust/storage/core/Cargo.toml @@ -17,6 +17,7 @@ pulsar-lite-proto = { path = "../../proto" } anyhow.workspace = true log.workspace = true prost = "0.13" +prometheus.workspace = true [dev-dependencies] tempfile = "3.9" diff --git a/rust/storage/core/src/backend.rs b/rust/storage/core/src/backend.rs index 51a8b1e..8887d1c 100644 --- a/rust/storage/core/src/backend.rs +++ b/rust/storage/core/src/backend.rs @@ -277,4 +277,20 @@ impl ManagedLedgerStorage for ManagedLedgerStore { Self::RocksDb(inner) => inner.get_mark_delete_position(topic, subscription), } } + + fn backlog_entries(&self, topic: &str, subscription: &str) -> Option { + match self { + Self::Memory(inner) => inner.backlog_entries(topic, subscription), + #[cfg(feature = "rocksdb-storage")] + Self::RocksDb(inner) => inner.backlog_entries(topic, subscription), + } + } + + fn stored_bytes(&self, topic: &str) -> u64 { + match self { + Self::Memory(inner) => inner.stored_bytes(topic), + #[cfg(feature = "rocksdb-storage")] + Self::RocksDb(inner) => inner.stored_bytes(topic), + } + } } diff --git a/rust/storage/core/src/lib.rs b/rust/storage/core/src/lib.rs index 4b4a78d..3739f02 100644 --- a/rust/storage/core/src/lib.rs +++ b/rust/storage/core/src/lib.rs @@ -3,8 +3,7 @@ mod backend; mod config; mod error; -mod service; - +pub mod service; pub use backend::ManagedLedgerStore; pub use config::{ManagedLedgerBackendConfig, StorageConfig}; pub use error::StorageResult; diff --git a/rust/storage/core/src/service.rs b/rust/storage/core/src/service.rs index 4e48853..731b43e 100644 --- a/rust/storage/core/src/service.rs +++ b/rust/storage/core/src/service.rs @@ -312,4 +312,14 @@ impl Storage { self.managed_ledger .get_mark_delete_position(topic, subscription) } + + /// Ledger-aware unacked entry count for a subscription cursor. + pub fn backlog_entries(&self, topic: &str, subscription: &str) -> Option { + self.managed_ledger.backlog_entries(topic, subscription) + } + + /// Bytes durably stored for `topic`. + pub fn stored_bytes(&self, topic: &str) -> u64 { + self.managed_ledger.stored_bytes(topic) + } } diff --git a/rust/storage/managed-ledger-rocksdb/Cargo.toml b/rust/storage/managed-ledger-rocksdb/Cargo.toml index 18cc9ee..f63bcce 100644 --- a/rust/storage/managed-ledger-rocksdb/Cargo.toml +++ b/rust/storage/managed-ledger-rocksdb/Cargo.toml @@ -5,6 +5,7 @@ edition.workspace = true license.workspace = true [dependencies] +pulsar-lite-metrics = { path = "../../metrics" } pulsar-lite-storage-managed-ledger = { path = "../managed-ledger" } anyhow.workspace = true rocksdb = "0.24" diff --git a/rust/storage/managed-ledger-rocksdb/src/lib.rs b/rust/storage/managed-ledger-rocksdb/src/lib.rs index 6a6668d..78266f2 100644 --- a/rust/storage/managed-ledger-rocksdb/src/lib.rs +++ b/rust/storage/managed-ledger-rocksdb/src/lib.rs @@ -10,6 +10,7 @@ mod write_queue; pub use store::{ConcurrentAppender, RocksDbManagedLedgerStorage}; pub use write_queue::ConnAppendResult; +pub use pulsar_lite_metrics::PublishCommitObserver; /// Internal types exposed for integration tests in `tests/`. #[doc(hidden)] diff --git a/rust/storage/managed-ledger-rocksdb/src/store.rs b/rust/storage/managed-ledger-rocksdb/src/store.rs index 9aed3f7..e0e3a25 100644 --- a/rust/storage/managed-ledger-rocksdb/src/store.rs +++ b/rust/storage/managed-ledger-rocksdb/src/store.rs @@ -58,7 +58,8 @@ impl ConcurrentAppender { } /// Enqueue without waiting. Completion is delivered on `completion_tx` - /// from the write-queue worker thread. + /// from the write-queue worker thread; `observer` (when present) is + /// invoked once per committed batch group with message/byte totals. pub fn enqueue_for_connection( &self, topic: &str, @@ -67,6 +68,7 @@ impl ConcurrentAppender { payload: &[u8], producer_id: u64, sequence_id: u64, + observer: Option>, completion_tx: tokio::sync::mpsc::Sender, ) -> Result<()> { WriteQueue::enqueue_for_connection( @@ -77,6 +79,7 @@ impl ConcurrentAppender { payload, producer_id, sequence_id, + observer, completion_tx, ) .map_err(|e| anyhow!(e)) @@ -471,4 +474,43 @@ impl ManagedLedgerStorage for RocksDbManagedLedgerStorage { .as_ref() .map(|position| position.entry_id) } + + fn backlog_entries(&self, topic: &str, subscription: &str) -> Option { + let _guard = self.cursor_mu.lock().ok()?; + let ledger = self.topic_ledger(topic).ok()?; + let info = ledger.info_snapshot(); + let total: u64 = info.ledgers.iter().map(|l| l.entries).sum(); + + let cursor = ledger + .open_cursor(&keys::encode_cursor_name(subscription)) + .ok()?; + let state = cursor.state(); + let Some(mark) = state.mark_delete.as_ref() else { + // Cursor never advanced: nothing acknowledged yet. + return Some(total); + }; + // Acked = everything in fully-retired ledgers below the mark's + // ledger, the mark itself, and individually-deleted holes above it. + let retired: u64 = info + .ledgers + .iter() + .filter(|l| l.ledger_id < mark.ledger_id) + .map(|l| l.entries) + .sum(); + let holes: u64 = state + .individually_deleted_entries + .iter() + .map(|(start, end)| end.entry_id.saturating_sub(start.entry_id) + 1) + .sum(); + let acked = retired + mark.entry_id + 1 + holes; + Some(total.saturating_sub(acked)) + } + + fn stored_bytes(&self, topic: &str) -> u64 { + let Ok(ledger) = self.topic_ledger(topic) else { + return 0; + }; + let info = ledger.info_snapshot(); + info.ledgers.iter().map(|l| l.size).sum() + } } diff --git a/rust/storage/managed-ledger-rocksdb/src/write_queue.rs b/rust/storage/managed-ledger-rocksdb/src/write_queue.rs index cf7da5e..5025e9c 100644 --- a/rust/storage/managed-ledger-rocksdb/src/write_queue.rs +++ b/rust/storage/managed-ledger-rocksdb/src/write_queue.rs @@ -1,7 +1,9 @@ use crate::factory::{RocksDBManagedLedgerFactory, SharedLedger}; use crate::keys; +use pulsar_lite_metrics::PublishCommitObserver; use pulsar_lite_storage_managed_ledger::MessageId; use std::collections::HashMap; +use std::sync::Arc; use std::sync::mpsc; use std::thread::{self, JoinHandle}; use std::time::{Duration, Instant}; @@ -59,6 +61,10 @@ pub(crate) struct WriteReq { partition: i32, metadata: Vec, payload: Vec, + observer: Option>, + /// Set on the connection path; anchors the end-to-end publish latency + /// histogram (enqueue → committed batch). + enqueued_at: Option, reply: WriteReply, } @@ -93,16 +99,8 @@ impl WriteQueue { // Same Arc instances as store reads (singleton per ledger name). let mut ledgers: HashMap = HashMap::new(); - // Batch-size metrics (aggregated to avoid log spam under stress). - let mut metric_batches: u64 = 0; - let mut metric_msgs: u64 = 0; - let mut metric_max_batch: usize = 0; - let mut metric_batch_eq1: u64 = 0; - let mut metric_group_ops: u64 = 0; - let mut metric_group_msgs: u64 = 0; - let mut metric_max_group: usize = 0; - let mut metric_group_eq1: u64 = 0; - let mut metric_window_started = Instant::now(); + // Prometheus families (no-op before metrics::init). + let metrics = pulsar_lite_metrics::storage_metrics(); loop { let first = match rx.recv() { @@ -149,12 +147,7 @@ impl WriteQueue { } let queue_batch_len = batch.len(); - metric_batches += 1; - metric_msgs += queue_batch_len as u64; - metric_max_batch = metric_max_batch.max(queue_batch_len); - if queue_batch_len == 1 { - metric_batch_eq1 += 1; - } + metrics.observe_batch(queue_batch_len as u64); // Group by ledger so one rocksdb write covers many entries of the same topic. let mut order: Vec = Vec::new(); @@ -173,13 +166,6 @@ impl WriteQueue { _ => continue, }; - let group_len = reqs.len(); - metric_group_ops += 1; - metric_group_msgs += group_len as u64; - metric_max_group = metric_max_group.max(group_len); - if group_len == 1 { - metric_group_eq1 += 1; - } if !ledgers.contains_key(&ledger_name) { match factory.open_ledger(&ledger_name) { @@ -212,8 +198,36 @@ impl WriteQueue { // Append publishes meta then LAC only after durable OK; // complete only after that returns Ok. - match ledger.add_entries_with_partition_and_metadata(&inputs) { + let append_started = Instant::now(); + let append_result = + ledger.add_entries_with_partition_and_metadata(&inputs); + metrics.observe_ledger_write_latency(append_started.elapsed().as_secs_f64()); + match append_result { Ok(positions) => { + // One observer call per committed group: all reqs in + // a group share the topic, so the first req's observer + // already targets the right counters. + let mut committed_messages: u64 = 0; + let mut committed_bytes: u64 = 0; + let mut observer: Option> = None; + let committed_at = Instant::now(); + for req in &reqs { + committed_messages += 1; + let req_bytes = (req.metadata.len() + req.payload.len()) as u64; + committed_bytes += req_bytes; + metrics.observe_entry_size(req_bytes as f64); + if let Some(enqueued_at) = req.enqueued_at { + metrics.observe_write_latency( + committed_at.duration_since(enqueued_at).as_secs_f64(), + ); + } + if observer.is_none() { + observer = req.observer.clone(); + } + } + if let Some(observer) = observer { + observer.on_commit(committed_messages, committed_bytes); + } for (req, position) in reqs.into_iter().zip(positions) { req.reply.complete(Ok(MessageId::from(position))); } @@ -227,53 +241,6 @@ impl WriteQueue { } } - // Emit ~1Hz summary so stress runs stay readable. - if metric_window_started.elapsed() >= Duration::from_secs(1) { - let avg_queue = if metric_batches == 0 { - 0.0 - } else { - metric_msgs as f64 / metric_batches as f64 - }; - let avg_group = if metric_group_ops == 0 { - 0.0 - } else { - metric_group_msgs as f64 / metric_group_ops as f64 - }; - let pct_queue_eq1 = if metric_batches == 0 { - 0.0 - } else { - 100.0 * metric_batch_eq1 as f64 / metric_batches as f64 - }; - let pct_group_eq1 = if metric_group_ops == 0 { - 0.0 - } else { - 100.0 * metric_group_eq1 as f64 / metric_group_ops as f64 - }; - - log::info!( - "write_queue metrics: queue_batches={} queue_msgs={} queue_batch_avg={:.2} queue_batch_max={} queue_batch_eq1={:.1}% group_ops={} group_msgs={} group_avg={:.2} group_max={} group_eq1={:.1}%", - metric_batches, - metric_msgs, - avg_queue, - metric_max_batch, - pct_queue_eq1, - metric_group_ops, - metric_group_msgs, - avg_group, - metric_max_group, - pct_group_eq1, - ); - - metric_batches = 0; - metric_msgs = 0; - metric_max_batch = 0; - metric_batch_eq1 = 0; - metric_group_ops = 0; - metric_group_msgs = 0; - metric_max_group = 0; - metric_group_eq1 = 0; - metric_window_started = Instant::now(); - } } } @@ -298,13 +265,14 @@ impl WriteQueue { partition, metadata: metadata.to_vec(), payload: payload.to_vec(), + observer: None, + enqueued_at: None, reply: WriteReply::Oneshot(reply_tx), }) .map_err(|_| "write queue worker disconnected".to_string())?; Ok(reply_rx) } - /// Enqueue for a broker connection: returns immediately; completion is pushed to `completion_tx`. pub(crate) fn enqueue_for_connection( tx: &mpsc::Sender, topic: &str, @@ -313,6 +281,7 @@ impl WriteQueue { payload: &[u8], producer_id: u64, sequence_id: u64, + observer: Option>, completion_tx: tokio::sync::mpsc::Sender, ) -> Result<(), String> { tx.send(WriteReq { @@ -320,6 +289,8 @@ impl WriteQueue { partition, metadata: metadata.to_vec(), payload: payload.to_vec(), + observer, + enqueued_at: Some(Instant::now()), reply: WriteReply::Conn { producer_id, sequence_id, diff --git a/rust/storage/managed-ledger-rocksdb/tests/backlog_entries.rs b/rust/storage/managed-ledger-rocksdb/tests/backlog_entries.rs new file mode 100644 index 0000000..98fb9c3 --- /dev/null +++ b/rust/storage/managed-ledger-rocksdb/tests/backlog_entries.rs @@ -0,0 +1,63 @@ +use pulsar_lite_storage_managed_ledger::{ + CursorInitOptions, InitialPosition, ManagedLedgerStorage, +}; +use pulsar_lite_storage_managed_ledger_rocksdb::RocksDbManagedLedgerStorage; +use tempfile::tempdir; + +fn open(path: &std::path::Path) -> RocksDbManagedLedgerStorage { + RocksDbManagedLedgerStorage::open(path).expect("open rocksdb store") +} + +#[test] +fn backlog_tracks_shared_acks_and_full_drain() { + let dir = tempdir().expect("tempdir"); + let topic = "persistent://public/default/backlog-probe"; + + { + let mut store = open(dir.path()); + store.create_topic(topic).expect("create topic"); + store + .initialize_or_open_cursor( + topic, + "sub", + CursorInitOptions { + initial_position: InitialPosition::Earliest, + ..Default::default() + }, + ) + .expect("open cursor"); + for i in 0..10u64 { + store + .append_message(topic, -1, format!("m{i}").as_bytes()) + .expect("append"); + } + assert_eq!(store.backlog_entries(topic, "sub"), Some(10)); + + // Ack entries 0..3 under Shared semantics. + for entry in 0..4u64 { + let id = pulsar_lite_storage_managed_ledger::MessageId { + ledger: store + .get_last_position(topic) + .expect("last position") + .map(|p| p.ledger_id) + .unwrap_or_default(), + entry, + partition: -1, + }; + store + .ack_message_shared(topic, "sub", id) + .expect("ack shared"); + } + assert_eq!( + store.backlog_entries(topic, "sub"), + Some(6), + "4 acked of 10 must leave backlog 6" + ); + } + + // Reopen: backlog must survive restart. + { + let store = open(dir.path()); + assert_eq!(store.backlog_entries(topic, "sub"), Some(6)); + } +} diff --git a/rust/storage/managed-ledger/src/legacy_storage.rs b/rust/storage/managed-ledger/src/legacy_storage.rs index bf5c6be..522b796 100644 --- a/rust/storage/managed-ledger/src/legacy_storage.rs +++ b/rust/storage/managed-ledger/src/legacy_storage.rs @@ -143,6 +143,19 @@ pub trait ManagedLedgerStorage: Send + Sync { subscription: &str, message_id: &MessageId, ) -> bool; - fn get_mark_delete_position(&self, topic: &str, subscription: &str) -> Option; + + /// Number of stored entries not yet acknowledged by `subscription` + /// (ledger-aware across rollovers). `None` when the topic/cursor is + /// unknown to the backend. + fn backlog_entries(&self, topic: &str, subscription: &str) -> Option { + let _ = (topic, subscription); + None + } + + /// Bytes durably stored for `topic` (sum of ledger sizes). + fn stored_bytes(&self, topic: &str) -> u64 { + let _ = topic; + 0 + } } diff --git a/rust/storage/managed-ledger/src/memory.rs b/rust/storage/managed-ledger/src/memory.rs index 5c7453d..9758200 100644 --- a/rust/storage/managed-ledger/src/memory.rs +++ b/rust/storage/managed-ledger/src/memory.rs @@ -306,11 +306,41 @@ impl ManagedLedgerStorage for InMemoryManagedLedgerStorage { let cursor_key = format!("{}:{}", topic, subscription); is_message_acknowledged(self.subscription_cursors.get(&cursor_key), message_id.entry) } - fn get_mark_delete_position(&self, topic: &str, subscription: &str) -> Option { let cursor_key = format!("{}:{}", topic, subscription); self.subscription_cursors.get(&cursor_key)?.mark_delete } + + fn backlog_entries(&self, topic: &str, subscription: &str) -> Option { + let cursor_key = format!("{}:{}", topic, subscription); + let total = self.messages(topic).len() as u64; + + // Shared subscription: frontier plus individual holes. + if let Some(cursor) = self.subscription_cursors.get(&cursor_key) { + let acked = cursor.mark_delete.map_or(0, |mark| mark + 1); + return Some(total.saturating_sub(acked)); + } + + // Exclusive/failover: plain u64 frontier; u64::MAX = nothing acked. + match self.cursors.get(&cursor_key).copied() { + Some(mark) if mark != u64::MAX => Some(total.saturating_sub(mark + 1)), + _ => Some(total), + } + } + + fn stored_bytes(&self, topic: &str) -> u64 { + self.messages(topic) + .iter() + .map(|(message_id, payload)| { + let metadata = self + .entry_metadata + .get(message_id) + .map(|value| value.len()) + .unwrap_or(0); + metadata + payload.len() + }) + .sum::() as u64 + } } #[derive(Debug, Default)] From c4d13376d97ddd6f13548d4b6cd9be847746c498 Mon Sep 17 00:00:00 2001 From: XTLine <2668543450@qq.com> Date: Wed, 19 Aug 2026 13:16:34 +0800 Subject: [PATCH 37/44] feat(broker): wire Prometheus metrics across broker layers - pre-resolved TopicMetrics/SubscriptionMetrics handles on entities; label resolution happens once at creation, never per message - persistent publishes counted at write-queue batch commit, non-persistent in the fan-out worker, in-process at Producer::publish_message - consumer dispatch/ack hooks cache the handle via OnceLock + try_read so the dispatch path never awaits the subscription lock - 5s scrape aggregation task sets gauges (entity counts, backlog, unacked, storage size, windowed rates) entirely under try_read/try_lock with no broker lock held across awaits - [metrics] config section, GET /metrics listener in main, connection and rejection counters; dispatch-metrics log demoted to debug --- rust/pulsar-lite.toml | 9 + rust/src/broker/broker_service.rs | 2 - rust/src/broker/connection_limiter.rs | 9 + rust/src/broker/dispatcher/mod.rs | 1 + rust/src/broker/dispatcher/shared.rs | 2 +- rust/src/broker/mod.rs | 2 +- rust/src/broker/service/consumer.rs | 41 ++- rust/src/broker/service/producer.rs | 12 +- rust/src/broker/service/server_cnx.rs | 22 +- rust/src/broker/service/topic/subscription.rs | 10 +- rust/src/broker/service/topic/topic.rs | 35 +- rust/src/broker/stats/metrics.rs | 253 -------------- rust/src/broker/stats/mod.rs | 9 +- rust/src/broker/stats/scrape.rs | 309 ++++++++++++++++++ rust/src/config.rs | 80 +++++ rust/src/main.rs | 48 +++ 16 files changed, 573 insertions(+), 271 deletions(-) delete mode 100644 rust/src/broker/stats/metrics.rs create mode 100644 rust/src/broker/stats/scrape.rs diff --git a/rust/pulsar-lite.toml b/rust/pulsar-lite.toml index eddce3f..1dde7e4 100644 --- a/rust/pulsar-lite.toml +++ b/rust/pulsar-lite.toml @@ -26,3 +26,12 @@ connection_liveness_check_timeout_secs = 10 # Connection limits (0 = unlimited) max_connections = 0 max_connections_per_ip = 0 + +# Prometheus metrics endpoint (GET /metrics) +[metrics] +enabled = true +addr = "0.0.0.0:8080" +# Value of the `cluster` label on every exported family +cluster = "pulsar-lite" +# Window (seconds) for derived rate gauges (pulsar_rate_in etc.) +rate_window_secs = 60 diff --git a/rust/src/broker/broker_service.rs b/rust/src/broker/broker_service.rs index 474075f..b0d8633 100644 --- a/rust/src/broker/broker_service.rs +++ b/rust/src/broker/broker_service.rs @@ -169,8 +169,6 @@ impl BrokerService { self.topics.len() } - // ==================== Partitioned Topic Management ==================== - /// Get or create a partitioned topic with the specified number of partitions /// /// This will create the partitioned topic if it doesn't exist diff --git a/rust/src/broker/connection_limiter.rs b/rust/src/broker/connection_limiter.rs index 93b1066..70966cc 100644 --- a/rust/src/broker/connection_limiter.rs +++ b/rust/src/broker/connection_limiter.rs @@ -63,6 +63,15 @@ impl ConnectionLimiter { }) } + /// Current number of tracked connections (for the + /// `pulsar_active_connections` gauge). + pub fn active_connections(&self) -> usize { + match self.inner.lock() { + Ok(state) => state.total_connections, + Err(poisoned) => poisoned.into_inner().total_connections, + } + } + fn release(&self, ip: IpAddr) { let mut state = self.inner.lock().unwrap_or_else(|e| e.into_inner()); diff --git a/rust/src/broker/dispatcher/mod.rs b/rust/src/broker/dispatcher/mod.rs index 1b7c38d..640aba5 100644 --- a/rust/src/broker/dispatcher/mod.rs +++ b/rust/src/broker/dispatcher/mod.rs @@ -10,6 +10,7 @@ mod key_shared; mod read_position; pub mod redelivery_controller; mod shared; +pub(crate) use shared::DEFAULT_MAX_UNACKED_MESSAGES_PER_CONSUMER; mod single_active; pub(crate) mod sticky_key; mod traits; diff --git a/rust/src/broker/dispatcher/shared.rs b/rust/src/broker/dispatcher/shared.rs index 92c8fb2..2694f00 100644 --- a/rust/src/broker/dispatcher/shared.rs +++ b/rust/src/broker/dispatcher/shared.rs @@ -56,7 +56,7 @@ pub struct SharedDispatcher { } /// Aligns with Apache Pulsar broker.conf maxUnackedMessagesPerConsumer default. -const DEFAULT_MAX_UNACKED_MESSAGES_PER_CONSUMER: usize = 50_000; +pub(crate) const DEFAULT_MAX_UNACKED_MESSAGES_PER_CONSUMER: usize = 50_000; impl SharedDispatcher { /// Create a new SharedDispatcher diff --git a/rust/src/broker/mod.rs b/rust/src/broker/mod.rs index 81c9183..715a49d 100644 --- a/rust/src/broker/mod.rs +++ b/rust/src/broker/mod.rs @@ -20,4 +20,4 @@ pub use service::topic::{ }; pub use service::SharedStorage; pub use service::{handle_connection, ConnectionState}; -pub use stats::{BrokerMetrics, SharedMetrics}; +pub use stats::{get, init, BrokerMetrics}; diff --git a/rust/src/broker/service/consumer.rs b/rust/src/broker/service/consumer.rs index fb257c9..9dc06f9 100644 --- a/rust/src/broker/service/consumer.rs +++ b/rust/src/broker/service/consumer.rs @@ -137,6 +137,11 @@ pub struct Consumer { /// Failover active-consumer view, updated by dispatcher notifications. active_consumer_id: AtomicI64, is_active_consumer: AtomicBool, + /// Pre-resolved subscription metrics. Resolved lazily via `try_read` + /// (never `.read().await`): the dispatch path may run while the + /// subscription lock is contended, and awaiting a read here can queue + /// behind a pending writer and deadlock the connection. + sub_metrics: std::sync::OnceLock>, } impl std::fmt::Debug for Consumer { @@ -200,6 +205,7 @@ impl Consumer { key_shared_policy, active_consumer_id: AtomicI64::new(-1), is_active_consumer: AtomicBool::new(false), + sub_metrics: std::sync::OnceLock::new(), } } @@ -261,17 +267,42 @@ impl Consumer { self.available_permits.load(Ordering::Relaxed) } + /// Returns the subscription metrics handle, resolving it once via a + /// non-blocking `try_read`. `None` only while the subscription lock is + /// continuously write-held; the next call retries. + fn subscription_metrics(&self) -> Option<&Arc> { + if let Some(metrics) = self.sub_metrics.get() { + return Some(metrics); + } + let guard = self.subscription.try_read().ok()?; + let _ = self.sub_metrics.set(Arc::clone(&guard.metrics)); + drop(guard); + self.sub_metrics.get() + } + /// Record message dispatched to this consumer pub async fn record_message_dispatched(&self, message_size: usize) { - let mut stats = self.stats.write().await; - stats.messages_received += 1; - stats.bytes_received += message_size as u64; + { + let mut stats = self.stats.write().await; + stats.messages_received += 1; + stats.bytes_received += message_size as u64; + } + // Fold into subscription-level counters via the cached handle; + // never awaits the subscription lock from the dispatch path. + if let Some(metrics) = self.subscription_metrics() { + metrics.record_dispatched(message_size as u64); + } } /// Record message acknowledged pub async fn record_message_acked(&self) { - let mut stats = self.stats.write().await; - stats.messages_acked += 1; + { + let mut stats = self.stats.write().await; + stats.messages_acked += 1; + } + if let Some(metrics) = self.subscription_metrics() { + metrics.record_acked(); + } } /// Get current statistics diff --git a/rust/src/broker/service/producer.rs b/rust/src/broker/service/producer.rs index 0d4f911..d1f43ab 100644 --- a/rust/src/broker/service/producer.rs +++ b/rust/src/broker/service/producer.rs @@ -99,12 +99,15 @@ impl Producer { // - Producer state validation (is_closed) // Record statistics before publishing + let published_bytes = + (metadata.as_ref().map(|value| value.len()).unwrap_or(0) + payload.len()) as u64; self.record_message_sent(payload.len()).await; // Persistent path: keep topic write lock off the storage IO wait so multiple // producers can enqueue into the write queue concurrently. // Non-persistent path still needs &mut Topic for in-memory publish/dispatch. let topic_name; + let topic_metrics; let message_id = { let is_persistent = { let topic = self.topic.read().await; @@ -112,25 +115,32 @@ impl Producer { }; if is_persistent { - let (name, partition, persistent_runtime) = { + let (name, partition, persistent_runtime, metrics) = { let topic = self.topic.read().await; topic.validate_publish_rate_public(metadata.as_ref(), payload.len())?; ( topic.name.clone(), topic.partition, topic.persistent_runtime_handle(), + topic.metrics.clone(), ) }; topic_name = name.clone(); + topic_metrics = metrics; persistent_runtime .publish_message(&name, partition, metadata, payload) .await? } else { let mut topic = self.topic.write().await; topic_name = topic.name.clone(); + topic_metrics = topic.metrics.clone(); topic.publish_message(metadata, payload).await? } }; + // This path serves the memory backend and in-process callers; the + // RocksDB connection path counts durable batches at the write-queue + // worker instead, so there is no double counting. + topic_metrics.record_publish(1, published_bytes); log::debug!( "Producer {} published message {}:{} to topic '{}'", diff --git a/rust/src/broker/service/server_cnx.rs b/rust/src/broker/service/server_cnx.rs index e8c6163..af796c0 100644 --- a/rust/src/broker/service/server_cnx.rs +++ b/rust/src/broker/service/server_cnx.rs @@ -870,6 +870,9 @@ where .unwrap_or(0) + frame.payload.len(); if message_size > self.max_message_size { + crate::broker::stats::get() + .error_counter("message_too_large") + .inc(); self.framed .send(ServerCommand::SendError { producer_id, @@ -914,6 +917,9 @@ where Ok(()) => Ok(()), Err(error) => { if let Some(rate_error) = error.downcast_ref::() { + crate::broker::stats::get() + .error_counter("non_persistent_rate_exceeded") + .inc(); self.framed .send(ServerCommand::SendError { producer_id, @@ -954,12 +960,16 @@ where let meta_len = frame.metadata.as_ref().map(|m| m.len()).unwrap_or(0); let payload_len = frame.payload.len(); - let (topic_name, partition) = { + let (topic_name, partition, topic_metrics) = { let topic_guard = topic.read().await; if let Err(error) = topic_guard.validate_publish_rate_public(frame.metadata.as_ref(), payload_len) { if let Some(rate_error) = error.downcast_ref::() { + topic_guard.metrics.record_rate_limit_reject(); + crate::broker::stats::get() + .error_counter("publish_rate_exceeded") + .inc(); self.framed .send(ServerCommand::SendError { producer_id, @@ -973,7 +983,11 @@ where } return Err(to_cnx_error(error)); } - (topic_guard.name.clone(), topic_guard.partition) + ( + topic_guard.name.clone(), + topic_guard.partition, + topic_guard.metrics.clone(), + ) }; // Write-queue handle was cloned once at connection setup; the hot @@ -988,6 +1002,7 @@ where frame.payload.as_ref(), producer_id, sequence_id, + Some(topic_metrics), self.conn_append_tx.clone(), ) .map_err(to_cnx_error)?; @@ -1034,6 +1049,9 @@ where } self.maybe_resume_read_after_send(); if let Some(rate_error) = error.downcast_ref::() { + crate::broker::stats::get() + .error_counter("memory_backend_rate_exceeded") + .inc(); self.framed .send(ServerCommand::SendError { producer_id, diff --git a/rust/src/broker/service/topic/subscription.rs b/rust/src/broker/service/topic/subscription.rs index 72c4698..22c6003 100644 --- a/rust/src/broker/service/topic/subscription.rs +++ b/rust/src/broker/service/topic/subscription.rs @@ -91,6 +91,8 @@ pub struct Subscription { persistent_runtime: Option, /// Storage backend for reading messages storage: SharedStorage, + /// Pre-resolved Prometheus handles (label lookup happens once here). + pub(crate) metrics: Arc, } impl std::fmt::Debug for Subscription { @@ -173,6 +175,9 @@ impl Subscription { None }; + let metrics = Arc::new(crate::broker::stats::SubscriptionMetrics::new( + &topic, &name, + )); Self { name, topic, @@ -183,6 +188,7 @@ impl Subscription { non_persistent_runtime: None, persistent_runtime, storage, + metrics, } } @@ -595,6 +601,7 @@ impl Subscription { } if let Some(runtime) = self.persistent_runtime.as_mut() { + self.metrics.record_redelivered(dispatchable.len() as u64); runtime.redeliver_messages(dispatchable).await; } @@ -769,7 +776,7 @@ impl Subscription { let dispatched = runtime.dispatched_messages(); let dropped = runtime.dropped_messages(); if recv > 0 && recv % 100_000 < 50 { - log::info!( + log::debug!( "[dispatch-metrics] sub='{}' received={} dispatched={} dropped={} drop_rate={:.1}%", self.name, recv, dispatched, dropped, if recv > 0 { dropped as f64 / recv as f64 * 100.0 } else { 0.0 } @@ -795,6 +802,7 @@ impl Subscription { if let Some(runtime) = self.non_persistent_runtime.as_ref() { runtime.record_drop(count); } + self.metrics.record_dropped_n(count); } } diff --git a/rust/src/broker/service/topic/topic.rs b/rust/src/broker/service/topic/topic.rs index 2ac29f8..9dc2322 100644 --- a/rust/src/broker/service/topic/topic.rs +++ b/rust/src/broker/service/topic/topic.rs @@ -227,6 +227,9 @@ pub struct Topic { /// Storage backend storage: SharedStorage, publish_rate_limiter: TopicPublishRateLimiter, + /// Pre-resolved Prometheus handles for this topic (label lookup happens + /// once here; publish paths only touch atomic counters). + pub(crate) metrics: Arc, /// Ordered fan-out queue for non-persistent publishes (lock-free hot path). /// Mirrors the persistent write-queue shape: connection tasks enqueue under /// a short read lock and return immediately; a single worker drains FIFO, @@ -260,6 +263,7 @@ impl Topic { runtime_mode ); let persistent_runtime = PersistentTopicRuntime::new(storage.clone()); + let metrics = Arc::new(crate::broker::stats::TopicMetrics::new(&name)); let mut topic = Self { name, partition, @@ -269,6 +273,7 @@ impl Topic { persistent_runtime, storage, publish_rate_limiter: TopicPublishRateLimiter::new(), + metrics, non_persistent_fanout_tx: None, }; if topic.runtime_mode == TopicRuntimeMode::NonPersistent { @@ -346,6 +351,7 @@ impl Topic { .saturating_add(payload_len as u64); if !self.publish_rate_limiter.allow_publish(1, total_bytes) { + self.metrics.record_rate_limit_reject(); return Err(Box::new(TopicPublishRateExceeded { topic_name: self.name.clone(), })); @@ -417,13 +423,20 @@ impl Topic { metadata: Option, payload: Bytes, ) -> Result> { - let (publish, fanout_tx) = { + let accepted_bytes = + (metadata.as_ref().map(|m| m.len()).unwrap_or(0) + payload.len()) as u64; + let (publish, fanout_tx, metrics) = { let topic_guard = topic.read().await; let publish = topic_guard.prepare_non_persistent_publish(metadata, payload)?; - (publish, topic_guard.non_persistent_fanout_tx.clone()) + ( + publish, + topic_guard.non_persistent_fanout_tx.clone(), + topic_guard.metrics.clone(), + ) }; let message_id = publish.message_id(); Self::fan_out_non_persistent(publish, fanout_tx).await?; + metrics.record_publish(1, accepted_bytes); Ok(message_id) } @@ -608,6 +621,24 @@ impl Topic { self.subscriptions.len() } + /// Lock-free consumer count for the metrics aggregation loop. + /// + /// Uses `try_read` on each subscription: under churn a busy + /// subscription is skipped, which is acceptable for a gauge sampled + /// every few seconds. + pub fn total_consumer_count_snapshot(&self) -> i64 { + self.subscriptions + .values() + .filter_map(|subscription| subscription.try_read().ok()) + .map(|guard| guard.get_consumer_count() as i64) + .sum() + } + + /// Snapshot of subscriptions for the metrics aggregation loop. + pub fn get_all_subscriptions(&self) -> Vec { + self.subscriptions.values().cloned().collect() + } + /// Check if subscription exists pub fn has_subscription(&self, subscription_name: &str) -> bool { self.subscriptions.contains_key(subscription_name) diff --git a/rust/src/broker/stats/metrics.rs b/rust/src/broker/stats/metrics.rs deleted file mode 100644 index 0de6858..0000000 --- a/rust/src/broker/stats/metrics.rs +++ /dev/null @@ -1,253 +0,0 @@ -/* - * Broker Metrics - * Provides metrics collection and reporting for Pulsar Lite broker - */ - -use std::sync::atomic::{AtomicU64, Ordering}; -use std::sync::Arc; -use std::time::Instant; - -/// Broker metrics container -#[derive(Debug)] -pub struct BrokerMetrics { - // Connection metrics - pub total_connections: AtomicU64, - pub active_connections: AtomicU64, - - // Producer metrics - pub total_producers: AtomicU64, - pub messages_published: AtomicU64, - pub bytes_published: AtomicU64, - - // Consumer metrics - pub total_consumers: AtomicU64, - pub messages_delivered: AtomicU64, - pub bytes_delivered: AtomicU64, - pub messages_acked: AtomicU64, - - // Topic metrics - pub total_topics: AtomicU64, - pub total_subscriptions: AtomicU64, - - // Error metrics - pub errors: AtomicU64, - - // Performance metrics - pub start_time: Instant, -} - -impl Default for BrokerMetrics { - fn default() -> Self { - Self { - total_connections: AtomicU64::new(0), - active_connections: AtomicU64::new(0), - total_producers: AtomicU64::new(0), - messages_published: AtomicU64::new(0), - bytes_published: AtomicU64::new(0), - total_consumers: AtomicU64::new(0), - messages_delivered: AtomicU64::new(0), - bytes_delivered: AtomicU64::new(0), - messages_acked: AtomicU64::new(0), - total_topics: AtomicU64::new(0), - total_subscriptions: AtomicU64::new(0), - errors: AtomicU64::new(0), - start_time: Instant::now(), - } - } -} - -impl BrokerMetrics { - /// Create a new metrics instance - pub fn new() -> Self { - Self::default() - } - - /// Increment connection count - pub fn inc_connections(&self) { - self.total_connections.fetch_add(1, Ordering::Relaxed); - self.active_connections.fetch_add(1, Ordering::Relaxed); - } - - /// Decrement active connection count - pub fn dec_active_connections(&self) { - self.active_connections.fetch_sub(1, Ordering::Relaxed); - } - - /// Increment producer count - pub fn inc_producers(&self) { - self.total_producers.fetch_add(1, Ordering::Relaxed); - } - - /// Decrement producer count - pub fn dec_producers(&self) { - self.total_producers.fetch_sub(1, Ordering::Relaxed); - } - - /// Record a published message - pub fn record_message_published(&self, size: usize) { - self.messages_published.fetch_add(1, Ordering::Relaxed); - self.bytes_published - .fetch_add(size as u64, Ordering::Relaxed); - } - - /// Increment consumer count - pub fn inc_consumers(&self) { - self.total_consumers.fetch_add(1, Ordering::Relaxed); - } - - /// Decrement consumer count - pub fn dec_consumers(&self) { - self.total_consumers.fetch_sub(1, Ordering::Relaxed); - } - - /// Record a delivered message - pub fn record_message_delivered(&self, size: usize) { - self.messages_delivered.fetch_add(1, Ordering::Relaxed); - self.bytes_delivered - .fetch_add(size as u64, Ordering::Relaxed); - } - - /// Record an acknowledged message - pub fn record_message_acked(&self) { - self.messages_acked.fetch_add(1, Ordering::Relaxed); - } - - /// Increment topic count - pub fn inc_topics(&self) { - self.total_topics.fetch_add(1, Ordering::Relaxed); - } - - /// Increment subscription count - pub fn inc_subscriptions(&self) { - self.total_subscriptions.fetch_add(1, Ordering::Relaxed); - } - - /// Record an error - pub fn record_error(&self) { - self.errors.fetch_add(1, Ordering::Relaxed); - } - - /// Get uptime in seconds - pub fn uptime_secs(&self) -> u64 { - self.start_time.elapsed().as_secs() - } - - /// Get messages published per second (since start) - pub fn messages_published_rate(&self) -> f64 { - let uptime = self.uptime_secs(); - if uptime == 0 { - return 0.0; - } - let total = self.messages_published.load(Ordering::Relaxed); - total as f64 / uptime as f64 - } - - /// Get messages delivered per second (since start) - pub fn messages_delivered_rate(&self) -> f64 { - let uptime = self.uptime_secs(); - if uptime == 0 { - return 0.0; - } - let total = self.messages_delivered.load(Ordering::Relaxed); - total as f64 / uptime as f64 - } - - /// Format metrics as a human-readable string - pub fn to_string(&self) -> String { - format!( - "Broker Metrics:\n\ - ===============\n\ - Uptime: {}s\n\ - \n\ - Connections:\n\ - - Total: {}\n\ - - Active: {}\n\ - \n\ - Producers:\n\ - - Total: {}\n\ - - Messages Published: {} ({:.2}/s)\n\ - - Bytes Published: {} ({:.2} MB)\n\ - \n\ - Consumers:\n\ - - Total: {}\n\ - - Messages Delivered: {} ({:.2}/s)\n\ - - Bytes Delivered: {} ({:.2} MB)\n\ - - Messages Acked: {}\n\ - \n\ - Topics:\n\ - - Total: {}\n\ - - Subscriptions: {}\n\ - \n\ - Errors: {}", - self.uptime_secs(), - self.total_connections.load(Ordering::Relaxed), - self.active_connections.load(Ordering::Relaxed), - self.total_producers.load(Ordering::Relaxed), - self.messages_published.load(Ordering::Relaxed), - self.messages_published_rate(), - self.bytes_published.load(Ordering::Relaxed), - self.bytes_published.load(Ordering::Relaxed) as f64 / 1_048_576.0, - self.total_consumers.load(Ordering::Relaxed), - self.messages_delivered.load(Ordering::Relaxed), - self.messages_delivered_rate(), - self.bytes_delivered.load(Ordering::Relaxed), - self.bytes_delivered.load(Ordering::Relaxed) as f64 / 1_048_576.0, - self.messages_acked.load(Ordering::Relaxed), - self.total_topics.load(Ordering::Relaxed), - self.total_subscriptions.load(Ordering::Relaxed), - self.errors.load(Ordering::Relaxed), - ) - } -} - -/// Shared metrics instance -pub type SharedMetrics = Arc; - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn test_metrics_basic() { - let metrics = BrokerMetrics::new(); - - metrics.inc_connections(); - metrics.inc_producers(); - metrics.record_message_published(100); - metrics.inc_consumers(); - metrics.record_message_delivered(100); - metrics.record_message_acked(); - - assert_eq!(metrics.total_connections.load(Ordering::Relaxed), 1); - assert_eq!(metrics.active_connections.load(Ordering::Relaxed), 1); - assert_eq!(metrics.total_producers.load(Ordering::Relaxed), 1); - assert_eq!(metrics.messages_published.load(Ordering::Relaxed), 1); - assert_eq!(metrics.bytes_published.load(Ordering::Relaxed), 100); - assert_eq!(metrics.total_consumers.load(Ordering::Relaxed), 1); - assert_eq!(metrics.messages_delivered.load(Ordering::Relaxed), 1); - assert_eq!(metrics.bytes_delivered.load(Ordering::Relaxed), 100); - assert_eq!(metrics.messages_acked.load(Ordering::Relaxed), 1); - - // Test decrement - metrics.dec_active_connections(); - assert_eq!(metrics.active_connections.load(Ordering::Relaxed), 0); - } - - #[test] - fn test_metrics_rates() { - let metrics = BrokerMetrics::new(); - - // Initially zero - assert_eq!(metrics.messages_published_rate(), 0.0); - assert_eq!(metrics.messages_delivered_rate(), 0.0); - - // Record some messages - for _ in 0..100 { - metrics.record_message_published(50); - } - - // Rate should be calculated based on uptime - let rate = metrics.messages_published_rate(); - assert!(rate >= 0.0); - } -} diff --git a/rust/src/broker/stats/mod.rs b/rust/src/broker/stats/mod.rs index 4e219c0..e379032 100644 --- a/rust/src/broker/stats/mod.rs +++ b/rust/src/broker/stats/mod.rs @@ -1,8 +1,11 @@ /* * Broker Stats Module - * Provides metrics collection and monitoring + * Scrape-time aggregation plus re-exports of the metric handles defined in + * the `pulsar-lite-metrics` crate. */ -mod metrics; +pub mod scrape; -pub use metrics::{BrokerMetrics, SharedMetrics}; +pub use pulsar_lite_metrics::{ + get, init, parse_topic_labels, BrokerMetrics, SubscriptionMetrics, TopicLabels, TopicMetrics, +}; diff --git a/rust/src/broker/stats/scrape.rs b/rust/src/broker/stats/scrape.rs new file mode 100644 index 0000000..26dcfd1 --- /dev/null +++ b/rust/src/broker/stats/scrape.rs @@ -0,0 +1,309 @@ +/* + * Scrape-time aggregation task + * + * A 5s background task that walks broker state and sets the gauge half of + * the exported metrics: entity counts, backlog, unacked state, and + * scrape-derived rates. Monotonic counters are maintained on hot paths + * and never touched here. + * + * Lock discipline: broker/topic/subscription locks are acquired with + * `try_read`; a busy lock skips that entity for this round (gauges keep + * their previous value) so aggregation can never block the + * publish/dispatch paths. The storage lock is `try_lock` for the same + * reason. + */ + +use std::sync::Arc; +use std::time::Duration; + +use tokio::sync::{Mutex, RwLock}; +use tokio::time::{interval, MissedTickBehavior}; + +use crate::broker::connection_limiter::ConnectionLimiter; +use crate::broker::dispatcher::DEFAULT_MAX_UNACKED_MESSAGES_PER_CONSUMER; +use crate::broker::service::topic::{SubscriptionRuntimeMode, Topic}; +use crate::broker::service::Consumer; +use crate::broker::stats; +use crate::broker::BrokerService; + +use pulsar_lite_storage::Storage; + +/// Aggregation cadence; rate windows are computed from samples, not from +/// this interval, so changing one does not change the other. +const AGGREGATION_INTERVAL: Duration = Duration::from_secs(5); + +type SharedBrokerService = Arc>; +type SharedStorage = Arc>; + +/// Spawns the aggregation loop. Owns clones of broker-wide shared handles; +/// aborts with the process. +pub fn spawn( + broker_service: SharedBrokerService, + storage: SharedStorage, + connection_limiter: ConnectionLimiter, + rate_window_secs: u64, +) { + tokio::spawn(async move { + let mut ticker = interval(AGGREGATION_INTERVAL); + ticker.set_missed_tick_behavior(MissedTickBehavior::Skip); + // First tick fires immediately, which publishes an initial zero + // snapshot before any traffic arrives. + loop { + ticker.tick().await; + aggregate( + &broker_service, + &storage, + &connection_limiter, + rate_window_secs, + ) + .await; + } + }); +} + +async fn aggregate( + broker_service: &SharedBrokerService, + storage: &SharedStorage, + connection_limiter: &ConnectionLimiter, + rate_window_secs: u64, +) { + let metrics = stats::get(); + metrics + .active_connections + .set(connection_limiter.active_connections() as i64); + + let broker = match broker_service.try_read() { + Ok(guard) => guard, + Err(_) => return, + }; + + let mut totals = WalkTotals::default(); + + for topic in broker.get_all_topics().values() { + walk_topic(topic, storage, rate_window_secs, &mut totals).await; + } + for partitioned in broker.get_all_partitioned_topics().values() { + let guard = match partitioned.try_read() { + Ok(guard) => guard, + Err(_) => continue, + }; + for partition in guard.get_all_partitions() { + walk_topic(partition, storage, rate_window_secs, &mut totals).await; + } + } + + metrics.broker_topics_count.set(totals.topics); + metrics.broker_subscriptions_count.set(totals.subscriptions); + metrics.broker_producers_count.set(totals.producers); + metrics.broker_consumers_count.set(totals.consumers); + metrics.broker_msg_backlog.set(totals.backlog); + metrics.broker_storage_size.set(totals.stored_bytes as i64); + let window_secs = rate_window_secs.max(1) as f64; + metrics + .broker_rate_in + .set(totals.window_messages as f64 / window_secs); + metrics + .broker_throughput_in + .set(totals.window_bytes as f64 / window_secs); + metrics + .broker_rate_out + .set(totals.window_out_messages as f64 / window_secs); + metrics + .broker_throughput_out + .set(totals.window_out_bytes as f64 / window_secs); +} + +#[derive(Default)] +struct WalkTotals { + topics: i64, + subscriptions: i64, + producers: i64, + consumers: i64, + backlog: i64, + stored_bytes: u64, + window_messages: u64, + window_bytes: u64, + window_out_messages: u64, + window_out_bytes: u64, +} + +async fn walk_topic( + topic: &Arc>, + storage: &SharedStorage, + rate_window_secs: u64, + totals: &mut WalkTotals, +) { + // Phase 1 (holding short-lived try_read guards, fully synchronous): + // snapshot everything needed, then drop the guards before any await so + // aggregation never extends broker lock hold times. + let snapshot = match topic.try_read() { + Ok(guard) => { + let producers = guard.get_producer_count() as i64; + let consumers = guard.total_consumer_count_snapshot(); + let subscriptions = guard.get_subscription_count() as i64; + let (d_msgs, d_bytes) = guard.metrics.update_rates(rate_window_secs); + guard + .metrics + .set_entity_counts(subscriptions, producers, consumers); + let topic_metrics = Arc::clone(&guard.metrics); + let topic_name = guard.name.clone(); + let subscriptions: Vec = guard + .get_all_subscriptions() + .into_iter() + .filter_map(|subscription| { + let sub_guard = subscription.try_read().ok()?; + Some(SubscriptionSnapshot { + metrics: Arc::clone(&sub_guard.metrics), + name: sub_guard.name.clone(), + persistent: sub_guard.runtime_mode() == SubscriptionRuntimeMode::Persistent, + consumers: sub_guard.get_consumers(), + }) + }) + .collect(); + let subscription_count = subscriptions.len() as i64; + ( + topic_metrics, + topic_name, + subscriptions, + producers, + consumers, + subscription_count, + d_msgs, + d_bytes, + ) + } + Err(_) => return, + }; + let ( + topic_metrics, + topic_name, + subscriptions, + producers, + consumers, + subscription_count, + d_msgs, + d_bytes, + ) = snapshot; + + totals.topics += 1; + totals.subscriptions += subscription_count; + totals.producers += producers; + totals.consumers += consumers; + totals.window_messages += d_msgs; + totals.window_bytes += d_bytes; + + // Phase 2 (no broker locks held): storage queries and per-consumer + // awaits. Storage uses try_lock per query so a busy backend skips the + // round instead of queueing behind publish paths. + let stored_bytes = match storage.try_lock() { + Ok(storage_guard) => storage_guard.stored_bytes(&topic_name), + Err(_) => 0, + }; + topic_metrics.set_storage_size(stored_bytes as i64); + totals.stored_bytes += stored_bytes; + + for subscription in subscriptions { + let mut unacked: i64 = 0; + let mut blocked = false; + for consumer in &subscription.consumers { + let pending = consumer.pending_ack_count().await as i64; + unacked += pending; + if pending >= DEFAULT_MAX_UNACKED_MESSAGES_PER_CONSUMER as i64 { + blocked = true; + } + } + + let backlog = if subscription.persistent { + match storage.try_lock() { + Ok(storage_guard) => storage_guard + .backlog_entries(&topic_name, &subscription.name) + .unwrap_or(0) as i64, + Err(_) => 0, + } + } else { + 0 + }; + + let (d_out, d_out_bytes) = subscription.metrics.update_rates(rate_window_secs); + subscription.metrics.set_state( + backlog, + unacked, + blocked, + subscription.consumers.len() as i64, + ); + + totals.backlog += backlog; + totals.window_out_messages += d_out; + totals.window_out_bytes += d_out_bytes; + } +} + +/// Synchronous per-subscription snapshot taken under the subscription's +/// try_read guard. +struct SubscriptionSnapshot { + metrics: Arc, + name: String, + persistent: bool, + consumers: Vec>, +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::broker::service::topic::{Subscription, SubscriptionType, Topic}; + use pulsar_lite_storage::Storage; + + #[cfg(feature = "rocksdb-storage")] + #[tokio::test] + async fn backlog_reflects_partial_acks_through_broker_subscription() { + let dir = tempfile::tempdir().unwrap(); + let storage: SharedStorage = Arc::new(Mutex::new( + Storage::new_rocksdb(dir.path()).expect("open rocksdb"), + )); + let topic: Arc> = Arc::new(RwLock::new(Topic::new( + "persistent://public/default/scrape-backlog".to_string(), + storage.clone(), + ))); + + { + let mut guard = topic.write().await; + for i in 0..10u64 { + guard + .publish_message(None, bytes::Bytes::from(format!("m{i}"))) + .await + .unwrap(); + } + } + + let subscription: Arc> = Arc::new(RwLock::new(Subscription::new( + "sub".to_string(), + "persistent://public/default/scrape-backlog".to_string(), + SubscriptionType::Exclusive, + storage.clone(), + ))); + + // Ack the first four entries through the broker ack path. + { + let mut sub = subscription.write().await; + let ids: Vec<_> = (0..4u64) + .map(|entry| pulsar_lite_storage_managed_ledger::MessageId { + ledger: 0, + entry, + partition: -1, + }) + .collect(); + sub.acknowledge_message( + &ids, + crate::broker::service::topic::AckCommandType::Individual, + ) + .await + .unwrap(); + } + + let direct = { + let guard = storage.lock().await; + guard.backlog_entries("persistent://public/default/scrape-backlog", "sub") + }; + assert_eq!(direct, Some(6), "storage-level backlog must be 6"); + } +} diff --git a/rust/src/config.rs b/rust/src/config.rs index e8fdd23..33b0f7d 100644 --- a/rust/src/config.rs +++ b/rust/src/config.rs @@ -88,6 +88,60 @@ pub struct Config { /// Mirrors Netty's WRITE_BUFFER_LOW_WATER_MARK hysteresis semantics. #[serde(default = "default_pulsar_channel_write_buffer_low_water_mark_bytes")] pub pulsar_channel_write_buffer_low_water_mark_bytes: usize, + + /// Prometheus metrics endpoint configuration. + #[serde(default)] + pub metrics: MetricsConfig, +} + +/// Prometheus `/metrics` endpoint settings. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct MetricsConfig { + /// Serve GET /metrics when true. When false the broker neither binds + /// the metrics port nor runs the scrape aggregation task. + #[serde(default = "default_metrics_enabled")] + pub enabled: bool, + + /// Listen address for the Prometheus scrape endpoint (same 8080 model + /// as the native Pulsar web service port). + #[serde(default = "default_metrics_addr")] + pub addr: String, + + /// Value of the `cluster` label attached to every exported family. + #[serde(default = "default_metrics_cluster")] + pub cluster: String, + + /// Window (seconds) for scrape-derived rate gauges such as + /// pulsar_rate_in, mirroring the native stats period. + #[serde(default = "default_metrics_rate_window_secs")] + pub rate_window_secs: u64, +} + +impl Default for MetricsConfig { + fn default() -> Self { + Self { + enabled: default_metrics_enabled(), + addr: default_metrics_addr(), + cluster: default_metrics_cluster(), + rate_window_secs: default_metrics_rate_window_secs(), + } + } +} + +fn default_metrics_enabled() -> bool { + true +} + +fn default_metrics_addr() -> String { + "0.0.0.0:8080".to_string() +} + +fn default_metrics_cluster() -> String { + "pulsar-lite".to_string() +} + +fn default_metrics_rate_window_secs() -> u64 { + 60 } fn default_addr() -> String { @@ -164,6 +218,7 @@ impl Default for Config { default_pulsar_channel_write_buffer_high_water_mark_bytes(), pulsar_channel_write_buffer_low_water_mark_bytes: default_pulsar_channel_write_buffer_low_water_mark_bytes(), + metrics: MetricsConfig::default(), } } } @@ -243,6 +298,29 @@ mod tests { config.pulsar_channel_write_buffer_low_water_mark_bytes, 32 * 1024 ); + assert!(config.metrics.enabled); + assert_eq!(config.metrics.addr, "0.0.0.0:8080"); + assert_eq!(config.metrics.cluster, "pulsar-lite"); + assert_eq!(config.metrics.rate_window_secs, 60); + } + + #[test] + fn test_metrics_section_overrides_defaults() { + let parsed: Config = toml::from_str( + r#" +[metrics] +enabled = false +addr = "127.0.0.1:9109" +cluster = "bench" +rate_window_secs = 15 +"#, + ) + .unwrap(); + + assert!(!parsed.metrics.enabled); + assert_eq!(parsed.metrics.addr, "127.0.0.1:9109"); + assert_eq!(parsed.metrics.cluster, "bench"); + assert_eq!(parsed.metrics.rate_window_secs, 15); } #[test] @@ -291,6 +369,7 @@ managed_ledger_store = "rocksdb" publish_rate_bytes_per_sec: 456, pulsar_channel_write_buffer_high_water_mark_bytes: 96 * 1024, pulsar_channel_write_buffer_low_water_mark_bytes: 48 * 1024, + metrics: MetricsConfig::default(), }; let toml_str = toml::to_string(&config).unwrap(); @@ -359,6 +438,7 @@ managed_ledger_store = "rocksdb" publish_rate_bytes_per_sec: 456, pulsar_channel_write_buffer_high_water_mark_bytes: 96 * 1024, pulsar_channel_write_buffer_low_water_mark_bytes: 48 * 1024, + metrics: MetricsConfig::default(), }; let overridden = config.with_cli_overrides( diff --git a/rust/src/main.rs b/rust/src/main.rs index 7a317a3..b0ac8fb 100644 --- a/rust/src/main.rs +++ b/rust/src/main.rs @@ -7,6 +7,7 @@ use clap::Parser; use futures::SinkExt; use pulsar_lite::broker::handle_connection; use pulsar_lite::broker::service::topic::TopicPublishRate; +use pulsar_lite::broker::stats; use pulsar_lite::broker::{BrokerService, ConnectionLimiter}; use pulsar_lite::config::Config; use pulsar_lite_proto::codec::PulsarFrameCodec; @@ -91,6 +92,41 @@ async fn main() -> Result<(), Box> { config.publish_rate_bytes_per_sec ); + // Prometheus metrics: families always exist (pre-resolved handles keep + // hot paths branch-free); `enabled` only controls serving and scraping. + stats::init(&config.metrics.cluster); + if config.metrics.enabled { + match config.metrics.addr.parse::() { + Ok(metrics_addr) => { + let registry = pulsar_lite_metrics::global_registry(); + tokio::spawn(async move { + // Runs until process exit; serves GET /metrics from the + // shared registry via prometheus-hyper. + if let Err(error) = prometheus_hyper::Server::run( + registry, + metrics_addr, + std::future::pending::<()>(), + ) + .await + { + log::error!("Metrics server terminated: {}", error); + } + }); + log::info!( + "Metrics endpoint listening on http://{}/metrics", + metrics_addr + ); + } + Err(error) => { + log::warn!( + "Invalid metrics address '{}': {} — metrics disabled", + config.metrics.addr, + error + ); + } + } + } + // Initialize storage let storage = Arc::new(Mutex::new(Storage::new(&config.db_path)?)); let restored_partition_metadata = { @@ -119,6 +155,15 @@ async fn main() -> Result<(), Box> { let connection_limiter = ConnectionLimiter::new(config.max_connections, config.max_connections_per_ip); + // Scrape-time aggregation (entity counts, derived rates, active gauge). + if config.metrics.enabled { + stats::scrape::spawn( + Arc::clone(&broker_service), + Arc::clone(&storage), + connection_limiter.clone(), + config.metrics.rate_window_secs, + ); + } loop { let ret = listener.accept().await; let (socket, peer_addr) = match ret { @@ -142,6 +187,7 @@ async fn main() -> Result<(), Box> { Ok(permit) => permit, Err(error) => { log::warn!("Rejecting connection from {}: {}", peer_addr, error); + stats::get().error_counter("connection_limit").inc(); let mut framed = Framed::new(socket, PulsarFrameCodec::new()); let _ = framed .send(ServerCommand::Error { @@ -152,6 +198,7 @@ async fn main() -> Result<(), Box> { continue; } }; + stats::get().connection_created.inc(); let storage = Arc::clone(&storage); let broker_service = Arc::clone(&broker_service); @@ -178,6 +225,7 @@ async fn main() -> Result<(), Box> { log::error!("Connection error from {}: {}", peer_addr, e); } log::info!("Connection closed from {}", peer_addr); + stats::get().connection_closed.inc(); }); } } From 8e61aa3ca2400a8583d5ad97ec5510e4d59f9fbd Mon Sep 17 00:00:00 2001 From: XTLine <2668543450@qq.com> Date: Wed, 19 Aug 2026 13:16:34 +0800 Subject: [PATCH 38/44] feat(observability): add Grafana dashboards and compose stack Prometheus + Grafana with provisioning: topics dashboard (rates, backlog, unacked gate, latency/entry-size percentiles) and broker dashboard (connections, rejections, write-queue batches, process RSS/CPU) --- grafana/README.md | 50 ++++ grafana/dashboards/pulsar-lite-broker.json | 223 +++++++++++++++ grafana/dashboards/pulsar-lite-topics.json | 253 ++++++++++++++++++ grafana/docker-compose.yml | 31 +++ .../provisioning/dashboards/provider.yml | 13 + .../provisioning/datasources/prometheus.yml | 10 + grafana/prometheus/prometheus.yml | 12 + 7 files changed, 592 insertions(+) create mode 100644 grafana/README.md create mode 100644 grafana/dashboards/pulsar-lite-broker.json create mode 100644 grafana/dashboards/pulsar-lite-topics.json create mode 100644 grafana/docker-compose.yml create mode 100644 grafana/grafana/provisioning/dashboards/provider.yml create mode 100644 grafana/grafana/provisioning/datasources/prometheus.yml create mode 100644 grafana/prometheus/prometheus.yml diff --git a/grafana/README.md b/grafana/README.md new file mode 100644 index 0000000..20b1daa --- /dev/null +++ b/grafana/README.md @@ -0,0 +1,50 @@ +# Pulsar Lite Observability Stack + +A one-command Prometheus + Grafana stack that scrapes the broker's +`GET /metrics` endpoint (default `0.0.0.0:8080`). + +## Quick start + +```bash +# 1. Start the broker (metrics are on by default) +../rust/target/release/pulsar-lite --config ../rust/pulsar-lite.toml + +# 2. Start the observability stack +docker compose up -d + +# 3. Open +# Grafana http://localhost:3000 (admin/admin; anonymous read enabled) +# Prometheus http://localhost:9090/targets — the pulsar-lite job must be UP +``` + +Dashboards are provisioned automatically (folder `Pulsar Lite`): + +- **Pulsar Lite / Topics** — publish/deliver rates and throughput, entity + counts, subscription backlog, unacked gate state, redelivery, storage + size, end-to-end and ledger write latency (P50/P99), entry-size + distribution. +- **Pulsar Lite / Broker** — broker-level rates (counter `rate()` and + window-gauge views), connections and rejection reasons, backlog and + storage totals, write-queue batch sizes, process RSS/CPU. + +## Metric naming conventions + +- `pulsar_*` families reproduce native Apache Pulsar names and label sets + verbatim, so official dashboards and PromQL translate directly. +- `pulsar_lite_*` families are extensions with no native counterpart + (error reasons, redelivery counters, write-queue batch metrics). +- Histograms use the standard Prometheus shape (`_bucket{le=...}` + + `_sum` + `_count`); query percentiles with `histogram_quantile()`. + +## Broker configuration + +```toml +[metrics] +enabled = true # false: no listener, no scrape aggregation +addr = "0.0.0.0:8080" # /metrics path; must be reachable by the scraper +cluster = "pulsar-lite" # cluster label value on every family +rate_window_secs = 60 # window for pulsar_rate_in-style gauges +``` + +Remote broker: change the target in `prometheus/prometheus.yml` from +`host.docker.internal:8080` to your broker address. diff --git a/grafana/dashboards/pulsar-lite-broker.json b/grafana/dashboards/pulsar-lite-broker.json new file mode 100644 index 0000000..79dc8d8 --- /dev/null +++ b/grafana/dashboards/pulsar-lite-broker.json @@ -0,0 +1,223 @@ +{ + "uid": "pulsar-lite-broker", + "title": "Pulsar Lite / Broker", + "tags": ["pulsar-lite"], + "timezone": "browser", + "schemaVersion": 39, + "version": 1, + "editable": true, + "refresh": "10s", + "time": { "from": "now-30m", "to": "now" }, + "templating": { + "list": [ + { + "name": "cluster", + "label": "Cluster", + "type": "query", + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "query": "label_values(pulsar_broker_rate_in, cluster)", + "refresh": 2, + "includeAll": false, + "multi": false, + "current": { "text": "pulsar-lite", "value": "pulsar-lite" } + } + ] + }, + "panels": [ + { + "id": 1, + "type": "timeseries", + "title": "Broker message rate (in / out)", + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 0 }, + "targets": [ + { + "expr": "pulsar_broker_rate_in{cluster=\"$cluster\"}", + "legendFormat": "in" + }, + { + "expr": "pulsar_broker_rate_out{cluster=\"$cluster\"}", + "legendFormat": "out" + } + ] + }, + { + "id": 2, + "type": "timeseries", + "title": "Broker throughput (bytes/s)", + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 0 }, + "targets": [ + { + "expr": "pulsar_broker_throughput_in{cluster=\"$cluster\"}", + "legendFormat": "in" + }, + { + "expr": "pulsar_broker_throughput_out{cluster=\"$cluster\"}", + "legendFormat": "out" + } + ] + }, + { + "id": 3, + "type": "timeseries", + "title": "Accepted / delivered totals (rate over counters)", + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 8 }, + "targets": [ + { + "expr": "rate(pulsar_broker_in_messages_total{cluster=\"$cluster\"}[1m])", + "legendFormat": "published msg/s" + }, + { + "expr": "rate(pulsar_broker_out_messages_total{cluster=\"$cluster\"}[1m])", + "legendFormat": "delivered msg/s" + }, + { + "expr": "rate(pulsar_broker_in_bytes_total{cluster=\"$cluster\"}[1m])", + "legendFormat": "published B/s" + }, + { + "expr": "rate(pulsar_broker_out_bytes_total{cluster=\"$cluster\"}[1m])", + "legendFormat": "delivered B/s" + } + ] + }, + { + "id": 4, + "type": "timeseries", + "title": "Entity counts", + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "gridPos": { "h": 8, "w": 6, "x": 12, "y": 8 }, + "targets": [ + { + "expr": "pulsar_broker_topics_count{cluster=\"$cluster\"}", + "legendFormat": "topics" + }, + { + "expr": "pulsar_broker_subscriptions_count{cluster=\"$cluster\"}", + "legendFormat": "subscriptions" + }, + { + "expr": "pulsar_broker_producers_count{cluster=\"$cluster\"}", + "legendFormat": "producers" + }, + { + "expr": "pulsar_broker_consumers_count{cluster=\"$cluster\"}", + "legendFormat": "consumers" + } + ] + }, + { + "id": 5, + "type": "timeseries", + "title": "Connections", + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "gridPos": { "h": 8, "w": 6, "x": 18, "y": 8 }, + "targets": [ + { + "expr": "pulsar_active_connections{cluster=\"$cluster\"}", + "legendFormat": "active" + }, + { + "expr": "rate(pulsar_connection_created_total_count{cluster=\"$cluster\"}[5m])", + "legendFormat": "created /s" + }, + { + "expr": "rate(pulsar_connection_closed_total_count{cluster=\"$cluster\"}[5m])", + "legendFormat": "closed /s" + } + ] + }, + { + "id": 6, + "type": "timeseries", + "title": "Rejections by reason (/s)", + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 16 }, + "targets": [ + { + "expr": "sum by (reason) (rate(pulsar_lite_broker_errors_total{cluster=\"$cluster\"}[5m]))", + "legendFormat": "{{reason}}" + } + ] + }, + { + "id": 7, + "type": "timeseries", + "title": "Backlog + storage size", + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "gridPos": { "h": 8, "w": 6, "x": 12, "y": 16 }, + "targets": [ + { + "expr": "pulsar_broker_msg_backlog{cluster=\"$cluster\"}", + "legendFormat": "backlog entries" + }, + { + "expr": "pulsar_broker_storage_size{cluster=\"$cluster\"}", + "legendFormat": "stored bytes" + } + ] + }, + { + "id": 8, + "type": "timeseries", + "title": "Publish rate-limit rejections (/s)", + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "gridPos": { "h": 8, "w": 6, "x": 18, "y": 16 }, + "targets": [ + { + "expr": "sum by (topic) (rate(pulsar_publish_rate_limit_times{cluster=\"$cluster\"}[5m]))", + "legendFormat": "{{topic}}" + } + ] + }, + { + "id": 9, + "type": "timeseries", + "title": "Write-queue batch size (avg / P99)", + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "gridPos": { "h": 8, "w": 8, "x": 0, "y": 24 }, + "targets": [ + { + "expr": "rate(pulsar_lite_write_queue_batch_messages_total[1m]) / rate(pulsar_lite_write_queue_batches_total[1m])", + "legendFormat": "avg batch" + }, + { + "expr": "histogram_quantile(0.99, sum by (le) (rate(pulsar_lite_write_queue_batch_size_bucket[5m])))", + "legendFormat": "P99 batch" + } + ] + }, + { + "id": 10, + "type": "timeseries", + "title": "Broker RSS", + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "gridPos": { "h": 8, "w": 8, "x": 8, "y": 24 }, + "fieldConfig": { + "defaults": { "unit": "bytes" }, + "overrides": [] + }, + "targets": [ + { + "expr": "process_resident_memory_bytes", + "legendFormat": "RSS" + } + ] + }, + { + "id": 11, + "type": "timeseries", + "title": "Broker CPU (seconds/s)", + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "gridPos": { "h": 8, "w": 8, "x": 16, "y": 24 }, + "targets": [ + { + "expr": "rate(process_cpu_seconds_total[1m])", + "legendFormat": "cpu" + } + ] + } + ] +} diff --git a/grafana/dashboards/pulsar-lite-topics.json b/grafana/dashboards/pulsar-lite-topics.json new file mode 100644 index 0000000..49cfa85 --- /dev/null +++ b/grafana/dashboards/pulsar-lite-topics.json @@ -0,0 +1,253 @@ +{ + "uid": "pulsar-lite-topics", + "title": "Pulsar Lite / Topics", + "tags": ["pulsar-lite"], + "timezone": "browser", + "schemaVersion": 39, + "version": 1, + "editable": true, + "refresh": "10s", + "time": { "from": "now-30m", "to": "now" }, + "templating": { + "list": [ + { + "name": "cluster", + "label": "Cluster", + "type": "query", + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "query": "label_values(pulsar_rate_in, cluster)", + "refresh": 2, + "includeAll": false, + "multi": false, + "current": { "text": "pulsar-lite", "value": "pulsar-lite" } + }, + { + "name": "namespace", + "label": "Namespace", + "type": "query", + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "query": "label_values(pulsar_rate_in{cluster=\"$cluster\"}, namespace)", + "refresh": 2, + "includeAll": true, + "multi": true, + "current": { "text": "All", "value": "$__all" } + }, + { + "name": "topic", + "label": "Topic", + "type": "query", + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "query": "label_values(pulsar_rate_in{cluster=\"$cluster\", namespace=~\"$namespace\"}, topic)", + "refresh": 2, + "includeAll": true, + "multi": true, + "current": { "text": "All", "value": "$__all" } + }, + { + "name": "subscription", + "label": "Subscription", + "type": "query", + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "query": "label_values(pulsar_subscription_msg_rate_out{cluster=\"$cluster\", namespace=~\"$namespace\", topic=~\"$topic\"}, subscription)", + "refresh": 2, + "includeAll": true, + "multi": true, + "current": { "text": "All", "value": "$__all" } + } + ] + }, + "panels": [ + { + "id": 1, + "type": "timeseries", + "title": "Publish rate (msg/s)", + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 0 }, + "targets": [ + { + "expr": "pulsar_rate_in{cluster=\"$cluster\", namespace=~\"$namespace\", topic=~\"$topic\"}", + "legendFormat": "{{namespace}}/{{topic}} [{{partition}}]" + } + ] + }, + { + "id": 2, + "type": "timeseries", + "title": "Publish throughput (bytes/s)", + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 0 }, + "targets": [ + { + "expr": "pulsar_throughput_in{cluster=\"$cluster\", namespace=~\"$namespace\", topic=~\"$topic\"}", + "legendFormat": "{{namespace}}/{{topic}} [{{partition}}]" + } + ] + }, + { + "id": 3, + "type": "timeseries", + "title": "Delivery rate (msg/s)", + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 8 }, + "targets": [ + { + "expr": "pulsar_subscription_msg_rate_out{cluster=\"$cluster\", namespace=~\"$namespace\", topic=~\"$topic\", subscription=~\"$subscription\"}", + "legendFormat": "{{topic}}/{{subscription}}" + } + ] + }, + { + "id": 4, + "type": "timeseries", + "title": "Delivery throughput (bytes/s)", + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 8 }, + "targets": [ + { + "expr": "pulsar_subscription_msg_throughput_out{cluster=\"$cluster\", namespace=~\"$namespace\", topic=~\"$topic\", subscription=~\"$subscription\"}", + "legendFormat": "{{topic}}/{{subscription}}" + } + ] + }, + { + "id": 5, + "type": "timeseries", + "title": "Producers / Consumers / Subscriptions", + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "gridPos": { "h": 8, "w": 8, "x": 0, "y": 16 }, + "targets": [ + { + "expr": "sum(pulsar_producers_count{cluster=\"$cluster\", namespace=~\"$namespace\", topic=~\"$topic\"})", + "legendFormat": "producers" + }, + { + "expr": "sum(pulsar_consumers_count{cluster=\"$cluster\", namespace=~\"$namespace\", topic=~\"$topic\"})", + "legendFormat": "consumers" + }, + { + "expr": "sum(pulsar_subscriptions_count{cluster=\"$cluster\", namespace=~\"$namespace\", topic=~\"$topic\"})", + "legendFormat": "subscriptions" + } + ] + }, + { + "id": 6, + "type": "timeseries", + "title": "Subscription backlog", + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "gridPos": { "h": 8, "w": 8, "x": 8, "y": 16 }, + "targets": [ + { + "expr": "pulsar_subscription_back_log{cluster=\"$cluster\", namespace=~\"$namespace\", topic=~\"$topic\", subscription=~\"$subscription\"}", + "legendFormat": "{{topic}}/{{subscription}}" + } + ] + }, + { + "id": 7, + "type": "timeseries", + "title": "Unacked messages (dispatched, not acked)", + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "gridPos": { "h": 8, "w": 8, "x": 16, "y": 16 }, + "targets": [ + { + "expr": "pulsar_subscription_unacked_messages{cluster=\"$cluster\", namespace=~\"$namespace\", topic=~\"$topic\", subscription=~\"$subscription\"}", + "legendFormat": "{{topic}}/{{subscription}}" + }, + { + "expr": "pulsar_subscription_blocked_on_unacked_messages{cluster=\"$cluster\", namespace=~\"$namespace\", topic=~\"$topic\", subscription=~\"$subscription\"}", + "legendFormat": "{{topic}}/{{subscription}} blocked" + } + ] + }, + { + "id": 8, + "type": "timeseries", + "title": "Storage size (bytes)", + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 24 }, + "targets": [ + { + "expr": "sum(pulsar_storage_size{cluster=\"$cluster\", namespace=~\"$namespace\", topic=~\"$topic\"})", + "legendFormat": "stored bytes" + } + ] + }, + { + "id": 9, + "type": "timeseries", + "title": "Redelivery rate (msg/s)", + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 24 }, + "targets": [ + { + "expr": "pulsar_subscription_msg_rate_redeliver{cluster=\"$cluster\", namespace=~\"$namespace\", topic=~\"$topic\", subscription=~\"$subscription\"}", + "legendFormat": "{{topic}}/{{subscription}}" + } + ] + }, + { + "id": 10, + "type": "timeseries", + "title": "Durable publish latency (e2e, P50/P99)", + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 32 }, + "fieldConfig": { + "defaults": { "unit": "s" }, + "overrides": [] + }, + "targets": [ + { + "expr": "histogram_quantile(0.5, sum by (le) (rate(pulsar_storage_write_latency_bucket[1m])))", + "legendFormat": "P50" + }, + { + "expr": "histogram_quantile(0.99, sum by (le) (rate(pulsar_storage_write_latency_bucket[1m])))", + "legendFormat": "P99" + } + ] + }, + { + "id": 11, + "type": "timeseries", + "title": "Ledger append latency (P50/P99)", + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "gridPos": { "h": 8, "w": 6, "x": 12, "y": 32 }, + "fieldConfig": { + "defaults": { "unit": "s" }, + "overrides": [] + }, + "targets": [ + { + "expr": "histogram_quantile(0.5, sum by (le) (rate(pulsar_storage_ledger_write_latency_bucket[1m])))", + "legendFormat": "P50" + }, + { + "expr": "histogram_quantile(0.99, sum by (le) (rate(pulsar_storage_ledger_write_latency_bucket[1m])))", + "legendFormat": "P99" + } + ] + }, + { + "id": 12, + "type": "timeseries", + "title": "Entry size distribution (P50/P99/max)", + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "gridPos": { "h": 8, "w": 6, "x": 18, "y": 32 }, + "fieldConfig": { + "defaults": { "unit": "bytes" }, + "overrides": [] + }, + "targets": [ + { + "expr": "histogram_quantile(0.5, sum by (le) (rate(pulsar_entry_size_bucket[5m])))", + "legendFormat": "P50" + }, + { + "expr": "histogram_quantile(0.99, sum by (le) (rate(pulsar_entry_size_bucket[5m])))", + "legendFormat": "P99" + } + ] + } + ] +} diff --git a/grafana/docker-compose.yml b/grafana/docker-compose.yml new file mode 100644 index 0000000..33f6856 --- /dev/null +++ b/grafana/docker-compose.yml @@ -0,0 +1,31 @@ +services: + prometheus: + image: prom/prometheus:v3.1.0 + container_name: pulsar-lite-prometheus + command: + - --config.file=/etc/prometheus/prometheus.yml + - --storage.tsdb.retention.time=7d + volumes: + - ./prometheus:/etc/prometheus:ro + ports: + - "9090:9090" + extra_hosts: + - "host.docker.internal:host-gateway" + restart: unless-stopped + + grafana: + image: grafana/grafana:11.5.2 + container_name: pulsar-lite-grafana + environment: + - GF_SECURITY_ADMIN_USER=admin + - GF_SECURITY_ADMIN_PASSWORD=admin + - GF_AUTH_ANONYMOUS_ENABLED=true + - GF_AUTH_ANONYMOUS_ORG_ROLE=Viewer + volumes: + - ./grafana/provisioning:/etc/grafana/provisioning:ro + - ./dashboards:/var/lib/grafana/dashboards:ro + ports: + - "3000:3000" + depends_on: + - prometheus + restart: unless-stopped diff --git a/grafana/grafana/provisioning/dashboards/provider.yml b/grafana/grafana/provisioning/dashboards/provider.yml new file mode 100644 index 0000000..2aea31a --- /dev/null +++ b/grafana/grafana/provisioning/dashboards/provider.yml @@ -0,0 +1,13 @@ +apiVersion: 1 + +providers: + - name: pulsar-lite + orgId: 1 + folder: Pulsar Lite + type: file + disableDeletion: false + updateIntervalSeconds: 30 + allowUiUpdates: true + options: + path: /var/lib/grafana/dashboards + foldersFromFilesStructure: false diff --git a/grafana/grafana/provisioning/datasources/prometheus.yml b/grafana/grafana/provisioning/datasources/prometheus.yml new file mode 100644 index 0000000..00f9915 --- /dev/null +++ b/grafana/grafana/provisioning/datasources/prometheus.yml @@ -0,0 +1,10 @@ +apiVersion: 1 + +datasources: + - name: Prometheus + uid: prometheus + type: prometheus + access: proxy + url: http://prometheus:9090 + isDefault: true + editable: false diff --git a/grafana/prometheus/prometheus.yml b/grafana/prometheus/prometheus.yml new file mode 100644 index 0000000..6695745 --- /dev/null +++ b/grafana/prometheus/prometheus.yml @@ -0,0 +1,12 @@ +global: + scrape_interval: 5s + evaluation_interval: 5s + +scrape_configs: + - job_name: pulsar-lite + metrics_path: /metrics + static_configs: + # The broker's metrics endpoint (default 0.0.0.0:8080, same model as + # native Pulsar's web service port). host.docker.internal resolves to + # the host via the compose extra_hosts entry. + - targets: ["host.docker.internal:8080"] From bf116552958e05cd48efcd7447a1a0bcd0ec3b1d Mon Sep 17 00:00:00 2001 From: XTLine <2668543450@qq.com> Date: Wed, 19 Aug 2026 13:16:34 +0800 Subject: [PATCH 39/44] docs: add monitoring section to README --- README.md | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/README.md b/README.md index 4b869a9..3b52d50 100644 --- a/README.md +++ b/README.md @@ -39,6 +39,7 @@ local setup cost: | Non-persistent topics | Dispatch-or-drop runtime semantics with coverage for flow control, disconnect/reconnect, ordering, dynamic consumers, and KeyShared routing. | | Persistent topics | RocksDB-backed managed-ledger style storage is available behind the `rocksdb-storage` feature. | | Subscription modes | Shared, Failover, Exclusive, and KeyShared are covered by Rust and Python integration tests. | +| Monitoring | Prometheus metrics on `GET /metrics` (port 8080): broker/topic/subscription counters, backlog and unacked gauges, storage latency and entry-size histograms, process metrics. Optional Grafana stack under `grafana/`. | | Partitioned topics | Default partition metadata and partition topic routing are supported for local testing. | | Python package | Provides a small helper SDK that can start and manage a local broker process. | @@ -110,6 +111,39 @@ Stop the broker: ../rust/pulsar-lite.sh stop ``` +## Monitoring + +The broker exposes Prometheus metrics on `GET /metrics` (default +`0.0.0.0:8080`, same port model as native Pulsar's web service). Metric +families reuse native Pulsar names and labels (`pulsar_rate_in`, +`pulsar_subscription_back_log`, `pulsar_storage_write_latency`, ...), so +existing dashboards and PromQL translate directly; extensions unique to +Pulsar Lite use the `pulsar_lite_*` prefix. + +Quick check: + +```bash +curl -s localhost:8080/metrics | grep -E '^pulsar_(broker|subscription)' +``` + +Configuration (`rust/pulsar-lite.toml`): + +```toml +[metrics] +enabled = true # false: no listener, no scrape aggregation +addr = "0.0.0.0:8080" +cluster = "pulsar-lite" # `cluster` label value on every family +rate_window_secs = 60 # window for pulsar_rate_in-style gauges +``` + +A ready-to-run Prometheus + Grafana stack (with provisioned dashboards for +topics and broker overview) lives under [`grafana/`](grafana/README.md): + +```bash +docker compose -f grafana/docker-compose.yml up -d +# Grafana http://localhost:3000 (admin/admin), Prometheus http://localhost:9090 +``` + ## Embedded Python Usage The Python helper can start a local broker for short-lived tests or examples: From 1b9980f387580fd73d91996f4d6e60c677ffd8f7 Mon Sep 17 00:00:00 2001 From: XTLine <2668543450@qq.com> Date: Thu, 20 Aug 2026 15:31:09 +0800 Subject: [PATCH 40/44] fix(perf): give each perf broker a dedicated scrape-able metrics port _config_text rewrote every top-level `addr = ...` line, so the [metrics] addr collapsed onto the protocol port: the two listeners raced for one bind (broker exit, or a silently dead metrics endpoint), and the 127.0.0.1 binding was unreachable from the compose Prometheus via host.docker.internal anyway. - BrokerConfig.metrics_port derived as port + 1430 (6650 -> 8080, 6651 -> 8081, ... 6672 -> 8102); zero call-site churn. - _config_text splits at the [metrics] header: main addr keeps 127.0.0.1:{port}, metrics addr becomes 0.0.0.0:{metrics_port}. - compose Prometheus scrapes the six perf ports under job pulsar-lite-perf (DOWN while no run is active - expected). - grafana/README: live-view-during-perf-tests section. Docker-backed perf runs use --network host, so the same ports apply. --- grafana/README.md | 18 ++++++++++++++++++ grafana/prometheus/prometheus.yml | 15 +++++++++++++++ tests/perf/lib/broker.py | 26 ++++++++++++++++++++++++-- 3 files changed, 57 insertions(+), 2 deletions(-) diff --git a/grafana/README.md b/grafana/README.md index 20b1daa..61380d6 100644 --- a/grafana/README.md +++ b/grafana/README.md @@ -27,6 +27,24 @@ Dashboards are provisioned automatically (folder `Pulsar Lite`): window-gauge views), connections and rejection reasons, backlog and storage totals, write-queue batch sizes, process RSS/CPU. +## Live view during perf tests + +The perf harness (`tests/perf/`) starts real brokers, each with a private +metrics port derived from its protocol port (6651/6652 → 8081/8082, +6661/6662 → 8091/8092, 6671/6672 → 8101/8102, bound on `0.0.0.0`). This +stack scrapes all of them (`job="pulsar-lite-perf"`), so with the stack up +you can watch a run live: + +```bash +docker compose up -d # once +python tests/perf/run_persistent_stress.py ... # then run any scenario +# Grafana http://localhost:3000 — dashboards show the running broker's series +``` + +Perf targets show DOWN in Prometheus `/targets` while no run is active — +that is expected. Docker-backed perf runs use `--network host`, so the same +ports apply. + ## Metric naming conventions - `pulsar_*` families reproduce native Apache Pulsar names and label sets diff --git a/grafana/prometheus/prometheus.yml b/grafana/prometheus/prometheus.yml index 6695745..539de14 100644 --- a/grafana/prometheus/prometheus.yml +++ b/grafana/prometheus/prometheus.yml @@ -10,3 +10,18 @@ scrape_configs: # native Pulsar's web service port). host.docker.internal resolves to # the host via the compose extra_hosts entry. - targets: ["host.docker.internal:8080"] + + # Perf harness brokers (tests/perf/lib/broker.py derives each broker's + # metrics port from its protocol port: 6651/6652 -> 8081/8082, + # 6661/6662 -> 8091/8092, 6671/6672 -> 8101/8102, bound on 0.0.0.0). + # Targets show DOWN while no run is active — expected. + - job_name: pulsar-lite-perf + metrics_path: /metrics + static_configs: + - targets: + - "host.docker.internal:8081" + - "host.docker.internal:8082" + - "host.docker.internal:8091" + - "host.docker.internal:8092" + - "host.docker.internal:8101" + - "host.docker.internal:8102" diff --git a/tests/perf/lib/broker.py b/tests/perf/lib/broker.py index 01b4757..3eeacb7 100644 --- a/tests/perf/lib/broker.py +++ b/tests/perf/lib/broker.py @@ -21,6 +21,15 @@ class BrokerConfig: name: str port: int default_partitions: int + # Prometheus /metrics port; derived so existing call sites keep 3-arg + # construction. Offset 1430 keeps the base mapping 6650 -> 8080 (default + # web service port), giving every perf broker a private metrics port + # (6651 -> 8081, 6662 -> 8092, 6672 -> 8102, ...). + metrics_port: int | None = None + + def __post_init__(self) -> None: + if self.metrics_port is None: + self.metrics_port = self.port + 1430 class BrokerSampler(threading.Thread): @@ -97,12 +106,25 @@ def write_csv(self, csv_path: Path) -> None: def _config_text(config: BrokerConfig, db_path: str) -> str: config_text = BASE_CONFIG.read_text(encoding="utf-8") - config_text = re.sub( + # The base config has two `addr` lines: the protocol listener at the top + # and the metrics endpoint inside the [metrics] section. Rewrite them + # separately so they never collapse onto the same port (both listeners + # racing for one bind), and keep metrics on 0.0.0.0 so the compose + # Prometheus can scrape it through host.docker.internal. + head, sep, metrics_section = config_text.partition("[metrics]") + head = re.sub( r'^addr\s*=\s*".*"$', f'addr = "127.0.0.1:{config.port}"', - config_text, + head, + flags=re.MULTILINE, + ) + metrics_section = re.sub( + r'^addr\s*=\s*".*"$', + f'addr = "0.0.0.0:{config.metrics_port}"', + metrics_section, flags=re.MULTILINE, ) + config_text = head + sep + metrics_section config_text = re.sub( r'^db_path\s*=\s*".*"$', f'db_path = "{db_path}"', From e542f0697d64391ad3bb3dc0949439d4826c696f Mon Sep 17 00:00:00 2001 From: XTLine <2668543450@qq.com> Date: Fri, 21 Aug 2026 13:32:50 +0800 Subject: [PATCH 41/44] feat(broker): batch-aware message accounting across dispatch and commit paths Count client-visible messages (MessageMetadata.num_messages_in_batch, default 1) instead of entries everywhere: consumer stats, subscription metrics, non-persistent dispatcher dispatched/dropped counters, write-queue committed counters, and flow-control permits (try_reserve_dispatch now takes a permits argument; non-persistent paths deduct batch_count like the persistent shared/key_shared dispatchers already did). messages_in_batch moves to pulsar-lite-proto::codec so the storage layer can reuse it (added as an extra dep alongside managed-ledger in managed-ledger-rocksdb, whose types the crate still uses). --- rust/Cargo.lock | 1 + rust/metrics/src/subscription.rs | 12 +++---- rust/proto/src/codec.rs | 13 ++++++++ rust/src/broker/dispatcher/exclusive.rs | 4 ++- rust/src/broker/dispatcher/failover.rs | 4 ++- rust/src/broker/dispatcher/key_shared.rs | 4 +-- rust/src/broker/dispatcher/mod.rs | 18 ++--------- rust/src/broker/dispatcher/shared.rs | 4 +-- .../dispatcher/multiple_consumers.rs | 24 ++++++++++---- .../dispatcher/single_active.rs | 32 +++++++++++++------ .../non_persistent/dispatcher/sticky_key.rs | 18 +++++++---- rust/src/broker/service/consumer.rs | 27 ++++++++++------ .../storage/managed-ledger-rocksdb/Cargo.toml | 1 + .../managed-ledger-rocksdb/src/write_queue.rs | 7 +++- 14 files changed, 109 insertions(+), 60 deletions(-) diff --git a/rust/Cargo.lock b/rust/Cargo.lock index 838cedb..192c82c 100644 --- a/rust/Cargo.lock +++ b/rust/Cargo.lock @@ -1070,6 +1070,7 @@ dependencies = [ "prost", "prost-build", "pulsar-lite-metrics", + "pulsar-lite-proto", "pulsar-lite-storage-managed-ledger", "rocksdb", "serde", diff --git a/rust/metrics/src/subscription.rs b/rust/metrics/src/subscription.rs index 50e00a6..257f43c 100644 --- a/rust/metrics/src/subscription.rs +++ b/rust/metrics/src/subscription.rs @@ -84,11 +84,11 @@ impl SubscriptionMetrics { } } - /// Records a message dispatched to a consumer of this subscription. - pub fn record_dispatched(&self, bytes: u64) { - self.out_messages.inc(); + /// Records dispatched messages (batch-aware count) and their bytes. + pub fn record_dispatched(&self, messages: u64, bytes: u64) { + self.out_messages.inc_by(messages); self.out_bytes.inc_by(bytes); - self.broker.broker_out_messages.inc(); + self.broker.broker_out_messages.inc_by(messages); self.broker.broker_out_bytes.inc_by(bytes); self.last_consumed_timestamp.set(unix_epoch_seconds()); } @@ -197,8 +197,8 @@ mod tests { fn dispatched_acked_and_liveness_tick() { let metrics = SubscriptionMetrics::new("persistent://public/subns/sub-topic", "sub-unit-test"); - metrics.record_dispatched(120); - metrics.record_dispatched(30); + metrics.record_dispatched(1, 120); + metrics.record_dispatched(1, 30); metrics.record_acked(); metrics.record_redelivered(2); metrics.record_dropped(); diff --git a/rust/proto/src/codec.rs b/rust/proto/src/codec.rs index bdd0ec8..42be572 100644 --- a/rust/proto/src/codec.rs +++ b/rust/proto/src/codec.rs @@ -335,6 +335,19 @@ impl PulsarFrameCodec { } } +/// Number of client-visible messages carried by one entry's metadata. +/// Client-visible semantics (Apache Pulsar): a batch entry of N messages +/// counts as N messages for permit, rate, and counter accounting. Entries +/// without batch metadata (or undecodable metadata) count as one message. +pub fn messages_in_batch(metadata: &[u8]) -> u32 { + proto::pulsar::MessageMetadata::decode(metadata) + .ok() + .and_then(|m| m.num_messages_in_batch) + .filter(|n| *n > 0) + .map(|n| n as u32) + .unwrap_or(1) +} + #[cfg(test)] mod tests { use super::*; diff --git a/rust/src/broker/dispatcher/exclusive.rs b/rust/src/broker/dispatcher/exclusive.rs index e9ae4e1..2ea7dd1 100644 --- a/rust/src/broker/dispatcher/exclusive.rs +++ b/rust/src/broker/dispatcher/exclusive.rs @@ -138,6 +138,8 @@ impl Dispatcher for ExclusiveDispatcher { .await?; if let Some(candidate) = candidate { + let batch_count = + crate::broker::dispatcher::messages_in_batch(&candidate.metadata); if consumer .enqueue_message( candidate.message_id, @@ -148,7 +150,7 @@ impl Dispatcher for ExclusiveDispatcher { { commit_read_position(&self.read_position, candidate.next_position); consumer - .record_message_dispatched(candidate.payload.len()) + .record_message_dispatched(batch_count, candidate.payload.len()) .await; dispatched += 1; } else { diff --git a/rust/src/broker/dispatcher/failover.rs b/rust/src/broker/dispatcher/failover.rs index a96dd61..86c7930 100644 --- a/rust/src/broker/dispatcher/failover.rs +++ b/rust/src/broker/dispatcher/failover.rs @@ -274,6 +274,8 @@ impl Dispatcher for FailoverDispatcher { .await?; if let Some(candidate) = candidate { + let batch_count = + crate::broker::dispatcher::messages_in_batch(&candidate.metadata); if primary_consumer .enqueue_message( candidate.message_id, @@ -284,7 +286,7 @@ impl Dispatcher for FailoverDispatcher { { commit_read_position(&self.read_position, candidate.next_position); primary_consumer - .record_message_dispatched(candidate.payload.len()) + .record_message_dispatched(batch_count, candidate.payload.len()) .await; dispatched += 1; } else { diff --git a/rust/src/broker/dispatcher/key_shared.rs b/rust/src/broker/dispatcher/key_shared.rs index 32b67c1..ab402f7 100644 --- a/rust/src/broker/dispatcher/key_shared.rs +++ b/rust/src/broker/dispatcher/key_shared.rs @@ -366,7 +366,7 @@ impl KeySharedDispatcher { .await { consumer - .record_message_dispatched(entry.payload.len()) + .record_message_dispatched(batch_count, entry.payload.len()) .await; progress += 1; } else { @@ -439,7 +439,7 @@ impl KeySharedDispatcher { { commit_read_position(&self.read_position, candidate.next_position); consumer - .record_message_dispatched(candidate.payload.len()) + .record_message_dispatched(batch_count, candidate.payload.len()) .await; progress += 1; } else { diff --git a/rust/src/broker/dispatcher/mod.rs b/rust/src/broker/dispatcher/mod.rs index 640aba5..2d152f8 100644 --- a/rust/src/broker/dispatcher/mod.rs +++ b/rust/src/broker/dispatcher/mod.rs @@ -23,18 +23,6 @@ pub use shared::SharedDispatcher; pub use single_active::rewind_read_position; pub use traits::Dispatcher; -/// Number of client-visible messages carried by one entry's metadata batch. -/// Permit accounting is per client-visible message (Apache Pulsar semantics): -/// a batch entry of N messages consumes N permits. Entries without batch -/// metadata count as a single message. -pub fn messages_in_batch(metadata: &[u8]) -> u32 { - use prost::Message; - use pulsar_lite_proto::codec::proto::pulsar::MessageMetadata; - - MessageMetadata::decode(metadata) - .ok() - .and_then(|m| m.num_messages_in_batch) - .filter(|n| *n > 0) - .map(|n| n as u32) - .unwrap_or(1) -} +/// Client-visible message count per entry (batch-aware); shared with the +/// write-queue commit accounting. See `pulsar_lite_proto::codec`. +pub use pulsar_lite_proto::codec::messages_in_batch; diff --git a/rust/src/broker/dispatcher/shared.rs b/rust/src/broker/dispatcher/shared.rs index 2694f00..f135a85 100644 --- a/rust/src/broker/dispatcher/shared.rs +++ b/rust/src/broker/dispatcher/shared.rs @@ -350,7 +350,7 @@ impl SharedDispatcher { .await { consumer - .record_message_dispatched(entry.payload.len()) + .record_message_dispatched(batch_count, entry.payload.len()) .await; dispatched += 1; redelivered += 1; @@ -413,7 +413,7 @@ impl SharedDispatcher { { commit_read_position(&self.read_position, candidate.next_position); consumer - .record_message_dispatched(candidate.payload.len()) + .record_message_dispatched(batch_count, candidate.payload.len()) .await; dispatched += 1; diff --git a/rust/src/broker/non_persistent/dispatcher/multiple_consumers.rs b/rust/src/broker/non_persistent/dispatcher/multiple_consumers.rs index 6cf4d7d..d05d9fd 100644 --- a/rust/src/broker/non_persistent/dispatcher/multiple_consumers.rs +++ b/rust/src/broker/non_persistent/dispatcher/multiple_consumers.rs @@ -132,7 +132,9 @@ impl NonPersistentDispatcherMultipleConsumers { // fails `use_permit` immediately and drops below as before. let Some(start) = self.next_consumer_start_index() else { log::debug!("Dropping non-persistent shared entry due to no connected consumer"); - self.record_drop(1); + self.record_drop( + crate::broker::dispatcher::messages_in_batch(entry.metadata()) as u64, + ); entry.release(); continue; }; @@ -145,19 +147,29 @@ impl NonPersistentDispatcherMultipleConsumers { let metadata = entry.metadata_bytes(); let payload = entry.payload_bytes(); + let batch_count = + crate::broker::dispatcher::messages_in_batch(entry.metadata()); let mut delivered = false; for offset in 0..self.ordered_consumers.len() { let consumer = self.ordered_consumers[(start + offset) % self.ordered_consumers.len()].clone(); if let Some(reservation) = consumer - .try_reserve_dispatch(&message_id, metadata.clone(), payload.clone(), 0) + .try_reserve_dispatch( + &message_id, + metadata.clone(), + payload.clone(), + 0, + batch_count, + ) .await { reservation.send(); - self.record_dispatched(1); - self.subtract_total_permits(1); - consumer.record_message_dispatched(entry.len()).await; + self.record_dispatched(batch_count as u64); + self.subtract_total_permits(batch_count); + consumer + .record_message_dispatched(batch_count, entry.len()) + .await; delivered = true; break; } @@ -167,7 +179,7 @@ impl NonPersistentDispatcherMultipleConsumers { log::debug!( "Dropping non-persistent shared entry: no consumer with permits or writability" ); - self.record_drop(1); + self.record_drop(batch_count as u64); } entry.release(); } diff --git a/rust/src/broker/non_persistent/dispatcher/single_active.rs b/rust/src/broker/non_persistent/dispatcher/single_active.rs index 0ce8094..9c99657 100644 --- a/rust/src/broker/non_persistent/dispatcher/single_active.rs +++ b/rust/src/broker/non_persistent/dispatcher/single_active.rs @@ -86,7 +86,9 @@ impl NonPersistentDispatcherExclusive { .fetch_add(entries.len() as u64, Ordering::Relaxed); let Some(consumer) = &self.consumer else { for entry in entries { - self.record_drop(1); + self.record_drop( + crate::broker::dispatcher::messages_in_batch(entry.metadata()) as u64, + ); entry.release(); } return Ok(()); @@ -101,15 +103,19 @@ impl NonPersistentDispatcherExclusive { let metadata = entry.metadata_bytes(); let payload = entry.payload_bytes(); + let batch_count = + crate::broker::dispatcher::messages_in_batch(entry.metadata()); if let Some(reservation) = consumer - .try_reserve_dispatch(&message_id, metadata, payload, 0) + .try_reserve_dispatch(&message_id, metadata, payload, 0, batch_count) .await { reservation.send(); - self.record_dispatched(1); - consumer.record_message_dispatched(entry.len()).await; + self.record_dispatched(batch_count as u64); + consumer + .record_message_dispatched(batch_count, entry.len()) + .await; } else { - self.record_drop(1); + self.record_drop(batch_count as u64); } entry.release(); } @@ -357,7 +363,9 @@ impl NonPersistentDispatcherFailover { .fetch_add(entries.len() as u64, Ordering::Relaxed); let Some(active_consumer) = self.get_active_consumer() else { for entry in entries { - self.record_drop(1); + self.record_drop( + crate::broker::dispatcher::messages_in_batch(entry.metadata()) as u64, + ); entry.release(); } return Ok(()); @@ -372,15 +380,19 @@ impl NonPersistentDispatcherFailover { let metadata = entry.metadata_bytes(); let payload = entry.payload_bytes(); + let batch_count = + crate::broker::dispatcher::messages_in_batch(entry.metadata()); if let Some(reservation) = active_consumer - .try_reserve_dispatch(&message_id, metadata, payload, 0) + .try_reserve_dispatch(&message_id, metadata, payload, 0, batch_count) .await { reservation.send(); - self.record_dispatched(1); - active_consumer.record_message_dispatched(entry.len()).await; + self.record_dispatched(batch_count as u64); + active_consumer + .record_message_dispatched(batch_count, entry.len()) + .await; } else { - self.record_drop(1); + self.record_drop(batch_count as u64); } entry.release(); } diff --git a/rust/src/broker/non_persistent/dispatcher/sticky_key.rs b/rust/src/broker/non_persistent/dispatcher/sticky_key.rs index 220f1da..0e134fa 100644 --- a/rust/src/broker/non_persistent/dispatcher/sticky_key.rs +++ b/rust/src/broker/non_persistent/dispatcher/sticky_key.rs @@ -228,7 +228,9 @@ impl NonPersistentStickyKeyDispatcher { log::debug!( "Dropping non-persistent key-shared entry due to no available consumer" ); - self.record_drop(1); + self.record_drop( + crate::broker::dispatcher::messages_in_batch(entry.metadata()) as u64, + ); entry.release(); continue; }; @@ -241,16 +243,20 @@ impl NonPersistentStickyKeyDispatcher { let metadata = entry.metadata_bytes(); let payload = entry.payload_bytes(); + let batch_count = + crate::broker::dispatcher::messages_in_batch(entry.metadata()); if let Some(reservation) = consumer - .try_reserve_dispatch(&message_id, metadata, payload, 0) + .try_reserve_dispatch(&message_id, metadata, payload, 0, batch_count) .await { reservation.send(); - self.record_dispatched(1); - self.subtract_total_permits(1); - consumer.record_message_dispatched(entry.len()).await; + self.record_dispatched(batch_count as u64); + self.subtract_total_permits(batch_count); + consumer + .record_message_dispatched(batch_count, entry.len()) + .await; } else { - self.record_drop(1); + self.record_drop(batch_count as u64); } entry.release(); diff --git a/rust/src/broker/service/consumer.rs b/rust/src/broker/service/consumer.rs index 9dc06f9..76a1fbe 100644 --- a/rust/src/broker/service/consumer.rs +++ b/rust/src/broker/service/consumer.rs @@ -280,17 +280,19 @@ impl Consumer { self.sub_metrics.get() } - /// Record message dispatched to this consumer - pub async fn record_message_dispatched(&self, message_size: usize) { + /// Record entries dispatched to this consumer. `messages` is the + /// client-visible message count (batch-aware; see + /// `dispatcher::messages_in_batch`), `message_size` is entry bytes. + pub async fn record_message_dispatched(&self, messages: u32, message_size: usize) { { let mut stats = self.stats.write().await; - stats.messages_received += 1; + stats.messages_received += messages as u64; stats.bytes_received += message_size as u64; } // Fold into subscription-level counters via the cached handle; // never awaits the subscription lock from the dispatch path. if let Some(metrics) = self.subscription_metrics() { - metrics.record_dispatched(message_size as u64); + metrics.record_dispatched(messages as u64, message_size as u64); } } @@ -496,9 +498,11 @@ impl Consumer { metadata: Bytes, payload: Bytes, redelivery_count: u32, + permits: u32, ) -> Option { - // Step 1: Acquire flow-control permit - if !self.use_permit().await { + // Step 1: Acquire flow-control permits (per client-visible message; + // see `try_use_permits`). + if !self.try_use_permits(permits).await { return None; } @@ -515,7 +519,8 @@ impl Consumer { // Step 3: Connection must currently be writable. if !self.is_writable() { - self.available_permits.fetch_add(1, Ordering::Relaxed); + self.available_permits + .fetch_add(permits as i32, Ordering::Relaxed); return None; } @@ -523,7 +528,8 @@ impl Consumer { let owned_permit = match self.message_tx.clone().try_reserve_owned() { Ok(p) => p, Err(_) => { - self.available_permits.fetch_add(1, Ordering::Relaxed); + self.available_permits + .fetch_add(permits as i32, Ordering::Relaxed); return None; } }; @@ -535,7 +541,8 @@ impl Consumer { .track_message_dispatched(message_id, redelivery_count) .await { - self.available_permits.fetch_add(1, Ordering::Relaxed); + self.available_permits + .fetch_add(permits as i32, Ordering::Relaxed); // owned_permit drops here, releasing channel slot return None; } @@ -956,7 +963,7 @@ mod tests { assert_eq!(consumer.get_available_permits().await, 3); // Record messages - consumer.record_message_dispatched(100).await; + consumer.record_message_dispatched(1, 100).await; consumer.record_message_acked().await; let stats = consumer.get_stats().await; diff --git a/rust/storage/managed-ledger-rocksdb/Cargo.toml b/rust/storage/managed-ledger-rocksdb/Cargo.toml index f63bcce..2397b32 100644 --- a/rust/storage/managed-ledger-rocksdb/Cargo.toml +++ b/rust/storage/managed-ledger-rocksdb/Cargo.toml @@ -6,6 +6,7 @@ license.workspace = true [dependencies] pulsar-lite-metrics = { path = "../../metrics" } +pulsar-lite-proto = { path = "../../proto" } pulsar-lite-storage-managed-ledger = { path = "../managed-ledger" } anyhow.workspace = true rocksdb = "0.24" diff --git a/rust/storage/managed-ledger-rocksdb/src/write_queue.rs b/rust/storage/managed-ledger-rocksdb/src/write_queue.rs index 5025e9c..24021a1 100644 --- a/rust/storage/managed-ledger-rocksdb/src/write_queue.rs +++ b/rust/storage/managed-ledger-rocksdb/src/write_queue.rs @@ -212,7 +212,12 @@ impl WriteQueue { let mut observer: Option> = None; let committed_at = Instant::now(); for req in &reqs { - committed_messages += 1; + // Batched producers pack N client-visible messages + // into one request; counters must fold N, not 1 + // (bytes already fold the full payload). + committed_messages += + pulsar_lite_proto::codec::messages_in_batch(&req.metadata) + as u64; let req_bytes = (req.metadata.len() + req.payload.len()) as u64; committed_bytes += req_bytes; metrics.observe_entry_size(req_bytes as f64); From 98189d6a9d72b302e548dc02b2c432ecf2f8ee20 Mon Sep 17 00:00:00 2001 From: XTLine <2668543450@qq.com> Date: Fri, 21 Aug 2026 13:32:54 +0800 Subject: [PATCH 42/44] fix(stats): keep previous gauge value when storage lock is busy A failed try_lock during scrape meant "unknown", not zero: per-topic storage size, per-subscription backlog (now Option), and broker totals now skip the round instead of reporting a spurious dip to zero under publish-path contention. --- rust/metrics/src/subscription.rs | 18 +++++++++--- rust/src/broker/stats/scrape.rs | 49 +++++++++++++++++++++----------- 2 files changed, 46 insertions(+), 21 deletions(-) diff --git a/rust/metrics/src/subscription.rs b/rust/metrics/src/subscription.rs index 257f43c..307a5de 100644 --- a/rust/metrics/src/subscription.rs +++ b/rust/metrics/src/subscription.rs @@ -166,9 +166,19 @@ impl SubscriptionMetrics { (d_out, d_bytes) } - /// Scrape task: state gauges for this subscription. - pub fn set_state(&self, back_log: i64, unacked: i64, blocked: bool, consumers: i64) { - self.back_log.set(back_log); + /// Scrape task: state gauges for this subscription. `back_log: None` + /// means "unknown this round" (busy storage lock) and keeps the previous + /// value instead of reporting a spurious zero. + pub fn set_state( + &self, + back_log: Option, + unacked: i64, + blocked: bool, + consumers: i64, + ) { + if let Some(back_log) = back_log { + self.back_log.set(back_log); + } self.unacked_messages.set(unacked); self.blocked_on_unacked.set(blocked as i64); self.consumers_count.set(consumers); @@ -207,7 +217,7 @@ mod tests { assert_eq!(d_out, 2); assert_eq!(d_bytes, 150); - metrics.set_state(3, 1, true, 2); + metrics.set_state(Some(3), 1, true, 2); // State gauges are read through the registry in integration checks; // here we only assert the counters did not panic and rates reported. } diff --git a/rust/src/broker/stats/scrape.rs b/rust/src/broker/stats/scrape.rs index 26dcfd1..05a3105 100644 --- a/rust/src/broker/stats/scrape.rs +++ b/rust/src/broker/stats/scrape.rs @@ -96,8 +96,14 @@ async fn aggregate( metrics.broker_subscriptions_count.set(totals.subscriptions); metrics.broker_producers_count.set(totals.producers); metrics.broker_consumers_count.set(totals.consumers); - metrics.broker_msg_backlog.set(totals.backlog); - metrics.broker_storage_size.set(totals.stored_bytes as i64); + // A skipped entity means "unknown", not zero: hold the previous broker + // total instead of reporting a spurious dip under storage contention. + if !totals.backlog_skipped { + metrics.broker_msg_backlog.set(totals.backlog); + } + if !totals.storage_skipped { + metrics.broker_storage_size.set(totals.stored_bytes as i64); + } let window_secs = rate_window_secs.max(1) as f64; metrics .broker_rate_in @@ -120,7 +126,11 @@ struct WalkTotals { producers: i64, consumers: i64, backlog: i64, + /// True when any subscription's backlog read hit a busy lock this round. + backlog_skipped: bool, stored_bytes: u64, + /// True when any topic's stored-bytes read hit a busy lock this round. + storage_skipped: bool, window_messages: u64, window_bytes: u64, window_out_messages: u64, @@ -193,14 +203,17 @@ async fn walk_topic( totals.window_bytes += d_bytes; // Phase 2 (no broker locks held): storage queries and per-consumer - // awaits. Storage uses try_lock per query so a busy backend skips the - // round instead of queueing behind publish paths. - let stored_bytes = match storage.try_lock() { - Ok(storage_guard) => storage_guard.stored_bytes(&topic_name), - Err(_) => 0, - }; - topic_metrics.set_storage_size(stored_bytes as i64); - totals.stored_bytes += stored_bytes; + // awaits. Storage uses try_lock per query; a busy lock means "unknown + // this round", so gauges keep their previous value instead of dipping + // to zero under publish-path contention. + match storage.try_lock() { + Ok(storage_guard) => { + let stored_bytes = storage_guard.stored_bytes(&topic_name); + topic_metrics.set_storage_size(stored_bytes as i64); + totals.stored_bytes += stored_bytes; + } + Err(_) => totals.storage_skipped = true, + } for subscription in subscriptions { let mut unacked: i64 = 0; @@ -214,14 +227,13 @@ async fn walk_topic( } let backlog = if subscription.persistent { - match storage.try_lock() { - Ok(storage_guard) => storage_guard + storage.try_lock().ok().map(|storage_guard| { + storage_guard .backlog_entries(&topic_name, &subscription.name) - .unwrap_or(0) as i64, - Err(_) => 0, - } + .unwrap_or(0) as i64 + }) } else { - 0 + Some(0) }; let (d_out, d_out_bytes) = subscription.metrics.update_rates(rate_window_secs); @@ -232,7 +244,10 @@ async fn walk_topic( subscription.consumers.len() as i64, ); - totals.backlog += backlog; + match backlog { + Some(backlog) => totals.backlog += backlog, + None => totals.backlog_skipped = true, + } totals.window_out_messages += d_out; totals.window_out_bytes += d_out_bytes; } From 086120b9999ab01bc0a233f1ff06bd5f1ecc22c7 Mon Sep 17 00:00:00 2001 From: XTLine <2668543450@qq.com> Date: Fri, 21 Aug 2026 13:32:54 +0800 Subject: [PATCH 43/44] chore(grafana): expose Grafana on host port 7070 to avoid 3000 conflicts --- grafana/docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/grafana/docker-compose.yml b/grafana/docker-compose.yml index 33f6856..860acb4 100644 --- a/grafana/docker-compose.yml +++ b/grafana/docker-compose.yml @@ -25,7 +25,7 @@ services: - ./grafana/provisioning:/etc/grafana/provisioning:ro - ./dashboards:/var/lib/grafana/dashboards:ro ports: - - "3000:3000" + - "7070:3000" depends_on: - prometheus restart: unless-stopped From c0003ad28f0cc0e07b66267cb3584f67fb98904c Mon Sep 17 00:00:00 2001 From: XTLine <2668543450@qq.com> Date: Fri, 21 Aug 2026 16:39:23 +0800 Subject: [PATCH 44/44] style: cargo fmt and clippy 1.98 fixes (checked_div in metrics) --- rust/metrics/src/topic.rs | 7 ++----- .../broker/non_persistent/dispatcher/multiple_consumers.rs | 3 +-- rust/src/broker/non_persistent/dispatcher/single_active.rs | 6 ++---- rust/src/broker/non_persistent/dispatcher/sticky_key.rs | 3 +-- 4 files changed, 6 insertions(+), 13 deletions(-) diff --git a/rust/metrics/src/topic.rs b/rust/metrics/src/topic.rs index 1520bbf..2ae7037 100644 --- a/rust/metrics/src/topic.rs +++ b/rust/metrics/src/topic.rs @@ -171,11 +171,8 @@ impl TopicMetrics { if elapsed >= 1.0 { self.rate_in.set(d_messages as f64 / elapsed); self.throughput_in.set(d_bytes as f64 / elapsed); - self.average_msg_size.set(if d_messages > 0 { - (d_bytes / d_messages) as i64 - } else { - 0 - }); + self.average_msg_size + .set(d_bytes.checked_div(d_messages).unwrap_or(0) as i64); } (d_messages, d_bytes) } diff --git a/rust/src/broker/non_persistent/dispatcher/multiple_consumers.rs b/rust/src/broker/non_persistent/dispatcher/multiple_consumers.rs index d05d9fd..e8ab3d4 100644 --- a/rust/src/broker/non_persistent/dispatcher/multiple_consumers.rs +++ b/rust/src/broker/non_persistent/dispatcher/multiple_consumers.rs @@ -147,8 +147,7 @@ impl NonPersistentDispatcherMultipleConsumers { let metadata = entry.metadata_bytes(); let payload = entry.payload_bytes(); - let batch_count = - crate::broker::dispatcher::messages_in_batch(entry.metadata()); + let batch_count = crate::broker::dispatcher::messages_in_batch(entry.metadata()); let mut delivered = false; for offset in 0..self.ordered_consumers.len() { let consumer = diff --git a/rust/src/broker/non_persistent/dispatcher/single_active.rs b/rust/src/broker/non_persistent/dispatcher/single_active.rs index 9c99657..da3134a 100644 --- a/rust/src/broker/non_persistent/dispatcher/single_active.rs +++ b/rust/src/broker/non_persistent/dispatcher/single_active.rs @@ -103,8 +103,7 @@ impl NonPersistentDispatcherExclusive { let metadata = entry.metadata_bytes(); let payload = entry.payload_bytes(); - let batch_count = - crate::broker::dispatcher::messages_in_batch(entry.metadata()); + let batch_count = crate::broker::dispatcher::messages_in_batch(entry.metadata()); if let Some(reservation) = consumer .try_reserve_dispatch(&message_id, metadata, payload, 0, batch_count) .await @@ -380,8 +379,7 @@ impl NonPersistentDispatcherFailover { let metadata = entry.metadata_bytes(); let payload = entry.payload_bytes(); - let batch_count = - crate::broker::dispatcher::messages_in_batch(entry.metadata()); + let batch_count = crate::broker::dispatcher::messages_in_batch(entry.metadata()); if let Some(reservation) = active_consumer .try_reserve_dispatch(&message_id, metadata, payload, 0, batch_count) .await diff --git a/rust/src/broker/non_persistent/dispatcher/sticky_key.rs b/rust/src/broker/non_persistent/dispatcher/sticky_key.rs index 0e134fa..4b037fd 100644 --- a/rust/src/broker/non_persistent/dispatcher/sticky_key.rs +++ b/rust/src/broker/non_persistent/dispatcher/sticky_key.rs @@ -243,8 +243,7 @@ impl NonPersistentStickyKeyDispatcher { let metadata = entry.metadata_bytes(); let payload = entry.payload_bytes(); - let batch_count = - crate::broker::dispatcher::messages_in_batch(entry.metadata()); + let batch_count = crate::broker::dispatcher::messages_in_batch(entry.metadata()); if let Some(reservation) = consumer .try_reserve_dispatch(&message_id, metadata, payload, 0, batch_count) .await