diff --git a/fluxon_py/config.py b/fluxon_py/config.py index 414bef0..daba755 100644 --- a/fluxon_py/config.py +++ b/fluxon_py/config.py @@ -26,6 +26,7 @@ def debug_print(*args): _YAML_KEY_TYPES = (str, int, float, bool) _YAML_SCALAR_TYPES = (str, int, float, bool, type(None)) _U64_MAX = 2**64 - 1 +_U32_MAX = 2**32 - 1 _SIZE_BYTE_UNITS = { "": 1, "b": 1, @@ -195,6 +196,7 @@ def _yaml_template(): fluxonkv_spec: # fluxon kv specific config (dict(optional)) etcd_addresses: # Etcd address list ((None|['{str}:{int}'])) + etcd_rpc_max_retries: # Retry count after the first etcd RPC attempt (int(optional)) cluster_name: # Cluster name (str) share_mem_path: # Shared bundle path for mmap.file/shared.json/peer metadata (str) large_file_paths: # Owner-mode ordered large-file roots (['{str}'](optional)) @@ -535,6 +537,7 @@ def _validate_fluxonkv_contract(cfg: Dict[str, Any]) -> None: if is_zero_contribution: forbidden_spec_keys = [ "etcd_addresses", + "etcd_rpc_max_retries", "redis_compat", "sub_cluster", "large_file_paths", @@ -559,6 +562,17 @@ def _validate_fluxonkv_contract(cfg: Dict[str, Any]) -> None: if not isinstance(etcd_addresses, list) or len(etcd_addresses) == 0: raise ValueError("fluxonkv_spec.etcd_addresses must be a non-empty list") + if "etcd_rpc_max_retries" in spec: + etcd_rpc_max_retries = spec["etcd_rpc_max_retries"] + if isinstance(etcd_rpc_max_retries, bool) or not isinstance( + etcd_rpc_max_retries, int + ): + raise ValueError("fluxonkv_spec.etcd_rpc_max_retries must be an int") + if not 0 <= etcd_rpc_max_retries <= _U32_MAX: + raise ValueError( + f"fluxonkv_spec.etcd_rpc_max_retries must be in 0..={_U32_MAX}" + ) + if "sub_cluster" not in spec: raise ValueError("fluxonkv_spec.sub_cluster is required for owner mode") sub_cluster = spec.get("sub_cluster") diff --git a/fluxon_py/tests/test_config.py b/fluxon_py/tests/test_config.py index c1f924b..fb49848 100644 --- a/fluxon_py/tests/test_config.py +++ b/fluxon_py/tests/test_config.py @@ -46,6 +46,7 @@ def _build_checks(selected_test_id: Optional[str]) -> List[Tuple[str, Callable[[ ("load_from_file", _run_test_load_from_file), ("to_yaml_str_roundtrip", _run_test_to_yaml_str_roundtrip), ("fluxonkv_sub_cluster_config", test_fluxonkv_sub_cluster_config), + ("fluxonkv_etcd_rpc_max_retries", test_fluxonkv_etcd_rpc_max_retries), ("fluxonkv_owner_requires_sub_cluster", test_fluxonkv_owner_requires_sub_cluster), ("fluxonkv_owner_requires_large_file_paths", test_fluxonkv_owner_requires_large_file_paths), ("fluxonkv_large_limit_size_contract", test_fluxonkv_large_limit_size_contract), @@ -184,6 +185,46 @@ def test_fluxonkv_sub_cluster_config(): print(f"❌ FAIL: test_fluxonkv_sub_cluster_config - {e}") +def test_fluxonkv_etcd_rpc_max_retries(): + """Ensure owner retry overrides accept zero and external configs inherit the value.""" + try: + owner = _owner_fluxonkv_base_config(tag="etcd_rpc_retries") + owner["fluxonkv_spec"]["etcd_rpc_max_retries"] = 0 + loaded = yaml.safe_load( + FluxonKvClientConfig(owner).to_fluxon_kv_client_config_yaml_str() + ) + assert loaded["fluxonkv_spec"]["etcd_rpc_max_retries"] == 0 + + for invalid_value in (None, True, -1, 2**32): + invalid = _owner_fluxonkv_base_config(tag="etcd_rpc_retries_invalid") + invalid["fluxonkv_spec"]["etcd_rpc_max_retries"] = invalid_value + try: + FluxonKvClientConfig(invalid) + raise AssertionError( + f"invalid etcd_rpc_max_retries accepted: {invalid_value!r}" + ) + except ValueError: + pass + + external = { + "instance_key": "test_external", + "fluxonkv_spec": { + "cluster_name": "test_cluster", + "share_mem_path": "/tmp/kvcache_shared_memory/test", + "etcd_rpc_max_retries": 0, + }, + } + try: + FluxonKvClientConfig(external) + raise AssertionError("external etcd retry override should be rejected") + except ValueError: + pass + + print("✅ PASS: test_fluxonkv_etcd_rpc_max_retries") + except Exception as e: + print(f"❌ FAIL: test_fluxonkv_etcd_rpc_max_retries - {e}") + + def test_fluxon_pyo3_import_authority(): """Ensure the PyO3 binding is imported only from the active venv authority.""" try: diff --git a/fluxon_release/closed_sdk/lib/libfluxon_commu_core.so b/fluxon_release/closed_sdk/lib/libfluxon_commu_core.so index f6a8a1c..0203db8 100755 Binary files a/fluxon_release/closed_sdk/lib/libfluxon_commu_core.so and b/fluxon_release/closed_sdk/lib/libfluxon_commu_core.so differ diff --git a/fluxon_release/closed_sdk/lib/libfluxon_rdma_probe.so b/fluxon_release/closed_sdk/lib/libfluxon_rdma_probe.so index fcdd879..edc4595 100755 Binary files a/fluxon_release/closed_sdk/lib/libfluxon_rdma_probe.so and b/fluxon_release/closed_sdk/lib/libfluxon_rdma_probe.so differ diff --git a/fluxon_release/closed_sdk/manifest.json b/fluxon_release/closed_sdk/manifest.json index 9ebb3a1..69fd4e3 100644 --- a/fluxon_release/closed_sdk/manifest.json +++ b/fluxon_release/closed_sdk/manifest.json @@ -1,15 +1,15 @@ { "object_kind": "FluxonCommuClosedSdk", - "schema_version": 7, - "abi_version": 9, - "sdk_version": "0.2.1", - "required_open_surface_version": "0.2.1", + "schema_version": 8, + "abi_version": 10, + "sdk_version": "0.2.2", + "required_open_surface_version": "0.2.2", "feature_contract": { "boundary_mode": "closed-sdk-consumer", "runtime_authority_crate": "fluxon_commu_private_impl", "transfer_link_snapshot_mode": "watch-v1" }, - "public_workspace_input_sha256": "59c66ec1fe0dc849a3e2a9f8ce4c721cfee6a387286bd25d3d28bf6ca1705197", + "public_workspace_input_sha256": "5ddc15e2f78eb17c793ed1595f7071879ca5710c07a5058652f080a70f942e48", "layout": { "native": "native", "lib": "lib" diff --git a/fluxon_release/closed_sdk/native/native_runtime/lib/libfluxon_rdma_probe.so b/fluxon_release/closed_sdk/native/native_runtime/lib/libfluxon_rdma_probe.so index fcdd879..edc4595 100755 Binary files a/fluxon_release/closed_sdk/native/native_runtime/lib/libfluxon_rdma_probe.so and b/fluxon_release/closed_sdk/native/native_runtime/lib/libfluxon_rdma_probe.so differ diff --git a/fluxon_release/release_notes/v0.2.2.md b/fluxon_release/release_notes/v0.2.2.md index bcc32c1..4d80196 100644 --- a/fluxon_release/release_notes/v0.2.2.md +++ b/fluxon_release/release_notes/v0.2.2.md @@ -1,13 +1,13 @@ # 🚀 Fluxon v0.2.2 -`v0.2.2` rolls up the mainline work merged from July 14 through August 5, 2026, together with the new release pipeline in the tagged revision. The largest changes are a distributed SSD backing tier for Fluxon KV, hybrid S3 object writes, communication ABI 9, event-driven TCP reactors, stronger MQ and framework lifecycle handling, and expanded release-grade CI. +`v0.2.2` rolls up the mainline work merged from July 14 through August 5, 2026, together with the new release pipeline in the tagged revision. The largest changes are a distributed SSD backing tier for Fluxon KV, hybrid S3 object writes, communication ABI 10, event-driven TCP reactors, stronger MQ and framework lifecycle handling, and expanded release-grade CI. ## ✨ Highlights - Added an owner-local SSD backing tier behind the existing Fluxon KV `put` / `get` / `delete` contract. - Added size-aware S3 writes, KV-backed write sessions, shared lease keepalive, and retryable temporary-key cleanup. - Added `FluxonFsVideoReader` and a pooled reader API for cached random access to video data. -- Upgraded the closed communication boundary to ABI 9 and added event-driven TCP reactor support. +- Upgraded the closed communication boundary to ABI 10 and added event-driven TCP reactor support. - Strengthened KV member cleanup, MQ close semantics, framework shutdown barriers, and background-task ownership. - Added rclone S3 integration coverage and a resource-bounded large-scale MPMC MQ CI scenario. - Unified GitHub Release, PyPI, and Docker Hub publication behind one parameter-free GitHub Actions entrypoint. @@ -49,7 +49,7 @@ The documented single-node `rclone v1.60.1` comparison reports that FluxonFS led ## 🌐 Communication and KV Lifecycle -- Updated the closed communication boundary to ABI 9 while retaining runtime ABI, open-surface, boundary-mode, and provider-anchor checks. +- Updated the closed communication boundary to ABI 10 while retaining runtime ABI, open-surface, boundary-mode, and provider-anchor checks. - Added event-driven TCP reactor mode alongside the existing busy-poll path, plus bounded test controls for reactor shards and control/bulk lanes. - Added explicit KV member lifecycle indexes and cleanup paths for member departure, in-flight requests, replicas, holders, and allocation ownership. - Tightened configuration validation and separated stable network configuration from developer-only `test_spec_config` switches. @@ -81,10 +81,11 @@ The documented single-node `rclone v1.60.1` comparison reports that FluxonFS led ## 🔐 Version and SDK Contract - Public Python package, Rust workspace, and Quick Start version: `0.2.2`. -- Closed communication SDK version: `0.2.1`. -- Closed SDK required open-surface contract version: `0.2.1`. +- Closed communication SDK version: `0.2.2`. +- Closed SDK required open-surface contract version: `0.2.2`. +- Closed communication ABI: `10`; SDK manifest schema: `8`. -The SDK version and open-surface contract version are independent from the public release version. The tagged runtime must still pass ABI, open-surface, boundary-mode, and provider-anchor validation. +The SDK version and open-surface contract version are independent from the public release version. They advance in this release because the `ClusterManagerNewArg` bitcode payload changed incompatibly. The tagged runtime must still pass ABI, open-surface, boundary-mode, and provider-anchor validation. ## 📦 Release Artifacts @@ -113,4 +114,5 @@ The Docker publication does not update `latest`. - #48, #49, #51, #52 — expand and refine the public Fluxon overview and AI-native distributed-data-plane positioning. - #47 — support communication ABI 9 and event-driven TCP reactors. - #50 — add hybrid S3 writes, shared lease keepalive, cleanup ownership, and shutdown barriers. +- The tagged revision advances the communication boundary from ABI 9 to ABI 10 to carry the etcd unary RPC retry configuration through the open/closed payload. - The tagged `v0.2.2` revision also contains the unified GitHub Release / PyPI / Docker Hub release workflow and its deterministic readiness gates. diff --git a/fluxon_rs/Cargo.lock b/fluxon_rs/Cargo.lock index 876f37f..6154ca9 100644 --- a/fluxon_rs/Cargo.lock +++ b/fluxon_rs/Cargo.lock @@ -1321,7 +1321,6 @@ dependencies = [ "serde_json", "thiserror 1.0.69", "tokio", - "tonic 0.10.2", "tracing", ] @@ -1461,6 +1460,7 @@ dependencies = [ "tempfile", "tokio", "tokio-stream", + "tonic 0.10.2", "tracing", "tracing-appender", "tracing-log 0.1.4", diff --git a/fluxon_rs/fluxon_commu/src/facade/closed_sdk.rs b/fluxon_rs/fluxon_commu/src/facade/closed_sdk.rs index 9a16171..9992c29 100644 --- a/fluxon_rs/fluxon_commu/src/facade/closed_sdk.rs +++ b/fluxon_rs/fluxon_commu/src/facade/closed_sdk.rs @@ -45,6 +45,11 @@ pub enum CurrentProviderContractError { expected_by_sdk: String, actual_open_surface: String, }, + VersionFieldMismatch { + field: &'static str, + expected_by_open: u32, + actual_sdk: u32, + }, RuntimeAnchorMismatch { field: &'static str, expected_by_sdk: usize, @@ -72,6 +77,15 @@ impl Display for CurrentProviderContractError { "closed SDK requires open surface version {}, but the current open contract version is {}", expected_by_sdk, actual_open_surface ), + Self::VersionFieldMismatch { + field, + expected_by_open, + actual_sdk, + } => write!( + f, + "closed SDK {} mismatch: open expects {}, sdk advertises {}", + field, expected_by_open, actual_sdk + ), Self::RuntimeAnchorMismatch { field, expected_by_sdk, @@ -109,6 +123,20 @@ pub fn assert_current_provider_contract() assert_abi_compatible()?; let version_info = query_version()?; + if version_info.abi_version != FLUXON_COMMU_CLOSED_ABI_VERSION { + return Err(CurrentProviderContractError::VersionFieldMismatch { + field: "ABI version", + expected_by_open: FLUXON_COMMU_CLOSED_ABI_VERSION, + actual_sdk: version_info.abi_version, + }); + } + if version_info.sdk_schema_version != FLUXON_COMMU_CLOSED_SDK_SCHEMA_VERSION { + return Err(CurrentProviderContractError::VersionFieldMismatch { + field: "schema version", + expected_by_open: FLUXON_COMMU_CLOSED_SDK_SCHEMA_VERSION, + actual_sdk: version_info.sdk_schema_version, + }); + } let expected_boundary_mode = crate::provider::CURRENT_PROVIDER_BOUNDARY_MODE; if version_info.boundary_mode != expected_boundary_mode { return Err(CurrentProviderContractError::BoundaryModeMismatch { @@ -225,7 +253,10 @@ pub(crate) fn spawn_deferred_drop_runtime_handle( #[cfg(test)] mod tests { - use super::assert_current_provider_contract; + use super::{ + FLUXON_COMMU_CLOSED_ABI_VERSION, FLUXON_COMMU_CLOSED_SDK_SCHEMA_VERSION, abi_version, + assert_current_provider_contract, sdk_schema_version, + }; use fluxon_commu_contract::FLUXON_COMMU_OPEN_SURFACE_VERSION; #[test] @@ -236,6 +267,16 @@ mod tests { snapshot.version_info.required_open_surface_version, FLUXON_COMMU_OPEN_SURFACE_VERSION ); + assert_eq!( + snapshot.version_info.abi_version, + FLUXON_COMMU_CLOSED_ABI_VERSION + ); + assert_eq!( + snapshot.version_info.sdk_schema_version, + FLUXON_COMMU_CLOSED_SDK_SCHEMA_VERSION + ); + assert_eq!(abi_version(), FLUXON_COMMU_CLOSED_ABI_VERSION); + assert_eq!(sdk_schema_version(), FLUXON_COMMU_CLOSED_SDK_SCHEMA_VERSION); assert_eq!( snapshot.sdk_runtime_anchor.cluster_manager_size, snapshot.provider_runtime_anchor.cluster_manager_size diff --git a/fluxon_rs/fluxon_commu/src/facade/cluster_manager.rs b/fluxon_rs/fluxon_commu/src/facade/cluster_manager.rs index 2146710..e1be442 100644 --- a/fluxon_rs/fluxon_commu/src/facade/cluster_manager.rs +++ b/fluxon_rs/fluxon_commu/src/facade/cluster_manager.rs @@ -648,6 +648,7 @@ impl ClusterManager { pub async fn new( etcd_endpoints: Vec, + etcd_rpc_max_retries: u32, cluster_name: String, instance_name: Option, port: Option, @@ -659,6 +660,7 @@ impl ClusterManager { ) -> crate::ClusterResult { Self::construct(ClusterManagerNewArg { etcd_endpoints, + etcd_rpc_max_retries, cluster_name, instance_name, port, diff --git a/fluxon_rs/fluxon_commu/src/lib.rs b/fluxon_rs/fluxon_commu/src/lib.rs index f2490eb..a1af42f 100644 --- a/fluxon_rs/fluxon_commu/src/lib.rs +++ b/fluxon_rs/fluxon_commu/src/lib.rs @@ -26,7 +26,7 @@ pub use closed_sdk::{ pub use cluster::{ ClusterError, ClusterEvent, ClusterMember, ClusterResult, ETCD_PREFIX_SCAN_PAGE_LIMIT, EtcdPrefixScanAction, EtcdPrefixScanError, NodeID, NodeIDStr, NodeIDString, NodeRole, - scan_etcd_prefix_paginated, + scan_etcd_prefix_paginated, scan_etcd_prefix_paginated_with_retry, }; pub use cluster_manager::{ ClusterManager, ClusterManagerNewArg, ClusterManagerRdmaControlInit, diff --git a/fluxon_rs/fluxon_commu_closed_sdk_consumer/src/lib.rs b/fluxon_rs/fluxon_commu_closed_sdk_consumer/src/lib.rs index eae5614..139c101 100644 --- a/fluxon_rs/fluxon_commu_closed_sdk_consumer/src/lib.rs +++ b/fluxon_rs/fluxon_commu_closed_sdk_consumer/src/lib.rs @@ -26,8 +26,8 @@ use fluxon_commu_contract::{ pub mod rdma_probe; -pub const FLUXON_COMMU_CLOSED_SDK_SCHEMA_VERSION: u32 = 7; -pub const FLUXON_COMMU_CLOSED_ABI_VERSION: u32 = 9; +pub const FLUXON_COMMU_CLOSED_SDK_SCHEMA_VERSION: u32 = 8; +pub const FLUXON_COMMU_CLOSED_ABI_VERSION: u32 = 10; pub const FLUXON_COMMU_CLOSED_HOST_CALLBACKS_ABI_VERSION: u32 = 8; pub const FLUXON_COMMU_CLOSED_RUNTIME_RESULT_OK: i32 = 0; pub const FLUXON_COMMU_CLOSED_RUNTIME_RESULT_ERR: i32 = 1; diff --git a/fluxon_rs/fluxon_commu_contract/src/cluster.rs b/fluxon_rs/fluxon_commu_contract/src/cluster.rs index b0f1dde..cda6387 100644 --- a/fluxon_rs/fluxon_commu_contract/src/cluster.rs +++ b/fluxon_rs/fluxon_commu_contract/src/cluster.rs @@ -1,6 +1,7 @@ use crate::config::NetworkConfig; use bitcode::{Decode, Encode}; use etcd_client::{Client, GetOptions}; +use fluxon_util::etcd::retry_etcd_rpc; use fluxon_util::prefix_scan::{ PrefixScanAction, prefix_scan_key_after, prefix_scan_range_end_exclusive, }; @@ -36,6 +37,23 @@ where pub async fn scan_etcd_prefix_paginated( client: &mut Client, prefix: &str, + on_kv: F, +) -> Result<(), EtcdPrefixScanError> +where + E: std::fmt::Display + std::fmt::Debug, + F: FnMut(&[u8], &[u8]) -> Result, +{ + scan_etcd_prefix_paginated_with_retry(client, prefix, 0, on_kv).await +} + +/// Scans an etcd prefix page by page and retries transient page-range failures. +/// +/// A page is delivered to `on_kv` only after its Range RPC succeeds, so retrying +/// the RPC never invokes the callback twice for the same failed attempt. +pub async fn scan_etcd_prefix_paginated_with_retry( + client: &mut Client, + prefix: &str, + max_retries: u32, mut on_kv: F, ) -> Result<(), EtcdPrefixScanError> where @@ -46,21 +64,29 @@ where let mut start_key = prefix.as_bytes().to_vec(); loop { - let resp = client - .get( - start_key.clone(), - Some( - GetOptions::new() - .with_range(range_end.clone()) - .with_limit(ETCD_PREFIX_SCAN_PAGE_LIMIT), - ), - ) - .await - .map_err(|source| EtcdPrefixScanError::Get { - prefix: prefix.to_string(), - start_key: start_key.clone(), - source, - })?; + let resp = retry_etcd_rpc(max_retries, "prefix_range", || { + let mut attempt_client = client.clone(); + let attempt_start_key = start_key.clone(); + let attempt_range_end = range_end.clone(); + async move { + attempt_client + .get( + attempt_start_key, + Some( + GetOptions::new() + .with_range(attempt_range_end) + .with_limit(ETCD_PREFIX_SCAN_PAGE_LIMIT), + ), + ) + .await + } + }) + .await + .map_err(|source| EtcdPrefixScanError::Get { + prefix: prefix.to_string(), + start_key: start_key.clone(), + source, + })?; if resp.kvs().is_empty() { break; diff --git a/fluxon_rs/fluxon_commu_contract/src/cluster_manager.rs b/fluxon_rs/fluxon_commu_contract/src/cluster_manager.rs index 7094347..6fae544 100644 --- a/fluxon_rs/fluxon_commu_contract/src/cluster_manager.rs +++ b/fluxon_rs/fluxon_commu_contract/src/cluster_manager.rs @@ -1,6 +1,7 @@ pub use crate::cluster::{ ClusterError, ClusterEvent, ClusterMember, ClusterResult, EtcdPrefixScanAction, NodeID, NodeIDStr, NodeIDString, NodeRole, scan_etcd_prefix_paginated, + scan_etcd_prefix_paginated_with_retry, }; pub use crate::config::NetworkConfig; pub use crate::member_metadata::{ @@ -80,6 +81,7 @@ impl IpcBandwidthAttributorHandle { #[derive(Clone, Debug, Encode, Decode)] pub struct ClusterManagerNewArg { pub etcd_endpoints: Vec, + pub etcd_rpc_max_retries: u32, pub cluster_name: String, pub instance_name: Option, pub port: Option, diff --git a/fluxon_rs/fluxon_commu_contract/src/lib.rs b/fluxon_rs/fluxon_commu_contract/src/lib.rs index 57569a1..3d02abd 100644 --- a/fluxon_rs/fluxon_commu_contract/src/lib.rs +++ b/fluxon_rs/fluxon_commu_contract/src/lib.rs @@ -1,7 +1,7 @@ extern crate self as fluxon_commu_contract; /// Version of the open contract consumed by the closed communication SDK. -pub const FLUXON_COMMU_OPEN_SURFACE_VERSION: &str = "0.2.1"; +pub const FLUXON_COMMU_OPEN_SURFACE_VERSION: &str = "0.2.2"; pub mod closed_runtime; pub mod cluster; diff --git a/fluxon_rs/fluxon_commu_contract/src/transfer_engine/surface.rs b/fluxon_rs/fluxon_commu_contract/src/transfer_engine/surface.rs index 7d3793f..1859255 100644 --- a/fluxon_rs/fluxon_commu_contract/src/transfer_engine/surface.rs +++ b/fluxon_rs/fluxon_commu_contract/src/transfer_engine/surface.rs @@ -47,6 +47,8 @@ pub enum TransferBackendActivationMode { #[derive(Clone, Debug, Encode, Decode)] pub struct ClientTransferEngineNewArg { pub metadata_uri: String, + /// Number of transient unary etcd RPC retries after the initial attempt. + pub etcd_rpc_max_retries: u32, pub instance_name: String, pub transfer_engine: TransferEngineType, pub enable_transfer_rpc_fast_path: bool, diff --git a/fluxon_rs/fluxon_kv/examples/cluster_example.rs b/fluxon_rs/fluxon_kv/examples/cluster_example.rs index b4c9d66..18640de 100644 --- a/fluxon_rs/fluxon_kv/examples/cluster_example.rs +++ b/fluxon_rs/fluxon_kv/examples/cluster_example.rs @@ -239,6 +239,7 @@ async fn main() -> Result<()> { cluster_example_arg: ClusterExampleNewArg::new(Arc::clone(&shutdown)), cluster_manager_arg: ClusterManagerNewArg { etcd_endpoints, + etcd_rpc_max_retries: fluxon_util::etcd::DEFAULT_ETCD_RPC_MAX_RETRIES, cluster_name: args.cluster_name.clone(), instance_name: Some(args.instance_name.clone()), port: Some(args.port), diff --git a/fluxon_rs/fluxon_kv/examples/p2p_rpc_example.rs b/fluxon_rs/fluxon_kv/examples/p2p_rpc_example.rs index 567599f..f3b5c38 100644 --- a/fluxon_rs/fluxon_kv/examples/p2p_rpc_example.rs +++ b/fluxon_rs/fluxon_kv/examples/p2p_rpc_example.rs @@ -359,6 +359,7 @@ async fn main() -> AnyResult<()> { let chat_arg = ChatModuleNewArg::new(node_name.to_string()); let cluster_manager_arg = ClusterManagerNewArg { etcd_endpoints: vec![args.etcd_endpoint], + etcd_rpc_max_retries: fluxon_util::etcd::DEFAULT_ETCD_RPC_MAX_RETRIES, cluster_name: args.cluster_name, instance_name: Some(args.instance_name.clone()), port: None, // Port is now discovered automatically diff --git a/fluxon_rs/fluxon_kv/src/client_seg_pool/mod.rs b/fluxon_rs/fluxon_kv/src/client_seg_pool/mod.rs index e3f9501..aaf2d8d 100644 --- a/fluxon_rs/fluxon_kv/src/client_seg_pool/mod.rs +++ b/fluxon_rs/fluxon_kv/src/client_seg_pool/mod.rs @@ -48,6 +48,7 @@ pub struct ClientSegPoolNewArg { pub large_file_paths: crate::config::LargeFilePaths, pub cluster_name: String, pub etcd_addresses: Vec, + pub etcd_rpc_max_retries: u32, pub attach_existing_meta: Option, pub side_transfer_worker: bool, pub require_transfer_rpc_fast_path_ready_timeout: Option, @@ -62,6 +63,8 @@ pub struct SharedJsonMeta { pub sub_cluster: Option, pub cluster_name: String, pub etcd_addresses: Vec, + #[serde(default = "crate::config::default_etcd_rpc_max_retries")] + pub etcd_rpc_max_retries: u32, pub share_mem_path: String, pub large_file_paths: crate::config::LargeFilePaths, pub protocol_version: String, @@ -209,6 +212,7 @@ pub struct ClientSegPoolInner { // Redundant fields written to shared.json for external bootstrap and strict validation. cluster_name: String, etcd_addresses: Vec, + etcd_rpc_max_retries: u32, require_transfer_rpc_fast_path_ready_timeout: Option, /// Whether we've already notified external by writing memory.file after readiness @@ -284,6 +288,7 @@ mod range_guard_tests { attach_owner_ref: None, cluster_name: String::new(), etcd_addresses: Vec::new(), + etcd_rpc_max_retries: crate::config::default_etcd_rpc_max_retries(), require_transfer_rpc_fast_path_ready_timeout: None, ready_notified: AtomicBool::new(false), }) @@ -357,6 +362,7 @@ impl ClientSegPool { let large_file_paths = arg.large_file_paths; let cluster_name = arg.cluster_name; let etcd_addresses = arg.etcd_addresses; + let etcd_rpc_max_retries = arg.etcd_rpc_max_retries; let attach_existing_meta = arg.attach_existing_meta; let side_transfer_worker = arg.side_transfer_worker; let require_transfer_rpc_fast_path_ready_timeout = @@ -453,6 +459,7 @@ impl ClientSegPool { attach_owner_ref, cluster_name: cluster_name.clone(), etcd_addresses: etcd_addresses.clone(), + etcd_rpc_max_retries, require_transfer_rpc_fast_path_ready_timeout, ready_notified: AtomicBool::new(false), }; @@ -469,6 +476,7 @@ impl ClientSegPool { attach_owner_ref, cluster_name: cluster_name.clone(), etcd_addresses: etcd_addresses.clone(), + etcd_rpc_max_retries, require_transfer_rpc_fast_path_ready_timeout, ready_notified: AtomicBool::new(false), }; @@ -619,6 +627,7 @@ impl ClientSegPool { attach_owner_ref, cluster_name, etcd_addresses, + etcd_rpc_max_retries, require_transfer_rpc_fast_path_ready_timeout, ready_notified: AtomicBool::new(false), }; @@ -1258,6 +1267,7 @@ impl ClientSegPool { cluster_name: inner.cluster_name.clone(), etcd_addresses: inner.etcd_addresses.clone(), + etcd_rpc_max_retries: inner.etcd_rpc_max_retries, share_mem_path: share_mem_canonical, large_file_paths: inner.large_file_paths.clone(), diff --git a/fluxon_rs/fluxon_kv/src/cluster_manager/cluster_manager_test.rs b/fluxon_rs/fluxon_kv/src/cluster_manager/cluster_manager_test.rs index 1b937a3..ecf5879 100644 --- a/fluxon_rs/fluxon_kv/src/cluster_manager/cluster_manager_test.rs +++ b/fluxon_rs/fluxon_kv/src/cluster_manager/cluster_manager_test.rs @@ -189,6 +189,7 @@ async fn test_cluster_manager_basic_functionality() { }, cluster_manager_arg: ClusterManagerNewArg { etcd_endpoints: get_etcd_endpoints(), + etcd_rpc_max_retries: fluxon_util::etcd::DEFAULT_ETCD_RPC_MAX_RETRIES, cluster_name: cluster_name.to_string(), instance_name: Some("test-member-1".to_string()), port: Some(8080), @@ -264,6 +265,7 @@ async fn test_cluster_manager_watch_functionality() { }, cluster_manager_arg: ClusterManagerNewArg { etcd_endpoints: get_etcd_endpoints(), + etcd_rpc_max_retries: fluxon_util::etcd::DEFAULT_ETCD_RPC_MAX_RETRIES, cluster_name: cluster_name.to_string(), instance_name: Some("test-member-1".to_string()), port: Some(8080), @@ -303,6 +305,7 @@ async fn test_cluster_manager_watch_functionality() { }, cluster_manager_arg: ClusterManagerNewArg { etcd_endpoints: get_etcd_endpoints(), + etcd_rpc_max_retries: fluxon_util::etcd::DEFAULT_ETCD_RPC_MAX_RETRIES, cluster_name: cluster_name.to_string(), instance_name: Some("test-member-2".to_string()), port: Some(8081), @@ -369,6 +372,7 @@ async fn test_cluster_manager_lease_management() { }, cluster_manager_arg: ClusterManagerNewArg { etcd_endpoints: get_etcd_endpoints(), + etcd_rpc_max_retries: fluxon_util::etcd::DEFAULT_ETCD_RPC_MAX_RETRIES, cluster_name: cluster_name.to_string(), instance_name: Some("test-member-1".to_string()), port: Some(8080), @@ -433,6 +437,7 @@ async fn test_cluster_manager_multiple_members() { }, cluster_manager_arg: ClusterManagerNewArg { etcd_endpoints: get_etcd_endpoints(), + etcd_rpc_max_retries: fluxon_util::etcd::DEFAULT_ETCD_RPC_MAX_RETRIES, cluster_name: cluster_name.to_string(), instance_name: Some("test-member-1".to_string()), port: Some(8080), @@ -464,6 +469,7 @@ async fn test_cluster_manager_multiple_members() { }, cluster_manager_arg: ClusterManagerNewArg { etcd_endpoints: get_etcd_endpoints(), + etcd_rpc_max_retries: fluxon_util::etcd::DEFAULT_ETCD_RPC_MAX_RETRIES, cluster_name: cluster_name.to_string(), instance_name: Some("test-member-2".to_string()), port: Some(8081), @@ -495,6 +501,7 @@ async fn test_cluster_manager_multiple_members() { }, cluster_manager_arg: ClusterManagerNewArg { etcd_endpoints: get_etcd_endpoints(), + etcd_rpc_max_retries: fluxon_util::etcd::DEFAULT_ETCD_RPC_MAX_RETRIES, cluster_name: cluster_name.to_string(), instance_name: Some("test-member-3".to_string()), port: Some(8082), @@ -581,6 +588,7 @@ async fn test_cluster_manager_concurrent_operations() { }, cluster_manager_arg: ClusterManagerNewArg { etcd_endpoints: get_etcd_endpoints(), + etcd_rpc_max_retries: fluxon_util::etcd::DEFAULT_ETCD_RPC_MAX_RETRIES, cluster_name: cluster_name.to_string(), instance_name: Some("test-member-concurrent".to_string()), port: Some(8080), @@ -651,6 +659,7 @@ async fn test_cluster_manager_duplicate_instance_name() { }, cluster_manager_arg: ClusterManagerNewArg { etcd_endpoints: get_etcd_endpoints(), + etcd_rpc_max_retries: fluxon_util::etcd::DEFAULT_ETCD_RPC_MAX_RETRIES, cluster_name: cluster_name.to_string(), instance_name: Some("duplicate-member".to_string()), port: Some(9000), @@ -681,6 +690,7 @@ async fn test_cluster_manager_duplicate_instance_name() { }, cluster_manager_arg: ClusterManagerNewArg { etcd_endpoints: get_etcd_endpoints(), + etcd_rpc_max_retries: fluxon_util::etcd::DEFAULT_ETCD_RPC_MAX_RETRIES, cluster_name: cluster_name.to_string(), instance_name: Some("duplicate-member".to_string()), port: Some(9001), diff --git a/fluxon_rs/fluxon_kv/src/config.rs b/fluxon_rs/fluxon_kv/src/config.rs index 6d8f84a..7fba520 100644 --- a/fluxon_rs/fluxon_kv/src/config.rs +++ b/fluxon_rs/fluxon_kv/src/config.rs @@ -11,6 +11,10 @@ use std::net::IpAddr; use std::path::{Path, PathBuf}; use std::str::FromStr; +pub(crate) fn default_etcd_rpc_max_retries() -> u32 { + fluxon_util::etcd::DEFAULT_ETCD_RPC_MAX_RETRIES +} + /// YAML wrapper to distinguish between: /// - key missing: `Option::None` /// - key present with null: `Some(YamlNullable::Null)` @@ -660,6 +664,8 @@ pub struct MasterConfigYaml { pub port: Option, pub etcd_endpoints: Vec, #[serde(skip_serializing_if = "Option::is_none")] + pub etcd_rpc_max_retries: Option, + #[serde(skip_serializing_if = "Option::is_none")] pub monitoring: Option, // monitoring config (prometheus base url, optional remote write, optional otlp_log_api) #[serde(skip_serializing_if = "Option::is_none")] pub network: Option, @@ -679,6 +685,7 @@ pub struct MasterConfig { pub cluster_name: String, pub port: Option, pub etcd_endpoints: Vec, + pub etcd_rpc_max_retries: u32, pub protocol: ProtocolConfig, pub transfer_engine: TransferEngineType, pub enable_transfer_rpc_fast_path: bool, @@ -704,6 +711,8 @@ pub struct ContributeToClusterPoolSizeYaml { pub struct FluxonKvSpecYaml { #[serde(skip_serializing_if = "Option::is_none")] pub etcd_addresses: Option>>, + #[serde(skip_serializing_if = "Option::is_none")] + pub etcd_rpc_max_retries: Option>, pub cluster_name: String, pub share_mem_path: String, #[serde(skip_serializing_if = "Option::is_none")] @@ -785,6 +794,7 @@ pub struct ContributeToClusterPoolSize { #[derive(Debug, Clone)] pub struct FluxonKvSpec { pub etcd_addresses: Vec, + pub etcd_rpc_max_retries: u32, pub cluster_name: String, pub p2p_listen_port: Option, pub transfer_engine: TransferEngineType, @@ -1346,6 +1356,12 @@ impl ClientConfigYaml { } .into_kverror()); } + if self.fluxonkv_spec.etcd_rpc_max_retries.is_some() { + return Err(ConfigError::InvalidClientConfig { + detail: "fluxonkv_spec.etcd_rpc_max_retries is forbidden in zero-contribution mode (it is inherited from owner shared.json)".to_string(), + } + .into_kverror()); + } if self.fluxonkv_spec.large_file_paths.is_some() { return Err(ConfigError::InvalidClientConfig { detail: "fluxonkv_spec.large_file_paths is forbidden in zero-contribution mode (it is inherited from owner shared.json)".to_string(), @@ -1485,6 +1501,18 @@ impl ClientConfigYaml { let fluxonkv_spec = FluxonKvSpec { etcd_addresses: etcd_endpoints, + etcd_rpc_max_retries: match self.fluxonkv_spec.etcd_rpc_max_retries { + None => default_etcd_rpc_max_retries(), + Some(YamlNullable::Value(value)) => value, + Some(YamlNullable::Null) => { + return Err(ConfigError::InvalidClientConfig { + detail: + "fluxonkv_spec.etcd_rpc_max_retries must be an integer when provided" + .to_string(), + } + .into_kverror()); + } + }, cluster_name: self.fluxonkv_spec.cluster_name, p2p_listen_port: self.fluxonkv_spec.p2p_listen_port, transfer_engine, @@ -1901,6 +1929,9 @@ impl MasterConfigYaml { cluster_name: self.cluster_name, port: self.port, etcd_endpoints: self.etcd_endpoints, + etcd_rpc_max_retries: self + .etcd_rpc_max_retries + .unwrap_or_else(default_etcd_rpc_max_retries), protocol, transfer_engine, enable_transfer_rpc_fast_path: resolve_enable_transfer_rpc_fast_path( @@ -2984,6 +3015,99 @@ test_spec_config: assert!(format!("{err}").contains("unknown field `legacy_transfer_backend`")); } + #[test] + fn client_config_etcd_rpc_retries_default_and_explicit_values() { + let owner_yaml = |retry_line: &str| { + format!( + r#" +instance_key: test_owner +contribute_to_cluster_pool_size: + dram: 16777216 + vram: {{}} +fluxonkv_spec: + etcd_addresses: ["127.0.0.1:2379"] +{retry_line} cluster_name: test_cluster + share_mem_path: /tmp/test_owner + large_file_paths: [/tmp/test_owner_large] + sub_cluster: rack-a +"# + ) + }; + + let defaulted = ClientConfigYaml::from_str(&owner_yaml("")) + .unwrap() + .verify() + .unwrap(); + assert_eq!(defaulted.fluxonkv_spec.etcd_rpc_max_retries, 2); + + let disabled = ClientConfigYaml::from_str(&owner_yaml(" etcd_rpc_max_retries: 0\n")) + .unwrap() + .verify() + .unwrap(); + assert_eq!(disabled.fluxonkv_spec.etcd_rpc_max_retries, 0); + + let overridden = ClientConfigYaml::from_str(&owner_yaml(" etcd_rpc_max_retries: 7\n")) + .unwrap() + .verify() + .unwrap(); + assert_eq!(overridden.fluxonkv_spec.etcd_rpc_max_retries, 7); + } + + #[test] + fn zero_contribution_config_rejects_etcd_rpc_retry_override() { + let cfg = ClientConfigYaml::from_str( + r#" +instance_key: test_external +fluxonkv_spec: + etcd_rpc_max_retries: 0 + cluster_name: test_cluster + share_mem_path: /tmp/test_external +"#, + ) + .unwrap(); + let err = cfg.verify().unwrap_err(); + assert!( + format!("{err}").contains( + "fluxonkv_spec.etcd_rpc_max_retries is forbidden in zero-contribution mode" + ) + ); + } + + #[test] + fn master_config_etcd_rpc_retries_default_and_explicit_values() { + let master_yaml = |retry_line: &str| { + format!( + r#" +instance_key: test_master +cluster_name: test_cluster +port: 18080 +etcd_endpoints: ["127.0.0.1:2379"] +{retry_line}monitoring: + prometheus_base_url: "http://127.0.0.1:4000/v1/prometheus" +log_dir: /tmp/test_master_logs +"# + ) + }; + + let defaulted = MasterConfigYaml::from_str(&master_yaml("")) + .unwrap() + .verify() + .unwrap(); + assert_eq!(defaulted.etcd_rpc_max_retries, 2); + + let disabled = MasterConfigYaml::from_str(&master_yaml("etcd_rpc_max_retries: 0\n")) + .unwrap() + .verify() + .unwrap(); + assert_eq!(disabled.etcd_rpc_max_retries, 0); + + let overridden = MasterConfigYaml::from_str(&master_yaml("etcd_rpc_max_retries: 9\n")) + .unwrap() + .verify() + .unwrap(); + assert_eq!(overridden.etcd_rpc_max_retries, 9); + } + #[test] fn master_config_rejects_removed_protocol_type() { let err = MasterConfigYaml::from_str( diff --git a/fluxon_rs/fluxon_kv/src/external_client_api/external_client_test.rs b/fluxon_rs/fluxon_kv/src/external_client_api/external_client_test.rs index e9f5110..865ca2c 100644 --- a/fluxon_rs/fluxon_kv/src/external_client_api/external_client_test.rs +++ b/fluxon_rs/fluxon_kv/src/external_client_api/external_client_test.rs @@ -31,6 +31,7 @@ fn new_master_config( cluster_name: cluster.to_string(), port, etcd_endpoints: vec![etcd.to_string()], + etcd_rpc_max_retries: fluxon_util::etcd::DEFAULT_ETCD_RPC_MAX_RETRIES, protocol: ProtocolConfig { protocol_type: ProtocolType::Tcp, rdma_device_names: None, @@ -81,6 +82,7 @@ fn new_client_config( redis_compat_listen_addr: None, fluxonkv_spec: FluxonKvSpec { etcd_addresses: vec![etcd.to_string()], + etcd_rpc_max_retries: fluxon_util::etcd::DEFAULT_ETCD_RPC_MAX_RETRIES, cluster_name: cluster.to_string(), p2p_listen_port: None, transfer_engine: TransferEngineType::Closed, @@ -126,6 +128,7 @@ fn new_zero_contribution_client_config( redis_compat_listen_addr: None, fluxonkv_spec: FluxonKvSpec { etcd_addresses: Vec::new(), + etcd_rpc_max_retries: fluxon_util::etcd::DEFAULT_ETCD_RPC_MAX_RETRIES, cluster_name: cluster.to_string(), p2p_listen_port: None, transfer_engine: TransferEngineType::P2p, diff --git a/fluxon_rs/fluxon_kv/src/kv_test.rs b/fluxon_rs/fluxon_kv/src/kv_test.rs index d002de1..c529562 100644 --- a/fluxon_rs/fluxon_kv/src/kv_test.rs +++ b/fluxon_rs/fluxon_kv/src/kv_test.rs @@ -1071,6 +1071,7 @@ fn new_master_launch( cluster_name: round.cluster_name.clone(), port, etcd_endpoints: vec![etcd.clone()], + etcd_rpc_max_retries: fluxon_util::etcd::DEFAULT_ETCD_RPC_MAX_RETRIES, protocol, transfer_engine, enable_transfer_rpc_fast_path, @@ -1127,6 +1128,7 @@ fn build_client_launch( redis_compat_listen_addr: None, fluxonkv_spec: FluxonKvSpec { etcd_addresses: fluxonkv_etcd_addresses, + etcd_rpc_max_retries: fluxon_util::etcd::DEFAULT_ETCD_RPC_MAX_RETRIES, cluster_name: round.cluster_name.clone(), p2p_listen_port: None, transfer_engine: options diff --git a/fluxon_rs/fluxon_kv/src/kvcore_test_lib.rs b/fluxon_rs/fluxon_kv/src/kvcore_test_lib.rs index 25a1b4c..19f4f48 100644 --- a/fluxon_rs/fluxon_kv/src/kvcore_test_lib.rs +++ b/fluxon_rs/fluxon_kv/src/kvcore_test_lib.rs @@ -78,6 +78,7 @@ fn new_master_config_with_cluster( cluster_name: cluster_name.to_string(), port, etcd_endpoints: vec![etcd.clone()], + etcd_rpc_max_retries: fluxon_util::etcd::DEFAULT_ETCD_RPC_MAX_RETRIES, protocol: ProtocolConfig { protocol_type: ProtocolType::Tcp, rdma_device_names: None, @@ -140,6 +141,7 @@ fn new_client_config_with_cluster_and_dram( redis_compat_listen_addr: None, fluxonkv_spec: FluxonKvSpec { etcd_addresses: vec![etcd], + etcd_rpc_max_retries: fluxon_util::etcd::DEFAULT_ETCD_RPC_MAX_RETRIES, cluster_name: cluster_name.to_string(), p2p_listen_port: None, transfer_engine: TransferEngineType::Closed, diff --git a/fluxon_rs/fluxon_kv/src/lib.rs b/fluxon_rs/fluxon_kv/src/lib.rs index 8fb6eb6..f805ff0 100644 --- a/fluxon_rs/fluxon_kv/src/lib.rs +++ b/fluxon_rs/fluxon_kv/src/lib.rs @@ -816,6 +816,7 @@ fn build_side_transfer_worker_config( redis_compat_listen_addr: None, fluxonkv_spec: FluxonKvSpec { etcd_addresses: Vec::new(), + etcd_rpc_max_retries: owner_config.fluxonkv_spec.etcd_rpc_max_retries, cluster_name: owner_config.cluster_name.clone(), p2p_listen_port, transfer_engine: TransferEngineType::P2p, @@ -871,6 +872,7 @@ fn build_side_transfer_worker_config_yaml( pprof_duration_seconds: side_config.pprof_duration_seconds, fluxonkv_spec: crate::config::FluxonKvSpecYaml { etcd_addresses: None, + etcd_rpc_max_retries: None, cluster_name: side_config.cluster_name, share_mem_path: side_config.share_mem_path, large_file_paths: None, @@ -1402,6 +1404,10 @@ async fn finish_framework_init( )) } +fn etcd_metadata_uri(endpoints: &[String]) -> String { + endpoints.join(",") +} + async fn run_master_impl( config_arg: ConfigArg, test_overrides: Option, @@ -1507,6 +1513,7 @@ async fn run_master_impl( let init_args = InitArgsMaster { cluster_manager_arg: ClusterManagerNewArg { etcd_endpoints: config.etcd_endpoints.clone(), + etcd_rpc_max_retries: config.etcd_rpc_max_retries, cluster_name: config.cluster_name.clone(), instance_name: Some(config.instance_key.clone()), port: None, @@ -1527,7 +1534,8 @@ async fn run_master_impl( config.test_spec_config.user_rpc_sync_handler_thread_count, ), client_transfer_engine_arg: ClientTransferEngineNewArg { - metadata_uri: config.etcd_endpoints[0].clone(), + metadata_uri: etcd_metadata_uri(&config.etcd_endpoints), + etcd_rpc_max_retries: config.etcd_rpc_max_retries, instance_name: config.instance_key.clone(), enable_transfer_rpc_fast_path: config.enable_transfer_rpc_fast_path, rpc_port: 12345, @@ -1650,6 +1658,7 @@ async fn bootstrap_zero_contribution_client_config(config: ClientConfig) -> KvRe let mut final_config = config; final_config.etcd_addresses_raw = metadata.meta.etcd_addresses.clone(); final_config.fluxonkv_spec.etcd_addresses = metadata.etcd_endpoints; + final_config.fluxonkv_spec.etcd_rpc_max_retries = metadata.meta.etcd_rpc_max_retries; final_config.fluxonkv_spec.sub_cluster = metadata.meta.sub_cluster.clone(); final_config.share_mem_path = metadata.share_mem_path; final_config.large_file_paths = metadata.meta.large_file_paths; @@ -2021,6 +2030,7 @@ async fn run_client_impl( let init_args = InitArgsExternal { cluster_manager_arg: ClusterManagerNewArg { etcd_endpoints: config.fluxonkv_spec.etcd_addresses.clone(), + etcd_rpc_max_retries: config.fluxonkv_spec.etcd_rpc_max_retries, cluster_name: config.cluster_name.clone(), instance_name: Some(config.instance_key.clone()), port: None, @@ -2096,6 +2106,7 @@ async fn run_client_impl( let init_args = InitArgsOwner { cluster_manager_arg: ClusterManagerNewArg { etcd_endpoints: config.fluxonkv_spec.etcd_addresses.clone(), + etcd_rpc_max_retries: config.fluxonkv_spec.etcd_rpc_max_retries, cluster_name: config.cluster_name.clone(), instance_name: Some(config.instance_key.clone()), port: None, @@ -2134,6 +2145,7 @@ async fn run_client_impl( large_file_paths: config.large_file_paths.clone(), cluster_name: config.cluster_name.clone(), etcd_addresses: config.etcd_addresses_raw.clone(), + etcd_rpc_max_retries: config.fluxonkv_spec.etcd_rpc_max_retries, attach_existing_meta: if is_side_transfer_worker { Some(bootstrapped_shared_meta.clone().ok_or_else(|| { anyhow::anyhow!( @@ -2150,7 +2162,8 @@ async fn run_client_impl( .map(Duration::from_secs), }, client_transfer_engine_arg: ClientTransferEngineNewArg { - metadata_uri: config.fluxonkv_spec.etcd_addresses[0].clone(), + metadata_uri: etcd_metadata_uri(&config.fluxonkv_spec.etcd_addresses), + etcd_rpc_max_retries: config.fluxonkv_spec.etcd_rpc_max_retries, instance_name: config.instance_key.clone(), enable_transfer_rpc_fast_path: config.fluxonkv_spec.enable_transfer_rpc_fast_path, rpc_port: 12345, @@ -2524,6 +2537,17 @@ mod tests { use std::path::Path; use uuid::Uuid; + #[test] + fn etcd_metadata_uri_keeps_all_configured_endpoints() { + assert_eq!( + etcd_metadata_uri(&[ + "http://etcd-a:2379".to_string(), + "http://etcd-b:2379".to_string(), + ]), + "http://etcd-a:2379,http://etcd-b:2379" + ); + } + fn new_test_dir(prefix: &str) -> std::path::PathBuf { let path = std::env::temp_dir().join(format!("{}_{}", prefix, Uuid::new_v4())); std::fs::create_dir_all(&path).unwrap(); @@ -2548,6 +2572,7 @@ mod tests { redis_compat_listen_addr: None, fluxonkv_spec: FluxonKvSpec { etcd_addresses: vec!["http://127.0.0.1:2379".to_string()], + etcd_rpc_max_retries: fluxon_util::etcd::DEFAULT_ETCD_RPC_MAX_RETRIES, cluster_name: "test_cluster".to_string(), p2p_listen_port: Some(41000), transfer_engine: TransferEngineType::P2p, @@ -2866,6 +2891,7 @@ mod tests { sub_cluster: Some("owner-sub".to_string()), cluster_name: "test_cluster".to_string(), etcd_addresses: vec!["127.0.0.1:2379".to_string()], + etcd_rpc_max_retries: 7, share_mem_path: std::fs::canonicalize(&share_mem_root) .unwrap() .to_string_lossy() @@ -2879,6 +2905,7 @@ mod tests { }; let shared_meta_json = serde_json::to_string(&shared_meta).unwrap(); assert!(shared_meta_json.contains("\"large_file_paths\":[")); + assert!(shared_meta_json.contains("\"etcd_rpc_max_retries\":7")); assert!(!shared_meta_json.contains("root_paths")); std::fs::write( share_mem_root.join("shared.json"), @@ -2903,6 +2930,7 @@ mod tests { redis_compat_listen_addr: None, fluxonkv_spec: FluxonKvSpec { etcd_addresses: Vec::new(), + etcd_rpc_max_retries: fluxon_util::etcd::DEFAULT_ETCD_RPC_MAX_RETRIES, cluster_name: "test_cluster".to_string(), p2p_listen_port: Some(41001), transfer_engine: TransferEngineType::P2p, @@ -2930,6 +2958,7 @@ mod tests { bootstrapped.fluxonkv_spec.etcd_addresses, vec!["http://127.0.0.1:2379".to_string()] ); + assert_eq!(bootstrapped.fluxonkv_spec.etcd_rpc_max_retries, 7); } #[test] diff --git a/fluxon_rs/fluxon_kv/src/memholder/memholder_test.rs b/fluxon_rs/fluxon_kv/src/memholder/memholder_test.rs index 1848835..cd1e154 100644 --- a/fluxon_rs/fluxon_kv/src/memholder/memholder_test.rs +++ b/fluxon_rs/fluxon_kv/src/memholder/memholder_test.rs @@ -42,6 +42,7 @@ fn new_master_config( cluster_name: cluster.to_string(), port, etcd_endpoints: vec![etcd.to_string()], + etcd_rpc_max_retries: fluxon_util::etcd::DEFAULT_ETCD_RPC_MAX_RETRIES, protocol: ProtocolConfig { protocol_type: ProtocolType::Tcp, rdma_device_names: None, @@ -92,6 +93,7 @@ fn new_client_config_with_size( redis_compat_listen_addr: None, fluxonkv_spec: FluxonKvSpec { etcd_addresses: vec![etcd.to_string()], + etcd_rpc_max_retries: fluxon_util::etcd::DEFAULT_ETCD_RPC_MAX_RETRIES, cluster_name: cluster.to_string(), p2p_listen_port: None, transfer_engine: TransferEngineType::Closed, @@ -129,6 +131,7 @@ fn new_zero_contribution_client_config( redis_compat_listen_addr: None, fluxonkv_spec: FluxonKvSpec { etcd_addresses: Vec::new(), + etcd_rpc_max_retries: fluxon_util::etcd::DEFAULT_ETCD_RPC_MAX_RETRIES, cluster_name: cluster.to_string(), p2p_listen_port: None, transfer_engine: TransferEngineType::P2p, diff --git a/fluxon_rs/fluxon_mq/Cargo.toml b/fluxon_rs/fluxon_mq/Cargo.toml index 8a0324e..b9675f2 100644 --- a/fluxon_rs/fluxon_mq/Cargo.toml +++ b/fluxon_rs/fluxon_mq/Cargo.toml @@ -18,7 +18,6 @@ paste = { workspace = true } serde = { workspace = true } serde_json = { workspace = true } etcd-client = { workspace = true } -tonic = { workspace = true } fluxon_util = { path = "../fluxon_util" } fluxon_observability = { path = "../fluxon_observability" } fluxon_commu = { package = "fluxon_commu", path = "../fluxon_commu", default-features = false } diff --git a/fluxon_rs/fluxon_mq/src/consumer.rs b/fluxon_rs/fluxon_mq/src/consumer.rs index f93ea5b..9b61b8a 100644 --- a/fluxon_rs/fluxon_mq/src/consumer.rs +++ b/fluxon_rs/fluxon_mq/src/consumer.rs @@ -1,6 +1,8 @@ use anyhow::{Context, Result}; use etcd_client as etcd; -use fluxon_commu::{scan_etcd_prefix_paginated, EtcdPrefixScanAction, EtcdPrefixScanError}; +use fluxon_commu::{ + scan_etcd_prefix_paginated_with_retry, EtcdPrefixScanAction, EtcdPrefixScanError, +}; use std::collections::{HashMap, HashSet, VecDeque}; use std::future::{poll_fn, Future}; @@ -29,7 +31,7 @@ use fluxon_observability::keys::{ }; use fluxon_observability::metrics_actor::MetricsHandle as ObserveMetricsHandle; use fluxon_util::etcd::{ - run_prefix_watch_loop, EtcdPrefixWatchLoopControl, OwnedEtcdWatchEvent, + retry_etcd_rpc, run_prefix_watch_loop, EtcdPrefixWatchLoopControl, OwnedEtcdWatchEvent, OwnedEtcdWatchEventKind, ETCD_PREFIX_WATCH_RESTART_SLEEP, }; use fluxon_util::lease_manager::LeaseManager; @@ -39,12 +41,15 @@ use crate::error::MpscError; use crate::keys; use crate::lifecycle::spawn_named; use crate::manager::{ - get_chan_meta, ChanManager, ChanMemberMeta, ChanRole, CONSUME_OFFSET_BEGIN, + get_chan_meta_with_retry, ChanManager, ChanMemberMeta, ChanRole, CONSUME_OFFSET_BEGIN, PRODUCE_OFFSET_BEGIN, }; use crate::nonblocking_monitor::{ spawn_nonblocking_monitor, NonblockingMonitorHandle, NonblockingMonitorKind, }; +use crate::offset_commit::{ + MonotonicOffsetCommit, OffsetCommitProgress, OffsetGeneration, OffsetObservation, +}; use crate::shutdown::ShutdownCtl; use crate::LifecycleView; use tracing::{debug, info, warn}; @@ -73,6 +78,200 @@ fn map_prefix_scan_error(err: EtcdPrefixScanError) -> MpscError { } } +#[derive(Clone, Debug, PartialEq, Eq)] +struct ConsumerMembershipGeneration { + value: Vec, + lease_id: i64, + mod_revision: i64, +} + +fn consumer_membership_get_generation( + txn_res: &etcd::TxnResponse, + key: &str, + operation: &str, +) -> Result, MpscError> { + let responses = txn_res.op_responses(); + let [etcd::TxnOpResponse::Get(get)] = responses.as_slice() else { + return Err(MpscError::Internal(format!( + "{} readback returned an invalid response shape: operations={}", + operation, + responses.len() + ))); + }; + if get.kvs().len() > 1 { + return Err(MpscError::Internal(format!( + "{} readback returned duplicate exact keys", + operation + ))); + } + let Some(kv) = get.kvs().first() else { + return Ok(None); + }; + if kv.key() != key.as_bytes() { + return Err(MpscError::Internal(format!( + "{} readback returned an unexpected key for {}", + operation, key + ))); + } + Ok(Some(ConsumerMembershipGeneration { + value: kv.value().to_vec(), + lease_id: kv.lease(), + mod_revision: kv.mod_revision(), + })) +} + +fn consumer_membership_generation_from_publish( + txn_res: &etcd::TxnResponse, + value: &[u8], + lease_id: i64, + operation: &str, +) -> Result { + let mod_revision = txn_res + .header() + .ok_or_else(|| MpscError::Internal(format!("{} response has no header", operation)))? + .revision(); + if mod_revision <= 0 { + return Err(MpscError::Internal(format!( + "{} returned invalid revision {}", + operation, mod_revision + ))); + } + Ok(ConsumerMembershipGeneration { + value: value.to_vec(), + lease_id, + mod_revision, + }) +} + +fn reconcile_consumer_membership_bind_observation( + current: Option, + key: &str, + value: &[u8], + lease_id: i64, +) -> Result { + match current { + Some(current) if current.value == value && current.lease_id == lease_id => Ok(current), + Some(_) => Err(MpscError::Internal(format!( + "consumer membership key {} already exists with conflicting state", + key + ))), + None => Err(MpscError::Internal(format!( + "consumer membership key {} disappeared while reconciling bind", + key + ))), + } +} + +fn reconcile_consumer_membership_update_observation( + current: Option, + key: &str, + previous: &ConsumerMembershipGeneration, + value: &[u8], + lease_id: i64, +) -> Result { + match current { + Some(current) + if current.value == value + && current.lease_id == lease_id + && current.mod_revision > previous.mod_revision => + { + Ok(current) + } + Some(_) => Err(MpscError::Internal(format!( + "consumer membership key {} changed concurrently while syncing kvclient_sub_cluster", + key + ))), + None => Err(MpscError::Internal(format!( + "consumer membership key {} missing while syncing kvclient_sub_cluster", + key + ))), + } +} + +async fn publish_consumer_membership( + client: &mut etcd::Client, + key: &str, + value: &[u8], + lease_id: i64, + max_retries: u32, +) -> Result { + let txn_res = retry_etcd_rpc(max_retries, "mq_bind_consumer_membership", || { + let mut attempt_client = client.clone(); + let compare = etcd::Compare::create_revision(key, etcd::CompareOp::Equal, 0); + let put = etcd::TxnOp::put( + key, + value, + Some(etcd::PutOptions::new().with_lease(lease_id)), + ); + let txn = etcd::Txn::new() + .when(vec![compare]) + .and_then(vec![put]) + .or_else(vec![etcd::TxnOp::get(key, None)]); + async move { attempt_client.txn(txn).await } + }) + .await?; + + if txn_res.succeeded() { + return consumer_membership_generation_from_publish( + &txn_res, + value, + lease_id, + "consumer membership bind", + ); + } + reconcile_consumer_membership_bind_observation( + consumer_membership_get_generation(&txn_res, key, "consumer membership bind")?, + key, + value, + lease_id, + ) +} + +async fn update_consumer_membership( + client: &mut etcd::Client, + key: &str, + previous: &ConsumerMembershipGeneration, + value: &[u8], + lease_id: i64, + max_retries: u32, +) -> Result { + let txn_res = retry_etcd_rpc(max_retries, "mq_update_consumer_membership", || { + let mut attempt_client = client.clone(); + let compares = vec![ + etcd::Compare::mod_revision(key, etcd::CompareOp::Equal, previous.mod_revision), + etcd::Compare::lease(key, etcd::CompareOp::Equal, previous.lease_id), + etcd::Compare::value(key, etcd::CompareOp::Equal, previous.value.clone()), + ]; + let put = etcd::TxnOp::put( + key, + value, + Some(etcd::PutOptions::new().with_lease(lease_id)), + ); + let txn = etcd::Txn::new() + .when(compares) + .and_then(vec![put]) + .or_else(vec![etcd::TxnOp::get(key, None)]); + async move { attempt_client.txn(txn).await } + }) + .await?; + + if txn_res.succeeded() { + return consumer_membership_generation_from_publish( + &txn_res, + value, + lease_id, + "consumer membership update", + ); + } + reconcile_consumer_membership_update_observation( + consumer_membership_get_generation(&txn_res, key, "consumer membership update")?, + key, + previous, + value, + lease_id, + ) +} + fn merge_monotonic_offset(cached: i64, probed: Option) -> i64 { match probed { Some(value) => value.max(cached), @@ -206,6 +405,7 @@ impl CommitSequencerFailure { struct CommitSequencerState { next_seq: usize, failure: Option, + consume_offset_generations: HashMap, } #[derive(Clone)] @@ -223,6 +423,7 @@ impl CommitSequencer { state: Arc::new(Mutex::new(CommitSequencerState { next_seq: 0, failure: None, + consume_offset_generations: HashMap::new(), })), notify: Arc::new(Notify::new()), progress: Arc::new(Mutex::new(HashMap::new())), @@ -273,6 +474,24 @@ impl CommitSequencer { } } + fn consume_offset_observation(&self, producer_id: &str) -> Option { + self.state + .lock() + .unwrap() + .consume_offset_generations + .get(producer_id) + .cloned() + .map(OffsetObservation::Present) + } + + fn record_consume_offset_generation(&self, producer_id: &str, generation: OffsetGeneration) { + self.state + .lock() + .unwrap() + .consume_offset_generations + .insert(producer_id.to_string(), generation); + } + fn is_failed(&self) -> bool { self.state.lock().unwrap().failure.is_some() } @@ -773,6 +992,11 @@ struct CommitOffsetPutTraceNs { first_poll_to_ready_ns: u128, } +struct CommitOffsetResult { + trace: CommitOffsetPutTraceNs, + generation: OffsetGeneration, +} + struct SelectNextMessageTrace { refresh_latency_ns: u128, refresh_call_count: usize, @@ -929,6 +1153,7 @@ enum ConsumerCmd { pub struct MpscConsumer { chan_id: i64, consumer_idx: String, + membership_generation: ConsumerMembershipGeneration, instance_id: usize, kvclient_sub_cluster: Option, external_client_id: Option, @@ -1466,12 +1691,13 @@ impl MpscConsumer { } let chan_id = chan_mgr.chan_id; + let etcd_rpc_max_retries = chan_mgr.etcd_rpc_max_retries(); let lease_manager = chan_mgr.lease_manager.clone(); let mut client = chan_mgr.etcd_client(); // 1) Ensure channel meta exists let mut meta_client = chan_mgr.etcd_client(); - let _meta = get_chan_meta(&mut meta_client, chan_id) + let _meta = get_chan_meta_with_retry(&mut meta_client, chan_id, etcd_rpc_max_retries) .await .with_context(|| format!("channel meta not found for chan_id={}", chan_id))?; @@ -1493,15 +1719,17 @@ impl MpscConsumer { // at least one ChanManager instance keeps the long lease alive. let mut idx_client = client.clone(); let allocator_lease_id = chan_mgr.global_long_lease.id() as i64; - let consumer_idx = - register_consumer_idx(&mut idx_client, chan_id, allocator_lease_id).await?; + let consumer_idx = register_consumer_idx( + &mut idx_client, + chan_id, + allocator_lease_id, + etcd_rpc_max_retries, + ) + .await?; // 4) Bind consumer membership key under member lease, storing // ChanMemberMeta as JSON for future introspection. let consumer_key = keys::etcd_consumer_key(chan_id, &consumer_idx); - let compare = - etcd::Compare::create_revision(consumer_key.clone(), etcd::CompareOp::Equal, 0); - let member_meta = ChanMemberMeta { member_id: consumer_idx.clone(), role: ChanRole::Consumer, @@ -1511,19 +1739,15 @@ impl MpscConsumer { let meta_bytes = serde_json::to_vec(&member_meta) .map_err(|e| MpscError::Internal(format!("serialize ChanMemberMeta failed: {}", e)))?; - let put_op = etcd::TxnOp::put( - consumer_key.clone(), - meta_bytes, - Some(etcd::PutOptions::new().with_lease(member_lease_id)), - ); - let txn = etcd::Txn::new().when(vec![compare]).and_then(vec![put_op]); - let txn_res = client - .txn(txn) - .await - .with_context(|| format!("failed to bind consumer membership key {}", consumer_key))?; - if !txn_res.succeeded() { - anyhow::bail!("consumer membership key {} already exists", consumer_key); - } + let membership_generation = publish_consumer_membership( + &mut client, + &consumer_key, + &meta_bytes, + member_lease_id, + etcd_rpc_max_retries, + ) + .await + .with_context(|| format!("failed to bind consumer membership key {}", consumer_key))?; // 5) 创建预取 actor，并通过共享队列与之协作。 // actor 自身负责启动 producer member metadata 的 watch； @@ -1548,6 +1772,7 @@ impl MpscConsumer { shutdown.clone(), category, global_lease_id, + etcd_rpc_max_retries, ); let nonblocking_monitor = spawn_nonblocking_monitor( &lifecycle, @@ -1563,6 +1788,7 @@ impl MpscConsumer { Ok(Self { chan_id, consumer_idx, + membership_generation, instance_id: commit_seq.instance_id, kvclient_sub_cluster, external_client_id, @@ -1662,23 +1888,18 @@ impl MpscConsumer { let member_lease_id = self.chan_mgr.member_lease_id(); let consumer_key = keys::etcd_consumer_key(self.chan_id, &self.consumer_idx); - let compare = - etcd::Compare::create_revision(consumer_key.clone(), etcd::CompareOp::Greater, 0); - let put_op = etcd::TxnOp::put( - consumer_key.clone(), - meta_bytes, - Some(etcd::PutOptions::new().with_lease(member_lease_id)), - ); - let txn = etcd::Txn::new().when(vec![compare]).and_then(vec![put_op]); let mut client = self.chan_mgr.etcd_client(); - let txn_res = client.txn(txn).await?; - if !txn_res.succeeded() { - return Err(MpscError::Internal(format!( - "consumer membership key {} missing while syncing kvclient_sub_cluster", - consumer_key - ))); - } + let membership_generation = update_consumer_membership( + &mut client, + &consumer_key, + &self.membership_generation, + &meta_bytes, + member_lease_id, + self.chan_mgr.etcd_rpc_max_retries(), + ) + .await?; + self.membership_generation = membership_generation; self.kvclient_sub_cluster = kvclient_sub_cluster; Ok(()) } @@ -2085,14 +2306,20 @@ impl MpscConsumer { consume_offset: i64, seq: usize, shutdown: ShutdownCtl, - ) -> Result { + initial_observation: Option, + ) -> Result { use tokio::time::sleep; let next_consume_offset = consume_offset + 1; let key = keys::etcd_consume_offset_one_producer_key(chan_id, producer_id); - let next_consume_offset_str = next_consume_offset.to_string(); let begin = Instant::now(); let mut attempts: usize = 0; + let mut offset_commit = MonotonicOffsetCommit::new( + key.clone(), + next_consume_offset, + global_lease_id, + initial_observation, + )?; loop { if shutdown.is_closed() { @@ -2104,12 +2331,8 @@ impl MpscConsumer { attempts += 1; let attempt_begin = Instant::now(); - let put = client.put( - key.clone(), - next_consume_offset_str.clone(), - Some(etcd::PutOptions::new().with_lease(global_lease_id)), - ); - tokio::pin!(put); + let commit = offset_commit.attempt(&mut client); + tokio::pin!(commit); let mut first_poll_at = None; let put_res = tokio::select! { biased; @@ -2125,13 +2348,13 @@ impl MpscConsumer { if first_poll_at.is_none() { first_poll_at = Some(Instant::now()); } - put.as_mut().poll(cx) + commit.as_mut().poll(cx) }), ) => res, }; match put_res { - Ok(Ok(_)) => { + Ok(Ok(OffsetCommitProgress::Complete(generation))) => { let total_elapsed = begin.elapsed(); let attempt_end = Instant::now(); let attempt_elapsed = attempt_end.duration_since(attempt_begin); @@ -2156,12 +2379,26 @@ impl MpscConsumer { first_poll_to_ready_ns / 1_000_000, ); } - return Ok(CommitOffsetPutTraceNs { - total_latency_ns: total_elapsed.as_nanos(), - first_poll_delay_ns, - first_poll_to_ready_ns, + return Ok(CommitOffsetResult { + trace: CommitOffsetPutTraceNs { + total_latency_ns: total_elapsed.as_nanos(), + first_poll_delay_ns, + first_poll_to_ready_ns, + }, + generation, }); } + Ok(Ok(OffsetCommitProgress::Retry(_))) => { + warn!( + "[MpscConsumer commit] consume-offset generation changed before fenced commit: chan_id={} seq={} producer_id={} consume_offset={} next_consume_offset={} attempt={}", + chan_id, + seq, + producer_id, + consume_offset, + next_consume_offset, + attempts, + ); + } Ok(Err(e)) => { warn!( "[MpscConsumer commit] consume-offset put failed: chan_id={} seq={} producer_id={} consume_offset={} next_consume_offset={} attempt={} err={}", @@ -2267,7 +2504,8 @@ impl MpscConsumer { stage.store(3, Ordering::Relaxed); commit_seq.mark_commit_begin(seq); - let put_trace = MpscConsumer::commit_consume_offset( + let initial_observation = commit_seq.consume_offset_observation(&fetched.producer_id); + let commit_result = MpscConsumer::commit_consume_offset( client, chan_id, global_lease_id, @@ -2275,11 +2513,14 @@ impl MpscConsumer { fetched.consume_offset, seq, shutdown.clone(), + initial_observation, ) .await?; - fetched.etcd_put_latency_ns = put_trace.total_latency_ns; - fetched.etcd_put_first_poll_delay_ns = put_trace.first_poll_delay_ns; - fetched.etcd_put_first_poll_to_ready_ns = put_trace.first_poll_to_ready_ns; + fetched.etcd_put_latency_ns = commit_result.trace.total_latency_ns; + fetched.etcd_put_first_poll_delay_ns = commit_result.trace.first_poll_delay_ns; + fetched.etcd_put_first_poll_to_ready_ns = commit_result.trace.first_poll_to_ready_ns; + commit_seq + .record_consume_offset_generation(&fetched.producer_id, commit_result.generation); stage.store(4, Ordering::Relaxed); commit_seq.mark_ready_to_advance(seq); @@ -2347,9 +2588,10 @@ async fn load_producer_meta_watch_snapshot( client: &mut etcd::Client, chan_id: i64, prefix: &str, + max_retries: u32, ) -> Result, MpscError> { let mut meta_set = HashSet::new(); - scan_etcd_prefix_paginated(client, prefix, |key, _value| { + scan_etcd_prefix_paginated_with_retry(client, prefix, max_retries, |key, _value| { match std::str::from_utf8(key) { Ok(key_str) => { if let Some(idx) = keys::parse_etcd_producer_key(key_str) { @@ -2376,10 +2618,11 @@ async fn refresh_producer_meta_watch_snapshot( prefix: &str, meta_tx: &mpsc::Sender>, shutdown: &ShutdownCtl, + max_retries: u32, ) -> EtcdPrefixWatchLoopControl { let meta_set = match tokio::select! { biased; - res = load_producer_meta_watch_snapshot(client, chan_id, prefix) => res, + res = load_producer_meta_watch_snapshot(client, chan_id, prefix, max_retries) => res, _ = shutdown.wait_closed() => return EtcdPrefixWatchLoopControl::Stop, } { Ok(meta_set) => meta_set, @@ -2466,10 +2709,11 @@ async fn load_produce_offset_watch_snapshot( client: &mut etcd::Client, chan_id: i64, prefix: &str, + max_retries: u32, ) -> Result, MpscError> { let watch_observed_at = Instant::now(); let mut updates = Vec::new(); - scan_etcd_prefix_paginated(client, prefix, |key, value| { + scan_etcd_prefix_paginated_with_retry(client, prefix, max_retries, |key, value| { let key_str = match std::str::from_utf8(key) { Ok(v) => v, Err(e) => { @@ -2551,10 +2795,11 @@ async fn refresh_produce_offset_watch_snapshot( prefix: &str, produce_offset_tx: &mpsc::Sender>, shutdown: &ShutdownCtl, + max_retries: u32, ) -> EtcdPrefixWatchLoopControl { let updates = match tokio::select! { biased; - res = load_produce_offset_watch_snapshot(client, chan_id, prefix) => res, + res = load_produce_offset_watch_snapshot(client, chan_id, prefix, max_retries) => res, _ = shutdown.wait_closed() => return EtcdPrefixWatchLoopControl::Stop, } { Ok(updates) => updates, @@ -2576,6 +2821,7 @@ struct ConsumerActor { instance_id: usize, lease_manager: LeaseManager, client: etcd::Client, + etcd_rpc_max_retries: u32, producer_selector: ProducerSelectorForConsumer, /// payload 回调，由上层通过 ConsumerCmd::SetCallback 设置. payload_cb: Option, @@ -3010,6 +3256,7 @@ impl ConsumerActor { shutdown: ShutdownCtl, category: MqCategory, global_lease_id: i64, + etcd_rpc_max_retries: u32, ) -> ( mpsc::Sender, mpsc::Receiver, @@ -3019,7 +3266,8 @@ impl ConsumerActor { CommitSequencer, ) { let instance_id = NEXT_CONSUMER_INSTANCE_ID.fetch_add(1, Ordering::Relaxed); - let producer_selector = ProducerSelectorForConsumer::new(client.clone(), Some(chan_id)); + let producer_selector = + ProducerSelectorForConsumer::new(client.clone(), Some(chan_id), etcd_rpc_max_retries); let (cmd_tx, cmd_rx) = mpsc::channel(8); let (meta_tx, meta_rx) = mpsc::channel(8); let (produce_offset_tx, produce_offset_rx) = mpsc::channel(128); @@ -3036,6 +3284,7 @@ impl ConsumerActor { instance_id, lease_manager: lease_manager.clone(), client: client.clone(), + etcd_rpc_max_retries, producer_selector, payload_cb: None, prefetch_offset_map: HashMap::new(), @@ -3084,6 +3333,7 @@ impl ConsumerActor { meta_tx, lifecycle.clone(), shutdown.clone(), + etcd_rpc_max_retries, ); ConsumerActor::spawn_produce_offset_watch( client, @@ -3091,6 +3341,7 @@ impl ConsumerActor { produce_offset_tx, lifecycle, shutdown, + etcd_rpc_max_retries, ); ( @@ -3109,6 +3360,7 @@ impl ConsumerActor { meta_tx: mpsc::Sender>, lifecycle: LifecycleView, shutdown: ShutdownCtl, + max_retries: u32, ) { spawn_named( &lifecycle, @@ -3147,6 +3399,7 @@ impl ConsumerActor { &prefix, &meta_tx, &shutdown, + max_retries, ) .await } @@ -3163,6 +3416,7 @@ impl ConsumerActor { &prefix, &meta_tx, &shutdown, + max_retries, ) .await } @@ -3179,6 +3433,7 @@ impl ConsumerActor { produce_offset_tx: mpsc::Sender>, lifecycle: LifecycleView, shutdown: ShutdownCtl, + max_retries: u32, ) { spawn_named( &lifecycle, @@ -3218,6 +3473,7 @@ impl ConsumerActor { &prefix, &produce_offset_tx, &shutdown, + max_retries, ) .await } @@ -3501,6 +3757,7 @@ impl ConsumerActor { let cached_consume_offset = self.cached_consume_offset(&producer_id); let client = self.client.clone(); let chan_id = self.chan_id; + let max_retries = self.etcd_rpc_max_retries; probe_join_set.spawn(async move { ConsumerActor::probe_single_producer_offsets_with_cache( client, @@ -3508,6 +3765,7 @@ impl ConsumerActor { producer_id, cached_produce_offset, cached_consume_offset, + max_retries, ) .await }); @@ -3622,11 +3880,17 @@ impl ConsumerActor { } async fn get_single_offset_optional_with_client( - mut client: etcd::Client, + client: etcd::Client, key: String, offset_name: &'static str, + max_retries: u32, ) -> Result, MpscError> { - let resp = client.get(key.clone(), None).await?; + let resp = retry_etcd_rpc(max_retries, "mq_get_offset", || { + let mut attempt_client = client.clone(); + let attempt_key = key.clone(); + async move { attempt_client.get(attempt_key, None).await } + }) + .await?; let Some(kv) = resp.kvs().first() else { return Ok(None); }; @@ -3650,17 +3914,20 @@ impl ConsumerActor { producer_id: String, cached_produce_offset: i64, cached_consume_offset: i64, + max_retries: u32, ) -> Result<(String, SingleProducerOffsets), MpscError> { let (produce_offset_opt, consume_offset_opt) = tokio::try_join!( ConsumerActor::get_single_offset_optional_with_client( client.clone(), keys::etcd_produce_offset_one_producer_key(chan_id, &producer_id), "produce_offset_of_all_producer", + max_retries, ), ConsumerActor::get_single_offset_optional_with_client( client, keys::etcd_consume_offset_one_producer_key(chan_id, &producer_id), "consume_offset_of_all_producer", + max_retries, ), )?; let produce_offset = merge_monotonic_offset(cached_produce_offset, produce_offset_opt); @@ -3678,24 +3945,29 @@ impl ConsumerActor { let mut client = self.client.clone(); let prefix = keys::etcd_produce_offset_all_producer_prefix(self.chan_id); let mut result = HashMap::new(); - scan_etcd_prefix_paginated(&mut client, &prefix, |key, value| { - let key = std::str::from_utf8(key).map_err(|e| { - MpscError::Internal(format!("invalid utf-8 key in produce_offset: {}", e)) - })?; - if let Some(idx) = key.split('/').last() { - let val_str = std::str::from_utf8(value).map_err(|e| { - MpscError::Internal(format!("invalid utf-8 value in produce_offset: {}", e)) - })?; - let offset: i64 = val_str.parse().map_err(|e| { - MpscError::Internal(format!( - "invalid offset '{}' in produce_offset: {}", - val_str, e - )) + scan_etcd_prefix_paginated_with_retry( + &mut client, + &prefix, + self.etcd_rpc_max_retries, + |key, value| { + let key = std::str::from_utf8(key).map_err(|e| { + MpscError::Internal(format!("invalid utf-8 key in produce_offset: {}", e)) })?; - result.insert(idx.to_string(), offset); - } - Ok::(EtcdPrefixScanAction::Continue) - }) + if let Some(idx) = key.split('/').last() { + let val_str = std::str::from_utf8(value).map_err(|e| { + MpscError::Internal(format!("invalid utf-8 value in produce_offset: {}", e)) + })?; + let offset: i64 = val_str.parse().map_err(|e| { + MpscError::Internal(format!( + "invalid offset '{}' in produce_offset: {}", + val_str, e + )) + })?; + result.insert(idx.to_string(), offset); + } + Ok::(EtcdPrefixScanAction::Continue) + }, + ) .await .map_err(map_prefix_scan_error)?; Ok(result) @@ -3705,24 +3977,29 @@ impl ConsumerActor { let mut client = self.client.clone(); let prefix = keys::etcd_consume_offset_all_producer_prefix(self.chan_id); let mut result = HashMap::new(); - scan_etcd_prefix_paginated(&mut client, &prefix, |key, value| { - let key = std::str::from_utf8(key).map_err(|e| { - MpscError::Internal(format!("invalid utf-8 key in consume_offset: {}", e)) - })?; - if let Some(idx) = key.split('/').last() { - let val_str = std::str::from_utf8(value).map_err(|e| { - MpscError::Internal(format!("invalid utf-8 value in consume_offset: {}", e)) - })?; - let offset: i64 = val_str.parse().map_err(|e| { - MpscError::Internal(format!( - "invalid offset '{}' in consume_offset: {}", - val_str, e - )) + scan_etcd_prefix_paginated_with_retry( + &mut client, + &prefix, + self.etcd_rpc_max_retries, + |key, value| { + let key = std::str::from_utf8(key).map_err(|e| { + MpscError::Internal(format!("invalid utf-8 key in consume_offset: {}", e)) })?; - result.insert(idx.to_string(), offset); - } - Ok::(EtcdPrefixScanAction::Continue) - }) + if let Some(idx) = key.split('/').last() { + let val_str = std::str::from_utf8(value).map_err(|e| { + MpscError::Internal(format!("invalid utf-8 value in consume_offset: {}", e)) + })?; + let offset: i64 = val_str.parse().map_err(|e| { + MpscError::Internal(format!( + "invalid offset '{}' in consume_offset: {}", + val_str, e + )) + })?; + result.insert(idx.to_string(), offset); + } + Ok::(EtcdPrefixScanAction::Continue) + }, + ) .await .map_err(map_prefix_scan_error)?; Ok(result) @@ -3741,15 +4018,20 @@ impl ConsumerActor { let mut client = self.client.clone(); let prefix = keys::etcd_producer_key_prefix(self.chan_id); let mut meta_set = HashSet::new(); - scan_etcd_prefix_paginated(&mut client, &prefix, |key, _value| { - let key_str = std::str::from_utf8(key).map_err(|e| { - MpscError::Internal(format!("invalid utf-8 key in producer meta: {}", e)) - })?; - if let Some(idx) = keys::parse_etcd_producer_key(key_str) { - meta_set.insert(idx); - } - Ok::(EtcdPrefixScanAction::Continue) - }) + scan_etcd_prefix_paginated_with_retry( + &mut client, + &prefix, + self.etcd_rpc_max_retries, + |key, _value| { + let key_str = std::str::from_utf8(key).map_err(|e| { + MpscError::Internal(format!("invalid utf-8 key in producer meta: {}", e)) + })?; + if let Some(idx) = keys::parse_etcd_producer_key(key_str) { + meta_set.insert(idx); + } + Ok::(EtcdPrefixScanAction::Continue) + }, + ) .await .map_err(map_prefix_scan_error)?; self.producer_meta_cache = meta_set; @@ -3762,9 +4044,12 @@ impl ConsumerActor { #[cfg(test)] mod tests { use super::{ - merge_monotonic_offset, merge_offset_cache_monotonic, CommitSequencer, MpscError, - OffsetSnapshotState, ShutdownCtl, + merge_monotonic_offset, merge_offset_cache_monotonic, + reconcile_consumer_membership_bind_observation, + reconcile_consumer_membership_update_observation, CommitSequencer, + ConsumerMembershipGeneration, MpscError, OffsetSnapshotState, ShutdownCtl, }; + use crate::offset_commit::{OffsetGeneration, OffsetObservation}; use std::collections::HashMap; use std::time::Duration; use tokio::sync::oneshot; @@ -3780,6 +4065,73 @@ mod tests { sequencer.mark_popped(seq); } + fn membership_generation( + value: &[u8], + lease_id: i64, + mod_revision: i64, + ) -> ConsumerMembershipGeneration { + ConsumerMembershipGeneration { + value: value.to_vec(), + lease_id, + mod_revision, + } + } + + #[test] + fn consumer_bind_reconciliation_accepts_only_owned_value_and_lease() { + let owned = membership_generation(b"new", 11, 42); + assert_eq!( + reconcile_consumer_membership_bind_observation( + Some(owned.clone()), + "member", + b"new", + 11, + ) + .unwrap(), + owned + ); + assert!(reconcile_consumer_membership_bind_observation( + Some(membership_generation(b"new", 12, 43)), + "member", + b"new", + 11, + ) + .is_err()); + } + + #[test] + fn consumer_update_reconciliation_requires_a_new_owned_generation() { + let previous = membership_generation(b"old", 11, 42); + let committed = membership_generation(b"new", 11, 43); + assert_eq!( + reconcile_consumer_membership_update_observation( + Some(committed.clone()), + "member", + &previous, + b"new", + 11, + ) + .unwrap(), + committed + ); + assert!(reconcile_consumer_membership_update_observation( + Some(membership_generation(b"new", 11, 42)), + "member", + &previous, + b"new", + 11, + ) + .is_err()); + assert!(reconcile_consumer_membership_update_observation( + Some(membership_generation(b"other", 11, 44)), + "member", + &previous, + b"new", + 11, + ) + .is_err()); + } + #[test] fn merge_monotonic_offset_keeps_cached_when_probe_missing() { assert_eq!(merge_monotonic_offset(62, None), 62); @@ -3831,6 +4183,27 @@ mod tests { assert_eq!(consume_cache.get("producer_a"), Some(&5)); } + #[test] + fn commit_sequencer_caches_offset_generations_per_producer() { + let sequencer = CommitSequencer::new(7); + assert_eq!(sequencer.consume_offset_observation("producer_a"), None); + + let first = OffsetGeneration::committed(41, 11, 101).unwrap(); + sequencer.record_consume_offset_generation("producer_a", first.clone()); + assert_eq!( + sequencer.consume_offset_observation("producer_a"), + Some(OffsetObservation::Present(first)) + ); + assert_eq!(sequencer.consume_offset_observation("producer_b"), None); + + let next = OffsetGeneration::committed(42, 11, 102).unwrap(); + sequencer.record_consume_offset_generation("producer_a", next.clone()); + assert_eq!( + sequencer.consume_offset_observation("producer_a"), + Some(OffsetObservation::Present(next)) + ); + } + #[tokio::test(flavor = "current_thread")] async fn commit_wait_turn_wakes_when_prior_sequence_advances() { let sequencer = CommitSequencer::new(1); @@ -4008,10 +4381,11 @@ pub struct ProducerSelectorForConsumer { tomb_first_ts: HashMap, client: etcd::Client, chan_id: Option, + etcd_rpc_max_retries: u32, } impl ProducerSelectorForConsumer { - pub fn new(client: etcd::Client, chan_id: Option) -> Self { + pub fn new(client: etcd::Client, chan_id: Option, etcd_rpc_max_retries: u32) -> Self { Self { producers: Vec::new(), producer_weight_map: HashMap::new(), @@ -4020,6 +4394,7 @@ impl ProducerSelectorForConsumer { tomb_first_ts: HashMap::new(), client, chan_id, + etcd_rpc_max_retries, } } @@ -4195,7 +4570,14 @@ impl ProducerSelectorForConsumer { None => return 1, }; let weight_key = keys::etcd_producer_weight_key(chan_id, producer_idx); - match self.client.get(weight_key.clone(), None).await { + let client = self.client.clone(); + match retry_etcd_rpc(self.etcd_rpc_max_retries, "mq_get_producer_weight", || { + let mut attempt_client = client.clone(); + let attempt_key = weight_key.clone(); + async move { attempt_client.get(attempt_key, None).await } + }) + .await + { Ok(resp) => { if let Some(kv) = resp.kvs().first() { if let Ok(txt) = std::str::from_utf8(kv.value()) { @@ -4233,6 +4615,7 @@ async fn register_consumer_idx( client: &mut etcd::Client, chan_id: i64, lease_id: i64, + max_retries: u32, ) -> Result { use fluxon_util::etcd::DistributeIdAllocator; @@ -4241,7 +4624,8 @@ async fn register_consumer_idx( // consumers. This avoids the fixed 0..1000 range and keeps // allocation monotonic per channel. let prefix = format!("channels/{}/consumers", chan_id); - let allocator = DistributeIdAllocator::new(client.clone(), prefix, lease_id); + let allocator = + DistributeIdAllocator::new_with_retry(client.clone(), prefix, lease_id, max_retries); allocator .allocate_id() diff --git a/fluxon_rs/fluxon_mq/src/create.rs b/fluxon_rs/fluxon_mq/src/create.rs index 676978c..7c8c1fc 100644 --- a/fluxon_rs/fluxon_mq/src/create.rs +++ b/fluxon_rs/fluxon_mq/src/create.rs @@ -2,31 +2,138 @@ use anyhow::Context; use etcd_client as etcd; use std::time::Duration; -use fluxon_util::etcd::{etcd_clients_pool, get_cluster_lease_id, DistributeIdAllocator}; +use fluxon_util::etcd::{ + etcd_clients_pool, get_cluster_lease_id_with_retry, is_transient_etcd_error, retry_etcd_rpc, + DistributeIdAllocator, +}; use fluxon_util::lease_manager::{ record_register_by as lm_record_register_by, registered_etcd_client, LeaseBackendUid, LeaseManager, LeaseRegisterKind, }; use crate::error::MpscError; -use crate::etcd_retry::is_transient_etcd_error; use crate::keys; use crate::manager::{ - get_chan_meta_with_version, ChanGlobalMeta, ChanManager, ChanMetaWithVersion, - MpscError as ManagerMpscError, + etcd_rpc_attempt_limit, get_chan_meta_with_version, get_chan_meta_with_version_with_retry, + ChanGlobalMeta, ChanManager, ChanMetaWithVersion, MpscError as ManagerMpscError, }; use crate::shutdown::ShutdownCtl; const MPMC_SUBCHANNEL_METADATA_TIMEOUT: Duration = Duration::from_secs(5); -const MPMC_SUBCHANNEL_METADATA_ATTEMPTS: usize = 3; const MPMC_SUBCHANNEL_METADATA_RETRY_DELAY: Duration = Duration::from_millis(100); +#[derive(Debug, Clone, PartialEq, Eq)] +struct StoredEtcdGeneration { + value: Vec, + lease_id: i64, + mod_revision: i64, +} + +impl StoredEtcdGeneration { + fn from_kv(kv: &etcd::KeyValue, expected_key: &str) -> anyhow::Result { + if kv.key() != expected_key.as_bytes() { + anyhow::bail!( + "etcd readback returned an unexpected key for {}", + expected_key + ); + } + Ok(Self { + value: kv.value().to_vec(), + lease_id: kv.lease(), + mod_revision: kv.mod_revision(), + }) + } +} + +fn txn_single_get_generation( + txn_res: &etcd::TxnResponse, + key: &str, + operation: &str, +) -> anyhow::Result> { + let responses = txn_res.op_responses(); + let [etcd::TxnOpResponse::Get(get)] = responses.as_slice() else { + anyhow::bail!( + "{} readback returned an invalid response shape: operations={}", + operation, + responses.len() + ); + }; + if get.kvs().len() > 1 { + anyhow::bail!("{} readback returned duplicate exact keys", operation); + } + get.kvs() + .first() + .map(|kv| StoredEtcdGeneration::from_kv(kv, key)) + .transpose() +} + +fn exact_meta_compares(key: &str, meta: &ChanMetaWithVersion) -> Vec { + vec![ + etcd::Compare::mod_revision(key, etcd::CompareOp::Equal, meta.mod_revision), + etcd::Compare::lease(key, etcd::CompareOp::Equal, meta.lease_id), + etcd::Compare::value(key, etcd::CompareOp::Equal, meta.raw_value.clone()), + ] +} + +fn reconcile_channel_meta_cleanup_observation( + observed: Option, + expected: &StoredEtcdGeneration, +) -> anyhow::Result> { + match observed { + None => Ok(None), + Some(current) if ¤t != expected => Ok(Some(current.mod_revision)), + Some(_) => anyhow::bail!( + "channel meta cleanup compare was false although the exact generation still exists" + ), + } +} + +async fn cleanup_exact_channel_meta( + client: &mut etcd::Client, + key: &str, + meta: &ChanMetaWithVersion, + max_retries: u32, +) -> anyhow::Result<()> { + let txn_res = retry_etcd_rpc(max_retries, "mq_cleanup_channel_meta", || { + let mut attempt_client = client.clone(); + let txn = etcd::Txn::new() + .when(exact_meta_compares(key, meta)) + .and_then(vec![etcd::TxnOp::delete(key, None)]) + .or_else(vec![etcd::TxnOp::get(key, None)]); + async move { attempt_client.txn(txn).await } + }) + .await?; + if txn_res.succeeded() { + return Ok(()); + } + + let observed = txn_single_get_generation(&txn_res, key, "channel meta cleanup")?; + let expected = StoredEtcdGeneration { + value: meta.raw_value.clone(), + lease_id: meta.lease_id, + mod_revision: meta.mod_revision, + }; + if let Some(current_mod_revision) = + reconcile_channel_meta_cleanup_observation(observed, &expected)? + { + tracing::warn!( + meta_key = key, + expected_mod_revision = expected.mod_revision, + current_mod_revision, + "channel meta cleanup skipped a different generation" + ); + } + Ok(()) +} + async fn load_mpmc_subchannel_metadata( client: &mut etcd::Client, chan_id: i64, shutdown: &ShutdownCtl, + etcd_rpc_max_retries: u32, ) -> anyhow::Result { - for attempt in 1..=MPMC_SUBCHANNEL_METADATA_ATTEMPTS { + let max_attempts = etcd_rpc_attempt_limit(etcd_rpc_max_retries); + for attempt in 1..=max_attempts { let attempt_started = std::time::Instant::now(); let result = tokio::select! { biased; @@ -61,7 +168,7 @@ async fn load_mpmc_subchannel_metadata( ), }; - if attempt == MPMC_SUBCHANNEL_METADATA_ATTEMPTS { + if attempt == max_attempts { anyhow::bail!( "get_chan_meta failed after {} attempts for chan_id={}: {}", attempt, @@ -73,7 +180,7 @@ async fn load_mpmc_subchannel_metadata( tracing::warn!( chan_id, attempt, - max_attempts = MPMC_SUBCHANNEL_METADATA_ATTEMPTS, + max_attempts, elapsed_ms = attempt_started.elapsed().as_millis(), reason = %retry_reason, "Retrying MPMC subchannel metadata read" @@ -98,6 +205,8 @@ pub struct ChanCreateConfig { pub capacity: i64, pub ttl_seconds: i64, pub weight: Option, + /// Number of unary etcd retries after the initial attempt. + pub etcd_rpc_max_retries: u32, /// Optional override for the channel-level global lease id. /// /// When present, `create_mpsc_channel` will reuse this lease id @@ -158,7 +267,11 @@ pub async fn create_mpsc_channel( // This top-level counter must remain monotonic across idle windows, so its // etcd key must not be tied to a short-lived lease. Old Python behavior // intentionally left this key unleased for the same reason. - let allocator = DistributeIdAllocator::new_without_lease(client.clone(), "channels"); + let allocator = DistributeIdAllocator::new_without_lease_with_retry( + client.clone(), + "channels", + cfg.etcd_rpc_max_retries, + ); let chan_id = allocator .allocate_id() .await @@ -171,10 +284,11 @@ pub async fn create_mpsc_channel( // `/channels/{chan_id}/next_producer_id` alive. let global_lease_id = match cfg.override_global_lease_id { Some(id) => id, - None => get_cluster_lease_id( + None => get_cluster_lease_id_with_retry( &mut client, &format!("channels/{}", chan_id), cfg.ttl_seconds, + cfg.etcd_rpc_max_retries, ) .await .map_err(|e| MpscError::Internal(format!("get_cluster_lease_id(meta) failed: {}", e)))?, @@ -184,10 +298,11 @@ pub async fn create_mpsc_channel( // allocator; this follows the design in `mpsc.md` where every // global chan owns a long-lived lease to guard id allocation // against short-term stale reads. - let global_long_lease_id = get_cluster_lease_id( + let global_long_lease_id = get_cluster_lease_id_with_retry( &mut client, &format!("id_allocator/channels/{}", chan_id), 30 * 60, + cfg.etcd_rpc_max_retries, ) .await .map_err(|e| { @@ -352,26 +467,43 @@ pub async fn create_mpsc_channel( let meta_key = keys::etcd_meta_key(chan_id); - let compare = etcd::Compare::create_revision(meta_key.clone(), etcd::CompareOp::Equal, 0); - let put_meta = etcd::TxnOp::put( - meta_key.clone(), - meta_bytes, - Some(etcd::PutOptions::new().with_lease(global_lease_id)), - ); - let txn = etcd::Txn::new() - .when(vec![compare]) - .and_then(vec![put_meta]); - let txn_res = client - .txn(txn) - .await - .with_context(|| format!("failed to write meta for chan_id={}", chan_id)) - .map_err(|e| MpscError::Internal(e.to_string()))?; + let txn_res = retry_etcd_rpc(cfg.etcd_rpc_max_retries, "mq_create_channel_meta", || { + let mut attempt_client = client.clone(); + let compare = etcd::Compare::create_revision(meta_key.clone(), etcd::CompareOp::Equal, 0); + let put_meta = etcd::TxnOp::put( + meta_key.clone(), + meta_bytes.clone(), + Some(etcd::PutOptions::new().with_lease(global_lease_id)), + ); + let txn = etcd::Txn::new() + .when(vec![compare]) + .and_then(vec![put_meta]) + .or_else(vec![etcd::TxnOp::get(meta_key.clone(), None)]); + async move { attempt_client.txn(txn).await } + }) + .await + .with_context(|| format!("failed to write meta for chan_id={}", chan_id)) + .map_err(|e| MpscError::Internal(e.to_string()))?; if !txn_res.succeeded() { - return Err(MpscError::Internal(format!( - "meta key already exists for chan_id={}", - chan_id - ))); + let observed = txn_single_get_generation(&txn_res, &meta_key, "channel meta publish") + .map_err(|e| MpscError::Internal(e.to_string()))?; + match observed { + Some(current) if current.value == meta_bytes && current.lease_id == global_lease_id => { + } + Some(_) => { + return Err(MpscError::Internal(format!( + "meta key already exists with conflicting state for chan_id={}", + chan_id + ))); + } + None => { + return Err(MpscError::Internal(format!( + "meta key disappeared while reconciling create for chan_id={}", + chan_id + ))); + } + } } let etcd_client = client; @@ -385,6 +517,7 @@ pub async fn create_mpsc_channel( global_lease: global_lease_handle, global_long_lease: global_long_lease_handle, payload_lease: payload_lease_handle, + etcd_rpc_max_retries: cfg.etcd_rpc_max_retries, etcd_client, }) } @@ -404,6 +537,7 @@ impl ChanManager { override_global_lease_id: i64, override_member_lease_id: i64, override_payload_lease_id: i64, + etcd_rpc_max_retries: u32, rt_handle: tokio::runtime::Handle, shutdown: ShutdownCtl, ) -> anyhow::Result { @@ -415,8 +549,13 @@ impl ChanManager { ) })?; let mut meta_client = client.clone(); - let meta_with_ver = - load_mpmc_subchannel_metadata(&mut meta_client, chan_id, &shutdown).await?; + let meta_with_ver = load_mpmc_subchannel_metadata( + &mut meta_client, + chan_id, + &shutdown, + etcd_rpc_max_retries, + ) + .await?; let meta = meta_with_ver.meta; if meta.global_lease_id != override_global_lease_id { @@ -541,6 +680,7 @@ impl ChanManager { global_lease, global_long_lease, payload_lease, + etcd_rpc_max_retries, etcd_client: client, }) } @@ -554,6 +694,7 @@ impl ChanManager { etcd_endpoints: Vec, kv_backend_uid: LeaseBackendUid, chan_id: i64, + etcd_rpc_max_retries: u32, rt_handle: tokio::runtime::Handle, ) -> anyhow::Result { fn is_hard_lease_failure(err: &anyhow::Error) -> bool { @@ -573,11 +714,13 @@ impl ChanManager { .await .map_err(|e| anyhow::anyhow!("connect etcd failed: {}", e))?; let mut meta_client = client.clone(); - let meta_with_ver = get_chan_meta_with_version(&mut meta_client, chan_id) - .await - .map_err(|e| anyhow::anyhow!("get_chan_meta failed for chan_id={}: {}", chan_id, e))?; - let meta = meta_with_ver.meta; - let meta_version = meta_with_ver.version; + let meta_with_ver = + get_chan_meta_with_version_with_retry(&mut meta_client, chan_id, etcd_rpc_max_retries) + .await + .map_err(|e| { + anyhow::anyhow!("get_chan_meta failed for chan_id={}: {}", chan_id, e) + })?; + let meta = meta_with_ver.meta.clone(); let meta_key = keys::etcd_meta_key(chan_id); // 2) 恢复 global lease handle：直接使用元数据中记录的 @@ -596,8 +739,12 @@ impl ChanManager { Err(e) => { if is_hard_lease_failure(&e) { let mut cleanup_client = client.clone(); - cleanup_client - .delete(meta_key.clone(), None) + cleanup_exact_channel_meta( + &mut cleanup_client, + &meta_key, + &meta_with_ver, + etcd_rpc_max_retries, + ) .await .map_err(|de| anyhow::anyhow!( "hard lease failure detected; failed to delete meta_key={} for chan_id={}: delete_err={}, cause={}", @@ -632,8 +779,12 @@ impl ChanManager { Err(e) => { if is_hard_lease_failure(&e) { let mut cleanup_client = client.clone(); - cleanup_client - .delete(meta_key.clone(), None) + cleanup_exact_channel_meta( + &mut cleanup_client, + &meta_key, + &meta_with_ver, + etcd_rpc_max_retries, + ) .await .map_err(|de| anyhow::anyhow!( "hard lease failure detected; failed to delete meta_key={} for chan_id={}: delete_err={}, cause={}", @@ -671,17 +822,24 @@ impl ChanManager { // get_chan_meta 读取到 meta 到这里期间，该 meta 没有被 // 删除或更新。如果校验失败，则直接返回错误，由上层 // 决定是否重试。 - let compare = - etcd::Compare::version(meta_key.clone(), etcd::CompareOp::Equal, meta_version); - let get_op = etcd::TxnOp::get(meta_key.clone(), None); - let txn = etcd::Txn::new().when(vec![compare]).and_then(vec![get_op]); - let txn_res = client2.txn(txn).await.map_err(|e| { - anyhow::anyhow!("meta version check failed for chan_id={}: {}", chan_id, e) - })?; + let txn_res = retry_etcd_rpc(etcd_rpc_max_retries, "mq_check_channel_meta", || { + let mut attempt_client = client2.clone(); + let get_op = etcd::TxnOp::get(meta_key.clone(), None); + let txn = etcd::Txn::new() + .when(exact_meta_compares(&meta_key, &meta_with_ver)) + .and_then(vec![get_op]) + .or_else(vec![etcd::TxnOp::get(meta_key.clone(), None)]); + async move { attempt_client.txn(txn).await } + }) + .await + .map_err(|e| anyhow::anyhow!("meta version check failed for chan_id={}: {}", chan_id, e))?; if !txn_res.succeeded() { + let observed = + txn_single_get_generation(&txn_res, &meta_key, "channel meta bootstrap check")?; anyhow::bail!( - "channel meta changed or deleted during ChanManager bootstrap, chan_id={}", - chan_id + "channel meta changed or deleted during ChanManager bootstrap, chan_id={}, current_mod_revision={:?}", + chan_id, + observed.map(|generation| generation.mod_revision) ); } @@ -759,7 +917,42 @@ impl ChanManager { global_lease, global_long_lease, payload_lease, + etcd_rpc_max_retries, etcd_client: client, }) } } + +#[cfg(test)] +mod tests { + use super::{reconcile_channel_meta_cleanup_observation, StoredEtcdGeneration}; + + fn generation(mod_revision: i64) -> StoredEtcdGeneration { + StoredEtcdGeneration { + value: b"meta".to_vec(), + lease_id: 11, + mod_revision, + } + } + + #[test] + fn hard_failure_cleanup_accepts_absent_or_newer_generation() { + let expected = generation(42); + assert_eq!( + reconcile_channel_meta_cleanup_observation(None, &expected).unwrap(), + None + ); + assert_eq!( + reconcile_channel_meta_cleanup_observation(Some(generation(43)), &expected).unwrap(), + Some(43) + ); + } + + #[test] + fn hard_failure_cleanup_rejects_false_compare_for_exact_generation() { + let expected = generation(42); + assert!( + reconcile_channel_meta_cleanup_observation(Some(expected.clone()), &expected).is_err() + ); + } +} diff --git a/fluxon_rs/fluxon_mq/src/etcd_retry.rs b/fluxon_rs/fluxon_mq/src/etcd_retry.rs deleted file mode 100644 index 898c027..0000000 --- a/fluxon_rs/fluxon_mq/src/etcd_retry.rs +++ /dev/null @@ -1,43 +0,0 @@ -use etcd_client::Error; -use tonic::Code; - -pub(crate) fn is_transient_etcd_error(error: &Error) -> bool { - match error { - Error::IoError(_) | Error::TransportError(_) => true, - Error::GRpcStatus(status) => matches!( - status.code(), - Code::Aborted | Code::DeadlineExceeded | Code::ResourceExhausted | Code::Unavailable - ), - Error::InvalidArgs(_) - | Error::InvalidUri(_) - | Error::WatchError(_) - | Error::Utf8Error(_) - | Error::LeaseKeepAliveError(_) - | Error::ElectError(_) - | Error::InvalidHeaderValue(_) - | Error::EndpointError(_) => false, - } -} - -#[cfg(test)] -mod tests { - use super::is_transient_etcd_error; - use etcd_client::Error; - use tonic::Status; - - #[test] - fn classifies_only_transient_grpc_statuses_for_retry() { - assert!(is_transient_etcd_error(&Error::GRpcStatus( - Status::unavailable("etcdserver: request timed out") - ))); - assert!(is_transient_etcd_error(&Error::GRpcStatus( - Status::deadline_exceeded("deadline") - ))); - assert!(!is_transient_etcd_error(&Error::GRpcStatus( - Status::invalid_argument("bad key") - ))); - assert!(!is_transient_etcd_error(&Error::InvalidArgs( - "bad options".to_string() - ))); - } -} diff --git a/fluxon_rs/fluxon_mq/src/lib.rs b/fluxon_rs/fluxon_mq/src/lib.rs index 292f2fc..16a7dc6 100644 --- a/fluxon_rs/fluxon_mq/src/lib.rs +++ b/fluxon_rs/fluxon_mq/src/lib.rs @@ -1,13 +1,13 @@ pub mod consumer; pub mod create; pub mod error; -mod etcd_retry; pub mod framework; pub mod keys; pub mod lease_manager; pub mod lifecycle; pub mod manager; pub mod nonblocking_monitor; +mod offset_commit; pub mod producer; pub mod shutdown; diff --git a/fluxon_rs/fluxon_mq/src/manager.rs b/fluxon_rs/fluxon_mq/src/manager.rs index b6d581a..6aa296a 100644 --- a/fluxon_rs/fluxon_mq/src/manager.rs +++ b/fluxon_rs/fluxon_mq/src/manager.rs @@ -1,12 +1,9 @@ use crate::keys; -use serde::{Deserialize, Serialize}; -use thiserror::Error; -use tracing::debug; - -use anyhow::Context; use etcd_client as etcd; -use fluxon_util::etcd::DistributeIdAllocator; +use fluxon_util::etcd::retry_etcd_rpc; use fluxon_util::lease_manager::{GeneralLease, LeaseBackendUid, LeaseManager}; +use serde::{Deserialize, Serialize}; +use thiserror::Error; /// Initial produce offset when no messages have been produced. pub const PRODUCE_OFFSET_BEGIN: i64 = -1; @@ -15,6 +12,10 @@ pub const CONSUME_OFFSET_BEGIN: i64 = 0; /// Minimum supported TTL for MQ metadata/member leases. pub const MIN_TTL_SECONDS: i64 = 90; +pub(crate) const fn etcd_rpc_attempt_limit(max_retries: u32) -> u64 { + max_retries as u64 + 1 +} + /// Channel type. #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] #[serde(rename_all = "lowercase")] @@ -121,32 +122,29 @@ pub enum MpscError { InvalidUtf8 { chan_id: i64 }, } -/// Helper result for channel meta that also carries the etcd version -/// of the meta key. 版本用于在后续步骤中做事务校验，确保在 -/// 使用 meta 构造 ChanManager 期间，该 meta 没有被删除或修改。 +/// Channel metadata plus its exact etcd generation. +/// +/// The raw value, lease and modification revision fence bootstrap checks and +/// hard-failure cleanup so a replay cannot validate or delete a newer key. +#[derive(Debug, Clone)] pub struct ChanMetaWithVersion { pub meta: ChanGlobalMeta, pub version: i64, + pub mod_revision: i64, + pub lease_id: i64, + pub raw_value: Vec, } -/// Get channel meta and its etcd version for the given channel id. -/// -/// 这是一个无对象的辅助函数，仅依赖 etcd client 和 chan_id， -/// 不再挂在 ChanManager 上，避免为了读取 meta 专门构造一个 -/// 临时的 ChanManager 实例。 -pub async fn get_chan_meta_with_version( - client: &mut etcd::Client, +fn chan_meta_from_get_response( + resp: &etcd::GetResponse, chan_id: i64, ) -> Result { - let key = keys::etcd_meta_key(chan_id); - let resp = client.get(key, None).await?; - let kvs = resp.kvs(); - let kv = match kvs.first() { + let kv = match resp.kvs().first() { Some(kv) => kv, None => return Err(MpscError::ChanMetaNotFound(chan_id)), }; - let value = kv.value(); - let meta: ChanGlobalMeta = serde_json::from_slice(value) + let raw_value = kv.value().to_vec(); + let meta: ChanGlobalMeta = serde_json::from_slice(&raw_value) .map_err(|source| MpscError::InvalidChanMeta { chan_id, source })?; if meta.ttl_seconds < MIN_TTL_SECONDS { return Err(MpscError::InvalidTtl { @@ -154,8 +152,43 @@ pub async fn get_chan_meta_with_version( ttl_seconds: meta.ttl_seconds, }); } - let version = kv.version(); - Ok(ChanMetaWithVersion { meta, version }) + Ok(ChanMetaWithVersion { + meta, + version: kv.version(), + mod_revision: kv.mod_revision(), + lease_id: kv.lease(), + raw_value, + }) +} + +/// Get channel meta and its etcd version for the given channel id. +/// +/// 这是一个无对象的辅助函数，仅依赖 etcd client 和 chan_id， +/// 不再挂在 ChanManager 上，避免为了读取 meta 专门构造一个 +/// 临时的 ChanManager 实例。 +pub async fn get_chan_meta_with_version( + client: &mut etcd::Client, + chan_id: i64, +) -> Result { + let key = keys::etcd_meta_key(chan_id); + let resp = client.get(key, None).await?; + chan_meta_from_get_response(&resp, chan_id) +} + +/// Get channel metadata with configured retries for transient unary RPC failures. +pub async fn get_chan_meta_with_version_with_retry( + client: &mut etcd::Client, + chan_id: i64, + max_retries: u32, +) -> Result { + let key = keys::etcd_meta_key(chan_id); + let resp = retry_etcd_rpc(max_retries, "mq_get_channel_meta", || { + let mut attempt_client = client.clone(); + let attempt_key = key.clone(); + async move { attempt_client.get(attempt_key, None).await } + }) + .await?; + chan_meta_from_get_response(&resp, chan_id) } /// Get channel meta for the given channel id, without exposing etcd @@ -169,6 +202,17 @@ pub async fn get_chan_meta( Ok(meta) } +/// Get channel metadata with configured retries for transient unary RPC failures. +pub async fn get_chan_meta_with_retry( + client: &mut etcd::Client, + chan_id: i64, + max_retries: u32, +) -> Result { + let ChanMetaWithVersion { meta, .. } = + get_chan_meta_with_version_with_retry(client, chan_id, max_retries).await?; + Ok(meta) +} + /// Channel manager that operates on etcd metadata and cooperates with /// the shared endpoints-scoped `LeaseManager` for lease registration. /// @@ -206,6 +250,7 @@ pub struct ChanManager { /// 决定好 payload lease id，并通过 LeaseManager 注册 /// 对应的 kvclient keepalive；此处始终持有一个有效句柄。 pub payload_lease: GeneralLease, + pub(crate) etcd_rpc_max_retries: u32, pub(crate) etcd_client: etcd::Client, } @@ -227,4 +272,21 @@ impl ChanManager { pub fn member_lease_id(&self) -> i64 { self.member_lease.id() as i64 } + + /// Number of unary etcd retries after the initial attempt. + pub fn etcd_rpc_max_retries(&self) -> u32 { + self.etcd_rpc_max_retries + } +} + +#[cfg(test)] +mod tests { + use super::etcd_rpc_attempt_limit; + + #[test] + fn retry_limit_counts_attempts_without_usize_overflow() { + assert_eq!(etcd_rpc_attempt_limit(0), 1); + assert_eq!(etcd_rpc_attempt_limit(2), 3); + assert_eq!(etcd_rpc_attempt_limit(u32::MAX), u64::from(u32::MAX) + 1); + } } diff --git a/fluxon_rs/fluxon_mq/src/offset_commit.rs b/fluxon_rs/fluxon_mq/src/offset_commit.rs new file mode 100644 index 0000000..e870bfb --- /dev/null +++ b/fluxon_rs/fluxon_mq/src/offset_commit.rs @@ -0,0 +1,402 @@ +use etcd_client as etcd; + +use crate::error::MpscError; + +#[derive(Clone, Debug, PartialEq, Eq)] +pub(crate) struct OffsetGeneration { + value: Vec, + offset: i64, + lease_id: i64, + mod_revision: i64, +} + +impl OffsetGeneration { + fn from_kv(kv: &etcd::KeyValue, expected_key: &str) -> Result { + if kv.key() != expected_key.as_bytes() { + return Err(MpscError::Internal(format!( + "offset read returned an unexpected key for {}", + expected_key + ))); + } + let value = kv.value().to_vec(); + let text = std::str::from_utf8(&value).map_err(|error| { + MpscError::Internal(format!( + "offset key {} contains invalid UTF-8: {}", + expected_key, error + )) + })?; + let offset = text.parse::().map_err(|error| { + MpscError::Internal(format!( + "offset key {} contains invalid value {:?}: {}", + expected_key, text, error + )) + })?; + Ok(Self { + value, + offset, + lease_id: kv.lease(), + mod_revision: kv.mod_revision(), + }) + } + + pub(crate) fn committed( + offset: i64, + lease_id: i64, + mod_revision: i64, + ) -> Result { + if mod_revision <= 0 { + return Err(MpscError::Internal(format!( + "offset commit returned invalid revision {}", + mod_revision + ))); + } + Ok(Self { + value: offset.to_string().into_bytes(), + offset, + lease_id, + mod_revision, + }) + } + + fn from_successful_txn( + response: &etcd::TxnResponse, + offset: i64, + lease_id: i64, + ) -> Result { + let mod_revision = response + .header() + .ok_or_else(|| MpscError::Internal("offset commit response has no header".to_string()))? + .revision(); + Self::committed(offset, lease_id, mod_revision) + } +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub(crate) enum OffsetObservation { + Absent, + Present(OffsetGeneration), +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub(crate) enum OffsetCommitProgress { + Complete(OffsetGeneration), + Retry(OffsetObservation), +} + +fn reconcile_offset_observation( + target: i64, + expected_lease_id: i64, + observed: OffsetObservation, +) -> Result { + match observed { + OffsetObservation::Present(generation) if generation.lease_id != expected_lease_id => { + Err(MpscError::Internal(format!( + "offset generation lease mismatch: expected={} actual={} offset={}", + expected_lease_id, generation.lease_id, generation.offset + ))) + } + OffsetObservation::Present(generation) if generation.offset >= target => { + Ok(OffsetCommitProgress::Complete(generation)) + } + observation => Ok(OffsetCommitProgress::Retry(observation)), + } +} + +#[derive(Clone)] +struct FencedOffsetTxn { + observation: OffsetObservation, + txn: etcd::Txn, +} + +impl FencedOffsetTxn { + fn new(key: &str, target: i64, lease_id: i64, observation: OffsetObservation) -> Self { + let compares = match &observation { + OffsetObservation::Present(generation) => vec![ + etcd::Compare::mod_revision(key, etcd::CompareOp::Equal, generation.mod_revision), + etcd::Compare::lease(key, etcd::CompareOp::Equal, generation.lease_id), + etcd::Compare::value(key, etcd::CompareOp::Equal, generation.value.clone()), + ], + OffsetObservation::Absent => vec![etcd::Compare::create_revision( + key, + etcd::CompareOp::Equal, + 0, + )], + }; + let put = etcd::TxnOp::put( + key, + target.to_string(), + Some(etcd::PutOptions::new().with_lease(lease_id)), + ); + let txn = etcd::Txn::new() + .when(compares) + .and_then(vec![put]) + .or_else(vec![etcd::TxnOp::get(key, None)]); + Self { observation, txn } + } + + fn still_matches(&self, observed: &OffsetObservation) -> bool { + &self.observation == observed + } +} + +enum OffsetCommitState { + NeedObservation, + Ready(FencedOffsetTxn), + Complete(OffsetGeneration), +} + +/// Generation-fenced monotonic offset commit state. +/// +/// A failed or timed-out mutation attempt leaves the ready transaction unchanged, +/// so the caller can replay the same fence. Once another generation is observed, +/// the next attempt is fenced against that exact generation instead. +pub(crate) struct MonotonicOffsetCommit { + key: String, + target: i64, + lease_id: i64, + state: OffsetCommitState, +} + +impl MonotonicOffsetCommit { + pub(crate) fn new( + key: String, + target: i64, + lease_id: i64, + initial_observation: Option, + ) -> Result { + let mut commit = Self { + key, + target, + lease_id, + state: OffsetCommitState::NeedObservation, + }; + if let Some(observation) = initial_observation { + commit.seed_cached_observation(observation)?; + } + Ok(commit) + } + + fn seed_cached_observation(&mut self, observation: OffsetObservation) -> Result<(), MpscError> { + match reconcile_offset_observation(self.target, self.lease_id, observation)? { + OffsetCommitProgress::Complete(_) => { + // A cache can seed a compare, but cannot prove that an already + // satisfied generation still exists at the time of this call. + self.state = OffsetCommitState::NeedObservation; + } + OffsetCommitProgress::Retry(observation) => { + self.state = OffsetCommitState::Ready(FencedOffsetTxn::new( + &self.key, + self.target, + self.lease_id, + observation, + )); + } + } + Ok(()) + } + + fn install_observation( + &mut self, + observation: OffsetObservation, + ) -> Result { + let progress = reconcile_offset_observation(self.target, self.lease_id, observation)?; + self.state = match &progress { + OffsetCommitProgress::Complete(generation) => { + OffsetCommitState::Complete(generation.clone()) + } + OffsetCommitProgress::Retry(observation) => OffsetCommitState::Ready( + FencedOffsetTxn::new(&self.key, self.target, self.lease_id, observation.clone()), + ), + }; + Ok(progress) + } + + /// Performs one bounded convergence attempt. + /// + /// Without an initial observation, the first call reads the exact generation + /// before issuing the mutation. Later calls replay the retained fenced + /// transaction unless a successful else-Get proves that the state changed. + pub(crate) async fn attempt( + &mut self, + client: &mut etcd::Client, + ) -> Result { + if matches!(self.state, OffsetCommitState::NeedObservation) { + let response = client.get(self.key.clone(), None).await?; + let observed = exact_observation_from_get(&response, &self.key)?; + if let progress @ OffsetCommitProgress::Complete(_) = + self.install_observation(observed)? + { + return Ok(progress); + } + } + + let fenced = match &self.state { + OffsetCommitState::Ready(fenced) => fenced.clone(), + OffsetCommitState::Complete(generation) => { + return Ok(OffsetCommitProgress::Complete(generation.clone())); + } + OffsetCommitState::NeedObservation => { + unreachable!("offset commit must have an observation after its initial read") + } + }; + let response = client.txn(fenced.txn.clone()).await?; + if response.succeeded() { + let generation = + OffsetGeneration::from_successful_txn(&response, self.target, self.lease_id)?; + self.state = OffsetCommitState::Complete(generation.clone()); + return Ok(OffsetCommitProgress::Complete(generation)); + } + + let observed = exact_observation_from_txn_readback(&response, &self.key)?; + if fenced.still_matches(&observed) { + return Err(MpscError::Internal(format!( + "offset transaction compare was false although generation still matches key {}", + self.key + ))); + } + + self.install_observation(observed) + } +} + +fn exact_observation_from_get( + response: &etcd::GetResponse, + key: &str, +) -> Result { + if response.kvs().len() > 1 { + return Err(MpscError::Internal(format!( + "exact offset read returned duplicate keys for {}", + key + ))); + } + Ok( + match response + .kvs() + .first() + .map(|kv| OffsetGeneration::from_kv(kv, key)) + .transpose()? + { + Some(generation) => OffsetObservation::Present(generation), + None => OffsetObservation::Absent, + }, + ) +} + +fn exact_observation_from_txn_readback( + response: &etcd::TxnResponse, + key: &str, +) -> Result { + let responses = response.op_responses(); + let [etcd::TxnOpResponse::Get(get)] = responses.as_slice() else { + return Err(MpscError::Internal(format!( + "offset transaction readback returned an invalid response shape for {}: operations={}", + key, + responses.len() + ))); + }; + exact_observation_from_get(get, key) +} + +#[cfg(test)] +mod tests { + use super::{ + reconcile_offset_observation, FencedOffsetTxn, MonotonicOffsetCommit, OffsetCommitProgress, + OffsetCommitState, OffsetGeneration, OffsetObservation, + }; + + fn generation(offset: i64, mod_revision: i64) -> OffsetGeneration { + OffsetGeneration { + value: offset.to_string().into_bytes(), + offset, + lease_id: 11, + mod_revision, + } + } + + #[test] + fn lower_offset_never_counts_as_converged() { + let current = generation(40, 7); + assert_eq!( + reconcile_offset_observation(41, 11, OffsetObservation::Present(current.clone())) + .unwrap(), + OffsetCommitProgress::Retry(OffsetObservation::Present(current)) + ); + } + + #[test] + fn equal_or_higher_offset_is_already_converged() { + assert_eq!( + reconcile_offset_observation(41, 11, OffsetObservation::Present(generation(41, 7))) + .unwrap(), + OffsetCommitProgress::Complete(generation(41, 7)) + ); + assert_eq!( + reconcile_offset_observation(41, 11, OffsetObservation::Present(generation(42, 8))) + .unwrap(), + OffsetCommitProgress::Complete(generation(42, 8)) + ); + } + + #[test] + fn absent_offset_requires_a_create_fenced_commit() { + assert_eq!( + reconcile_offset_observation(41, 11, OffsetObservation::Absent).unwrap(), + OffsetCommitProgress::Retry(OffsetObservation::Absent) + ); + } + + #[test] + fn lower_offset_with_foreign_lease_fails_closed() { + let mut foreign = generation(40, 7); + foreign.lease_id = 12; + let error = + reconcile_offset_observation(41, 11, OffsetObservation::Present(foreign)).unwrap_err(); + assert!(error.to_string().contains("lease mismatch")); + } + + #[test] + fn higher_offset_with_foreign_lease_fails_closed() { + let mut foreign = generation(42, 8); + foreign.lease_id = 12; + let error = + reconcile_offset_observation(41, 11, OffsetObservation::Present(foreign)).unwrap_err(); + assert!(error.to_string().contains("lease mismatch")); + } + + #[test] + fn old_transaction_fence_does_not_match_a_new_generation() { + let old = generation(40, 7); + let fenced = + FencedOffsetTxn::new("offset", 41, 11, OffsetObservation::Present(old.clone())); + assert!(fenced.still_matches(&OffsetObservation::Present(old))); + + let newer = generation(41, 8); + assert!(!fenced.still_matches(&OffsetObservation::Present(newer))); + assert!(!fenced.still_matches(&OffsetObservation::Absent)); + } + + #[test] + fn known_absence_seeds_a_fenced_txn_without_an_initial_read() { + let commit = MonotonicOffsetCommit::new( + "offset".to_string(), + 41, + 11, + Some(OffsetObservation::Absent), + ) + .unwrap(); + assert!(matches!(commit.state, OffsetCommitState::Ready(_))); + } + + #[test] + fn cached_current_generation_requires_current_readback() { + let current = generation(41, 7); + let commit = MonotonicOffsetCommit::new( + "offset".to_string(), + 41, + 11, + Some(OffsetObservation::Present(current.clone())), + ) + .unwrap(); + assert!(matches!(commit.state, OffsetCommitState::NeedObservation)); + } +} diff --git a/fluxon_rs/fluxon_mq/src/producer.rs b/fluxon_rs/fluxon_mq/src/producer.rs index 04a4b7a..b439bbf 100644 --- a/fluxon_rs/fluxon_mq/src/producer.rs +++ b/fluxon_rs/fluxon_mq/src/producer.rs @@ -1,6 +1,8 @@ use anyhow::{Context, Result}; use etcd_client as etcd; -use fluxon_commu::{scan_etcd_prefix_paginated, EtcdPrefixScanAction, EtcdPrefixScanError}; +use fluxon_commu::{ + scan_etcd_prefix_paginated_with_retry, EtcdPrefixScanAction, EtcdPrefixScanError, +}; use serde::{Deserialize, Serialize}; use std::sync::Arc; @@ -13,20 +15,23 @@ use fluxon_observability::keys::{ }; use fluxon_observability::metrics_actor::MetricsHandle as ObserveMetricsHandle; use fluxon_util::etcd::{ - run_prefix_watch_loop, DistributeIdAllocator, EtcdPrefixWatchLoopControl, - ETCD_PREFIX_WATCH_RESTART_SLEEP, + is_transient_etcd_error, run_prefix_watch_loop, DistributeIdAllocator, + EtcdPrefixWatchLoopControl, ETCD_PREFIX_WATCH_RESTART_SLEEP, }; use fluxon_util::lease_manager::LeaseManager; use fluxon_util::prom_remote_write::{Label, Sample, TimeSeries, LABEL_NAME as RW_LABEL_NAME}; use crate::error::MpscError; -use crate::etcd_retry::is_transient_etcd_error; use crate::keys::{self, MqCategory}; use crate::lifecycle::spawn_named; -use crate::manager::{get_chan_meta, ChanManager, ChanMemberMeta, ChanRole, PRODUCE_OFFSET_BEGIN}; +use crate::manager::{ + etcd_rpc_attempt_limit, get_chan_meta_with_retry, ChanManager, ChanMemberMeta, ChanRole, + PRODUCE_OFFSET_BEGIN, +}; use crate::nonblocking_monitor::{ spawn_nonblocking_monitor, NonblockingMonitorHandle, NonblockingMonitorKind, }; +use crate::offset_commit::{MonotonicOffsetCommit, OffsetCommitProgress, OffsetObservation}; use crate::shutdown::ShutdownCtl; use crate::LifecycleView; use tokio::sync::watch; @@ -34,10 +39,8 @@ use tracing::warn; const PRODUCE_OFFSET_ETCD_SLOW_WARN_THRESHOLD: Duration = Duration::from_secs(1); const PRODUCE_OFFSET_PUT_TIMEOUT: Duration = Duration::from_secs(5); -const PRODUCE_OFFSET_PUT_ATTEMPTS: usize = 3; const PRODUCE_OFFSET_PUT_RETRY_DELAY: Duration = Duration::from_millis(100); const PRODUCER_MEMBERSHIP_RPC_TIMEOUT: Duration = Duration::from_secs(5); -const PRODUCER_MEMBERSHIP_RPC_ATTEMPTS: usize = 3; const PRODUCER_MEMBERSHIP_RETRY_DELAY: Duration = Duration::from_millis(100); #[derive(Debug, Clone, Serialize, Deserialize)] @@ -72,13 +75,13 @@ fn producer_membership_txn( weight_key: &str, weight_value: &str, global_lease_id: i64, + produce_offset_key: &str, ) -> etcd::Txn { etcd::Txn::new() - .when(vec![etcd::Compare::create_revision( - key, - etcd::CompareOp::Equal, - 0, - )]) + .when(vec![ + etcd::Compare::create_revision(key, etcd::CompareOp::Equal, 0), + etcd::Compare::create_revision(produce_offset_key, etcd::CompareOp::Equal, 0), + ]) .and_then(vec![ etcd::TxnOp::put( key, @@ -94,62 +97,305 @@ fn producer_membership_txn( .or_else(vec![ etcd::TxnOp::get(key, None), etcd::TxnOp::get(weight_key, None), + etcd::TxnOp::get(produce_offset_key, None), ]) } -fn existing_producer_membership_matches( +#[derive(Debug, Clone, PartialEq, Eq)] +struct EtcdKeyGeneration { + key: String, + value: Vec, + lease_id: i64, + mod_revision: i64, +} + +impl EtcdKeyGeneration { + fn from_kv(kv: &etcd::KeyValue, expected_key: &str) -> Result { + if kv.key() != expected_key.as_bytes() { + anyhow::bail!( + "producer membership readback returned unexpected key for expected_key={}", + expected_key + ); + } + Ok(Self { + key: expected_key.to_string(), + value: kv.value().to_vec(), + lease_id: kv.lease(), + mod_revision: kv.mod_revision(), + }) + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +struct ProducerMembershipGeneration { + member: EtcdKeyGeneration, + weight: EtcdKeyGeneration, +} + +struct ProducerMembershipPublication { + generation: ProducerMembershipGeneration, + produce_offset_observation: OffsetObservation, +} + +enum ProducerMembershipReadback { + Absent, + Owned(ProducerMembershipPublication), + Conflicting { + member_count: usize, + weight_count: usize, + produce_offset_count: usize, + }, +} + +fn producer_membership_get_state( txn_res: &etcd::TxnResponse, - member_meta_bytes: &[u8], - member_lease_id: i64, - weight_value: &str, - global_lease_id: i64, -) -> Result { + key: &str, + weight_key: &str, + produce_offset_key: &str, +) -> Result<( + Option, + Option, + Option, +)> { let responses = txn_res.op_responses(); - let [etcd::TxnOpResponse::Get(member_get), etcd::TxnOpResponse::Get(weight_get)] = + let [etcd::TxnOpResponse::Get(member_get), etcd::TxnOpResponse::Get(weight_get), etcd::TxnOpResponse::Get(produce_offset_get)] = responses.as_slice() else { anyhow::bail!( - "producer membership conflict returned an invalid response shape: operations={}", + "producer membership readback returned an invalid response shape: operations={}", responses.len() ); }; let member_kvs = member_get.kvs(); let weight_kvs = weight_get.kvs(); - if member_kvs.is_empty() && weight_kvs.is_empty() { - return Ok(false); - } - if member_kvs.len() == 1 - && weight_kvs.len() == 1 - && member_kvs[0].value() == member_meta_bytes - && member_kvs[0].lease() == member_lease_id - && weight_kvs[0].value() == weight_value.as_bytes() - && weight_kvs[0].lease() == global_lease_id - { - return Ok(true); + let produce_offset_kvs = produce_offset_get.kvs(); + if member_kvs.len() > 1 || weight_kvs.len() > 1 || produce_offset_kvs.len() > 1 { + anyhow::bail!( + "producer membership readback returned duplicate exact keys: member_count={} weight_count={} produce_offset_count={}", + member_kvs.len(), + weight_kvs.len(), + produce_offset_kvs.len() + ); } - anyhow::bail!( - "producer membership key already exists with conflicting state: member_count={} weight_count={}", - member_kvs.len(), - weight_kvs.len() - ) + let member = member_kvs + .first() + .map(|kv| EtcdKeyGeneration::from_kv(kv, key)) + .transpose()?; + let weight = weight_kvs + .first() + .map(|kv| EtcdKeyGeneration::from_kv(kv, weight_key)) + .transpose()?; + let produce_offset = produce_offset_kvs + .first() + .map(|kv| EtcdKeyGeneration::from_kv(kv, produce_offset_key)) + .transpose()?; + Ok((member, weight, produce_offset)) } -async fn cleanup_producer_membership( - client: &mut etcd::Client, +fn producer_membership_get_pair( + txn_res: &etcd::TxnResponse, + key: &str, + weight_key: &str, +) -> Result<(Option, Option)> { + let responses = txn_res.op_responses(); + let [etcd::TxnOpResponse::Get(member_get), etcd::TxnOpResponse::Get(weight_get)] = + responses.as_slice() + else { + anyhow::bail!( + "producer membership cleanup readback returned an invalid response shape: operations={}", + responses.len() + ); + }; + if member_get.kvs().len() > 1 || weight_get.kvs().len() > 1 { + anyhow::bail!( + "producer membership cleanup readback returned duplicate exact keys: member_count={} weight_count={}", + member_get.kvs().len(), + weight_get.kvs().len() + ); + } + let member = member_get + .kvs() + .first() + .map(|kv| EtcdKeyGeneration::from_kv(kv, key)) + .transpose()?; + let weight = weight_get + .kvs() + .first() + .map(|kv| EtcdKeyGeneration::from_kv(kv, weight_key)) + .transpose()?; + Ok((member, weight)) +} + +fn classify_producer_membership_readback( + txn_res: &etcd::TxnResponse, key: &str, + member_meta_bytes: &[u8], + member_lease_id: i64, weight_key: &str, + weight_value: &str, + global_lease_id: i64, + produce_offset_key: &str, +) -> Result { + let (member, weight, produce_offset) = + producer_membership_get_state(txn_res, key, weight_key, produce_offset_key)?; + match (member, weight, produce_offset) { + (None, None, None) => Ok(ProducerMembershipReadback::Absent), + (Some(member), Some(weight), None) + if member.value == member_meta_bytes + && member.lease_id == member_lease_id + && weight.value == weight_value.as_bytes() + && weight.lease_id == global_lease_id => + { + Ok(ProducerMembershipReadback::Owned( + ProducerMembershipPublication { + generation: ProducerMembershipGeneration { member, weight }, + produce_offset_observation: OffsetObservation::Absent, + }, + )) + } + (member, weight, produce_offset) => Ok(ProducerMembershipReadback::Conflicting { + member_count: usize::from(member.is_some()), + weight_count: usize::from(weight.is_some()), + produce_offset_count: usize::from(produce_offset.is_some()), + }), + } +} + +fn published_producer_membership_generation( + txn_res: &etcd::TxnResponse, + key: &str, + member_meta_bytes: &[u8], + member_lease_id: i64, + weight_key: &str, + weight_value: &str, + global_lease_id: i64, +) -> Result { + let mod_revision = txn_res + .header() + .ok_or_else(|| anyhow::anyhow!("producer membership publish response has no header"))? + .revision(); + if mod_revision <= 0 { + anyhow::bail!( + "producer membership publish returned invalid revision {}", + mod_revision + ); + } + Ok(ProducerMembershipPublication { + generation: ProducerMembershipGeneration { + member: EtcdKeyGeneration { + key: key.to_string(), + value: member_meta_bytes.to_vec(), + lease_id: member_lease_id, + mod_revision, + }, + weight: EtcdKeyGeneration { + key: weight_key.to_string(), + value: weight_value.as_bytes().to_vec(), + lease_id: global_lease_id, + mod_revision, + }, + }, + produce_offset_observation: OffsetObservation::Absent, + }) +} + +fn producer_membership_cleanup_txn(generation: &ProducerMembershipGeneration) -> etcd::Txn { + let member = &generation.member; + let weight = &generation.weight; + etcd::Txn::new() + .when(vec![ + etcd::Compare::mod_revision( + member.key.clone(), + etcd::CompareOp::Equal, + member.mod_revision, + ), + etcd::Compare::lease(member.key.clone(), etcd::CompareOp::Equal, member.lease_id), + etcd::Compare::value( + member.key.clone(), + etcd::CompareOp::Equal, + member.value.clone(), + ), + etcd::Compare::mod_revision( + weight.key.clone(), + etcd::CompareOp::Equal, + weight.mod_revision, + ), + etcd::Compare::lease(weight.key.clone(), etcd::CompareOp::Equal, weight.lease_id), + etcd::Compare::value( + weight.key.clone(), + etcd::CompareOp::Equal, + weight.value.clone(), + ), + ]) + .and_then(vec![ + etcd::TxnOp::delete(member.key.clone(), None), + etcd::TxnOp::delete(weight.key.clone(), None), + ]) + .or_else(vec![ + etcd::TxnOp::get(member.key.clone(), None), + etcd::TxnOp::get(weight.key.clone(), None), + ]) +} + +fn reconcile_producer_membership_cleanup( + txn_res: &etcd::TxnResponse, + generation: &ProducerMembershipGeneration, +) -> Result<()> { + let (member, weight) = + producer_membership_get_pair(txn_res, &generation.member.key, &generation.weight.key)?; + reconcile_producer_membership_cleanup_observation(member.as_ref(), weight.as_ref(), generation) +} + +fn reconcile_producer_membership_cleanup_observation( + member: Option<&EtcdKeyGeneration>, + weight: Option<&EtcdKeyGeneration>, + generation: &ProducerMembershipGeneration, +) -> Result<()> { + let Some(member) = member else { + return Ok(()); + }; + if member != &generation.member { + // A different member generation owns this logical key. Never delete it. + return Ok(()); + } + + match weight { + Some(weight) if weight == &generation.weight => anyhow::bail!( + "producer membership cleanup compare was false although both keys still match generation" + ), + _ => anyhow::bail!( + "producer membership cleanup refused partial state: owned member still exists but weight generation differs" + ), + } +} + +async fn cleanup_producer_membership( + client: &mut etcd::Client, + generation: &ProducerMembershipGeneration, + etcd_rpc_max_retries: u32, ) -> Result<()> { let mut last_error = String::new(); - for attempt in 1..=PRODUCER_MEMBERSHIP_RPC_ATTEMPTS { - let txn = etcd::Txn::new().and_then(vec![ - etcd::TxnOp::delete(key, None), - etcd::TxnOp::delete(weight_key, None), - ]); + let max_attempts = etcd_rpc_attempt_limit(etcd_rpc_max_retries); + for attempt in 1..=max_attempts { + let txn = producer_membership_cleanup_txn(generation); match tokio::time::timeout(PRODUCER_MEMBERSHIP_RPC_TIMEOUT, client.txn(txn)).await { - Ok(Ok(_)) => return Ok(()), - Ok(Err(err)) => last_error = err.to_string(), + Ok(Ok(txn_res)) if txn_res.succeeded() => return Ok(()), + Ok(Ok(txn_res)) => { + reconcile_producer_membership_cleanup(&txn_res, generation)?; + return Ok(()); + } + Ok(Err(err)) if is_transient_etcd_error(&err) => last_error = err.to_string(), + Ok(Err(err)) => { + anyhow::bail!( + "failed to delete producer membership and weight keys {}, {} on attempt {}: {}", + generation.member.key, + generation.weight.key, + attempt, + err + ); + } Err(_) => { last_error = format!( "request timed out after {} ms", @@ -157,15 +403,15 @@ async fn cleanup_producer_membership( ) } } - if attempt < PRODUCER_MEMBERSHIP_RPC_ATTEMPTS { + if attempt < max_attempts { tokio::time::sleep(PRODUCER_MEMBERSHIP_RETRY_DELAY).await; } } anyhow::bail!( "failed to delete producer membership and weight keys {}, {} after {} attempts: {}", - key, - weight_key, - PRODUCER_MEMBERSHIP_RPC_ATTEMPTS, + generation.member.key, + generation.weight.key, + max_attempts, last_error ) } @@ -178,16 +424,26 @@ async fn publish_producer_membership( weight_key: &str, weight_value: &str, global_lease_id: i64, + produce_offset_key: &str, + etcd_rpc_max_retries: u32, shutdown: &ShutdownCtl, -) -> Result<()> { +) -> Result { let mut last_error = String::new(); let mut request_started = false; - for attempt in 1..=PRODUCER_MEMBERSHIP_RPC_ATTEMPTS { + let max_attempts = etcd_rpc_attempt_limit(etcd_rpc_max_retries); + let mut attempts_performed = 0u64; + for attempt in 1..=max_attempts { + attempts_performed = attempt; if shutdown.is_closed() { - if request_started { - cleanup_producer_membership(client, key, weight_key).await?; - } - anyhow::bail!("producer binding stopped by shutdown during membership publish"); + let cleanup = if request_started { + "skipped: publish generation was never acknowledged" + } else { + "not needed" + }; + anyhow::bail!( + "producer binding stopped by shutdown during membership publish; cleanup={}", + cleanup + ); } request_started = true; @@ -198,54 +454,87 @@ async fn publish_producer_membership( weight_key, weight_value, global_lease_id, + produce_offset_key, ); - match tokio::time::timeout(PRODUCER_MEMBERSHIP_RPC_TIMEOUT, client.txn(txn)).await { - Ok(Ok(txn_res)) if txn_res.succeeded() => return Ok(()), - Ok(Ok(txn_res)) => match existing_producer_membership_matches( - &txn_res, - member_meta_bytes, - member_lease_id, - weight_value, - global_lease_id, - ) { - Ok(true) => return Ok(()), - Ok(false) => { - last_error = "membership disappeared while reconciling a retry".to_string() + let retryable = + match tokio::time::timeout(PRODUCER_MEMBERSHIP_RPC_TIMEOUT, client.txn(txn)).await { + Ok(Ok(txn_res)) if txn_res.succeeded() => { + return published_producer_membership_generation( + &txn_res, + key, + member_meta_bytes, + member_lease_id, + weight_key, + weight_value, + global_lease_id, + ); } - Err(conflict) => return Err(conflict), - }, - Ok(Err(err)) => last_error = err.to_string(), - Err(_) => { - last_error = format!( - "request timed out after {} ms", - PRODUCER_MEMBERSHIP_RPC_TIMEOUT.as_millis() - ) - } + Ok(Ok(txn_res)) => match classify_producer_membership_readback( + &txn_res, + key, + member_meta_bytes, + member_lease_id, + weight_key, + weight_value, + global_lease_id, + produce_offset_key, + ) { + Ok(ProducerMembershipReadback::Owned(publication)) => { + return Ok(publication); + } + Ok(ProducerMembershipReadback::Absent) => { + last_error = "membership disappeared while reconciling a retry".to_string(); + true + } + Ok(ProducerMembershipReadback::Conflicting { + member_count, + weight_count, + produce_offset_count, + }) => anyhow::bail!( + "producer membership or produce offset key already exists with conflicting state: member_count={} weight_count={} produce_offset_count={}", + member_count, + weight_count, + produce_offset_count, + ), + Err(error) => return Err(error), + }, + Ok(Err(err)) => { + let retryable = is_transient_etcd_error(&err); + last_error = err.to_string(); + retryable + } + Err(_) => { + last_error = format!( + "request timed out after {} ms", + PRODUCER_MEMBERSHIP_RPC_TIMEOUT.as_millis() + ); + true + } + }; + + if !retryable { + break; } - warn!( - chan_membership_key = key, - attempt, - total = PRODUCER_MEMBERSHIP_RPC_ATTEMPTS, - error = %last_error, - "producer membership publish did not complete; retrying" - ); - if attempt < PRODUCER_MEMBERSHIP_RPC_ATTEMPTS { + if attempt < max_attempts { + warn!( + chan_membership_key = key, + attempt, + total = max_attempts, + error = %last_error, + "producer membership publish did not complete; retrying" + ); tokio::time::sleep(PRODUCER_MEMBERSHIP_RETRY_DELAY).await; } } - let cleanup = cleanup_producer_membership(client, key, weight_key).await; anyhow::bail!( "failed to publish producer membership and weight keys {}, {} after {} attempts: {}; cleanup={}", key, weight_key, - PRODUCER_MEMBERSHIP_RPC_ATTEMPTS, + attempts_performed, last_error, - match cleanup { - Ok(()) => "ok".to_string(), - Err(err) => err.to_string(), - } + "skipped: publish generation was never acknowledged", ) } @@ -265,6 +554,7 @@ pub struct MpscProducer { /// `produce_offset` and relies on the invariant that a given /// producer handle is single-writer. next_msg_id: i64, + produce_offset_observation: OffsetObservation, /// Shared shutdown controller used by higher layers (via PyO3 /// handle) to signal that this producer should stop retrying and /// exit ongoing operations as soon as possible. @@ -313,6 +603,7 @@ impl MpscProducer { } let chan_id = chan_mgr.chan_id; + let etcd_rpc_max_retries = chan_mgr.etcd_rpc_max_retries(); let lease_manager = chan_mgr.lease_manager.clone(); let mut client = chan_mgr.etcd_client(); @@ -336,7 +627,7 @@ impl MpscProducer { _ = shutdown.wait_closed() => { anyhow::bail!("producer binding stopped by shutdown while loading channel metadata"); } - result = get_chan_meta(&mut meta_client, chan_id) => result, + result = get_chan_meta_with_retry(&mut meta_client, chan_id, etcd_rpc_max_retries) => result, }; let _meta = meta_result .with_context(|| format!("channel meta not found for chan_id={}", chan_id))?; @@ -382,8 +673,12 @@ impl MpscProducer { let weight = weight.unwrap_or(1); let weight_value = weight.to_string(); let weight_key = keys::etcd_producer_weight_key(chan_id, &producer_idx); + let produce_offset_key = keys::etcd_produce_offset_one_producer_key(chan_id, &producer_idx); let global_lease_id = chan_mgr.global_lease.id() as i64; - publish_producer_membership( + let ProducerMembershipPublication { + generation: membership_generation, + produce_offset_observation, + } = publish_producer_membership( &mut client, &key, &member_meta_bytes, @@ -391,12 +686,19 @@ impl MpscProducer { &weight_key, &weight_value, global_lease_id, + &produce_offset_key, + etcd_rpc_max_retries, &shutdown, ) .await?; if shutdown.is_closed() { - let cleanup = cleanup_producer_membership(&mut client, &key, &weight_key).await; + let cleanup = cleanup_producer_membership( + &mut client, + &membership_generation, + etcd_rpc_max_retries, + ) + .await; anyhow::bail!( "producer binding stopped by shutdown after membership publish; cleanup for {}, {}: {}", key, @@ -417,6 +719,7 @@ impl MpscProducer { producer_idx.clone(), lifecycle.clone(), shutdown.clone(), + etcd_rpc_max_retries, ); let nonblocking_monitor = spawn_nonblocking_monitor( &lifecycle, @@ -436,6 +739,7 @@ impl MpscProducer { chan_mgr, // First id = PRODUCE_OFFSET_BEGIN + 1 next_msg_id: PRODUCE_OFFSET_BEGIN + 1, + produce_offset_observation, // shutdown 控制器由上层（例如 PyO3 层）构造并注入， // 这里直接复用同一个实例，以便 handle/重试循环 // 共享关闭信号。 @@ -658,13 +962,15 @@ impl MpscProducer { // 与 Python 版保持一致（等价于 self.chan_lease）。 let global_lease_id = self.chan_mgr.global_lease.id() as i64; let offset_put_begin = Instant::now(); - let mut committed_attempt = 0usize; - for attempt in 1..=PRODUCE_OFFSET_PUT_ATTEMPTS { - let put = client.put( - offset_key.clone(), - next_id.to_string(), - Some(etcd::PutOptions::new().with_lease(global_lease_id)), - ); + let max_attempts = etcd_rpc_attempt_limit(self.chan_mgr.etcd_rpc_max_retries()); + let mut committed_attempt = 0u64; + let mut offset_commit = MonotonicOffsetCommit::new( + offset_key.clone(), + next_id, + global_lease_id, + Some(self.produce_offset_observation.clone()), + )?; + for attempt in 1..=max_attempts { let result = tokio::select! { biased; _ = self.shutdown.wait_closed() => { @@ -673,30 +979,39 @@ impl MpscProducer { offset_key, next_id ))); } - result = tokio::time::timeout(PRODUCE_OFFSET_PUT_TIMEOUT, put) => result, + result = tokio::time::timeout( + PRODUCE_OFFSET_PUT_TIMEOUT, + offset_commit.attempt(&mut client), + ) => result, }; let retry_reason = match result { - Ok(Ok(_)) => { + Ok(Ok(OffsetCommitProgress::Complete(generation))) => { + self.produce_offset_observation = OffsetObservation::Present(generation); committed_attempt = attempt; break; } - Ok(Err(error)) if is_transient_etcd_error(&error) => { - format!("transient etcd error: {error}") - } - Ok(Err(error)) => { - return Err(MpscError::Internal(format!( - "failed to update produce offset for key {}, leaseid: {}, attempt: {}, err: {}", - offset_key, global_lease_id, attempt, error - ))); + Ok(Ok(OffsetCommitProgress::Retry(_))) => { + "offset generation changed before the fenced commit".to_string() } + Ok(Err(error)) => match error { + MpscError::Etcd(error) if is_transient_etcd_error(&error) => { + format!("transient etcd error: {error}") + } + error => { + return Err(MpscError::Internal(format!( + "failed to update produce offset for key {}, leaseid: {}, attempt: {}, err: {}", + offset_key, global_lease_id, attempt, error + ))); + } + }, Err(_) => format!( "timed out after {} ms", PRODUCE_OFFSET_PUT_TIMEOUT.as_millis() ), }; - if attempt == PRODUCE_OFFSET_PUT_ATTEMPTS { + if attempt == max_attempts { return Err(MpscError::Internal(format!( "failed to update produce offset for key {}, leaseid: {}, msg_id: {} after {} attempts: {}", offset_key, global_lease_id, next_id, attempt, retry_reason @@ -709,7 +1024,7 @@ impl MpscProducer { msg_id = next_id, offset_key = %offset_key, attempt, - max_attempts = PRODUCE_OFFSET_PUT_ATTEMPTS, + max_attempts, reason = %retry_reason, "Retrying produce-offset commit" ); @@ -751,6 +1066,7 @@ fn spawn_consumer_meta_watch( producer_idx: String, lifecycle: LifecycleView, shutdown: ShutdownCtl, + max_retries: u32, ) { let name = format!( "fluxon_mq.producer.consumer_meta_watch.chan_id={}.producer_idx={}", @@ -761,9 +1077,14 @@ fn spawn_consumer_meta_watch( let opts = etcd::WatchOptions::new().with_prefix(); let mut initial_refresh_client = client.clone(); - let _ = - refresh_consumer_bind_state(&mut initial_refresh_client, chan_id, &prefix, &state_tx) - .await; + let _ = refresh_consumer_bind_state( + &mut initial_refresh_client, + chan_id, + &prefix, + &state_tx, + max_retries, + ) + .await; let watch_label = format!("[MpscProducer chan_id={}] consumer meta watch", chan_id); let stop = shutdown; @@ -786,8 +1107,14 @@ fn spawn_consumer_meta_watch( let prefix = resync_prefix.clone(); let state_tx = resync_state_tx.clone(); async move { - refresh_consumer_bind_state(&mut refresh_client, chan_id, &prefix, &state_tx) - .await + refresh_consumer_bind_state( + &mut refresh_client, + chan_id, + &prefix, + &state_tx, + max_retries, + ) + .await } }, move |_events| { @@ -795,8 +1122,14 @@ fn spawn_consumer_meta_watch( let prefix = batch_prefix.clone(); let state_tx = batch_state_tx.clone(); async move { - refresh_consumer_bind_state(&mut refresh_client, chan_id, &prefix, &state_tx) - .await + refresh_consumer_bind_state( + &mut refresh_client, + chan_id, + &prefix, + &state_tx, + max_retries, + ) + .await } }, ) @@ -809,8 +1142,10 @@ async fn refresh_consumer_bind_state( chan_id: i64, prefix: &str, state_tx: &watch::Sender, + max_retries: u32, ) -> EtcdPrefixWatchLoopControl { - let state = match load_consumer_bind_state_snapshot(client, chan_id, prefix).await { + let state = match load_consumer_bind_state_snapshot(client, chan_id, prefix, max_retries).await + { Ok(v) => v, Err(e) => { let reason = format!( @@ -831,11 +1166,12 @@ async fn load_consumer_bind_state_snapshot( client: &mut etcd::Client, _chan_id: i64, prefix: &str, + max_retries: u32, ) -> Result { let mut binding_count = 0usize; let mut first_value: Option> = None; let mut keys_dbg: Vec = Vec::new(); - scan_etcd_prefix_paginated(client, prefix, |key, value| { + scan_etcd_prefix_paginated_with_retry(client, prefix, max_retries, |key, value| { binding_count += 1; if keys_dbg.len() < 8 { match std::str::from_utf8(key) { @@ -897,10 +1233,86 @@ async fn allocate_producer_idx(chan_mgr: &ChanManager) -> Result { // producer id allocator 提供稳定的 lease 语义。 let lease_id = chan_mgr.global_long_lease.id() as i64; - let allocator = - DistributeIdAllocator::new(client.clone(), format!("channels/{}", chan_id), lease_id); + let allocator = DistributeIdAllocator::new_with_retry( + client.clone(), + format!("channels/{}", chan_id), + lease_id, + chan_mgr.etcd_rpc_max_retries(), + ); allocator .allocate_id() .await .with_context(|| format!("failed to allocate producer id for chan_id={}", chan_id)) } + +#[cfg(test)] +mod tests { + use super::{ + reconcile_producer_membership_cleanup_observation, EtcdKeyGeneration, + ProducerMembershipGeneration, + }; + + fn generation() -> ProducerMembershipGeneration { + ProducerMembershipGeneration { + member: EtcdKeyGeneration { + key: "member".to_string(), + value: b"member-value".to_vec(), + lease_id: 11, + mod_revision: 101, + }, + weight: EtcdKeyGeneration { + key: "weight".to_string(), + value: b"1".to_vec(), + lease_id: 22, + mod_revision: 101, + }, + } + } + + #[test] + fn cleanup_replay_accepts_already_deleted_generation() { + let expected = generation(); + reconcile_producer_membership_cleanup_observation(None, None, &expected).unwrap(); + } + + #[test] + fn cleanup_replay_never_claims_a_new_member_generation() { + let expected = generation(); + let mut newer_member = expected.member.clone(); + newer_member.mod_revision += 1; + + reconcile_producer_membership_cleanup_observation( + Some(&newer_member), + Some(&expected.weight), + &expected, + ) + .unwrap(); + } + + #[test] + fn cleanup_replay_rejects_partial_owned_generation() { + let expected = generation(); + let mut newer_weight = expected.weight.clone(); + newer_weight.mod_revision += 1; + + let error = reconcile_producer_membership_cleanup_observation( + Some(&expected.member), + Some(&newer_weight), + &expected, + ) + .unwrap_err(); + assert!(error.to_string().contains("refused partial state")); + } + + #[test] + fn cleanup_false_compare_rejects_impossible_exact_readback() { + let expected = generation(); + let error = reconcile_producer_membership_cleanup_observation( + Some(&expected.member), + Some(&expected.weight), + &expected, + ) + .unwrap_err(); + assert!(error.to_string().contains("compare was false")); + } +} diff --git a/fluxon_rs/fluxon_pyo3/src/etcd.rs b/fluxon_rs/fluxon_pyo3/src/etcd.rs index 304d506..626c02f 100644 --- a/fluxon_rs/fluxon_pyo3/src/etcd.rs +++ b/fluxon_rs/fluxon_pyo3/src/etcd.rs @@ -1,5 +1,8 @@ use etcd_client as etcd; -use fluxon_util::etcd::{PooledEtcdClient, etcd_clients_pool}; +use fluxon_util::etcd::{ + DEFAULT_ETCD_RPC_MAX_RETRIES, PooledEtcdClient, etcd_clients_pool, is_transient_etcd_error, + retry_etcd_rpc, +}; use fluxon_util::run_async_from_sync::SyncAsyncBridge; use pyo3::prelude::*; use pyo3::pybacked::PyBackedBytes; @@ -48,23 +51,10 @@ fn normalize_raw_endpoints(endpoints: Vec, component: &str) -> PyResult< Ok(normalized) } -fn is_reconnectable_etcd_error(err: &etcd::Error) -> bool { - is_reconnectable_etcd_error_text(&format!("{:?}", err)) -} - -fn is_reconnectable_etcd_error_text(msg: &str) -> bool { - let msg = msg.to_ascii_lowercase(); - msg.contains("unavailable") - || msg.contains("connection") - || msg.contains("transport") - || msg.contains("timed out") - || msg.contains("timeout") - || msg.contains("broken pipe") - || msg.contains("closed") -} - async fn run_etcd_op( pool_entry: PooledEtcdClient, + max_retries: u32, + operation: &'static str, context: String, mut op: F, ) -> anyhow::Result @@ -72,44 +62,232 @@ where F: FnMut(etcd::Client) -> Fut, Fut: Future>, { - let mut last_err = None; - for attempt in 1..=2 { - let snapshot = pool_entry.snapshot().await?; - match op(snapshot.client()).await { - Ok(value) => return Ok(value), - Err(err) => { - let should_retry = attempt == 1 && is_reconnectable_etcd_error(&err); - last_err = Some(err); - if should_retry { - snapshot.invalidate().await; - continue; - } - let err = last_err.take().expect("etcd error must be recorded"); - return Err(anyhow::anyhow!("{}: {:?}", context, err)); - } + let snapshot = pool_entry.snapshot().await?; + let result = retry_etcd_rpc(max_retries, operation, || op(snapshot.client())).await; + + if let Err(error) = &result { + if is_transient_etcd_error(error) { + snapshot.invalidate().await; } } - let err = last_err.expect("etcd retry loop must record the last error"); - Err(anyhow::anyhow!("{}: {:?}", context, err)) + result.map_err(|error| anyhow::anyhow!("{}: {:?}", context, error)) +} + +#[derive(Clone, Debug, PartialEq, Eq)] +struct EtcdKeyGeneration { + value: Vec, + lease_id: i64, + mod_revision: i64, +} + +impl EtcdKeyGeneration { + fn from_kv(kv: &etcd::KeyValue, expected_key: &str) -> anyhow::Result { + if kv.key() != expected_key.as_bytes() { + anyhow::bail!( + "etcd exact-key read returned an unexpected key: expected={} actual={:?}", + expected_key, + kv.key() + ); + } + Ok(Self { + value: kv.value().to_vec(), + lease_id: kv.lease(), + mod_revision: kv.mod_revision(), + }) + } + + fn matches_value_and_lease(&self, value: &[u8], lease_id: Option) -> bool { + self.value == value && self.lease_id == lease_id.unwrap_or(0) + } +} + +fn exact_generation_from_get( + response: &etcd::GetResponse, + key: &str, +) -> anyhow::Result> { + if response.kvs().len() > 1 { + anyhow::bail!( + "etcd exact-key read returned duplicate keys: key={} count={}", + key, + response.kvs().len() + ); + } + response + .kvs() + .first() + .map(|kv| EtcdKeyGeneration::from_kv(kv, key)) + .transpose() +} + +fn exact_generation_from_txn_readback( + response: &etcd::TxnResponse, + key: &str, +) -> anyhow::Result> { + let responses = response.op_responses(); + let [etcd::TxnOpResponse::Get(get_response)] = responses.as_slice() else { + anyhow::bail!( + "etcd conditional mutation returned an invalid readback shape: key={} operations={}", + key, + responses.len() + ); + }; + exact_generation_from_get(get_response, key) +} + +fn generation_compares(key: &str, generation: Option<&EtcdKeyGeneration>) -> Vec { + match generation { + Some(generation) => vec![ + etcd::Compare::mod_revision(key, etcd::CompareOp::Equal, generation.mod_revision), + etcd::Compare::lease(key, etcd::CompareOp::Equal, generation.lease_id), + etcd::Compare::value(key, etcd::CompareOp::Equal, generation.value.clone()), + ], + None => vec![etcd::Compare::create_revision( + key, + etcd::CompareOp::Equal, + 0, + )], + } +} + +async fn get_exact_generation( + pool_entry: PooledEtcdClient, + max_retries: u32, + key: &str, + operation: &'static str, +) -> anyhow::Result> { + let key_for_op = key.to_string(); + let response = run_etcd_op( + pool_entry, + max_retries, + operation, + format!("etcd generation read failed for key={}", key), + move |mut client| { + let key = key_for_op.clone(); + async move { client.get(key, None).await } + }, + ) + .await?; + exact_generation_from_get(&response, key) +} + +async fn put_with_generation_fence( + pool_entry: PooledEtcdClient, + max_retries: u32, + key: String, + value: Vec, + lease_id: Option, +) -> anyhow::Result<()> { + let observed = + get_exact_generation(pool_entry.clone(), max_retries, &key, "put_generation_read").await?; + let txn = etcd::Txn::new() + .when(generation_compares(&key, observed.as_ref())) + .and_then(vec![etcd::TxnOp::put( + key.clone(), + value.clone(), + lease_id.map(|id| etcd::PutOptions::new().with_lease(id)), + )]) + .or_else(vec![etcd::TxnOp::get(key.clone(), None)]); + + let response = run_etcd_op( + pool_entry, + max_retries, + "put", + format!("etcd conditional put failed for key={}", key), + move |mut client| { + let txn = txn.clone(); + async move { client.txn(txn).await } + }, + ) + .await?; + if response.succeeded() { + return Ok(()); + } + + match exact_generation_from_txn_readback(&response, &key)? { + Some(current) if current.matches_value_and_lease(&value, lease_id) => Ok(()), + Some(current) => anyhow::bail!( + "etcd conditional put for key={} encountered a concurrent generation; preserving current mod_revision={} lease_id={}", + key, + current.mod_revision, + current.lease_id + ), + None => anyhow::bail!( + "etcd conditional put for key={} was not committed and the key is absent", + key + ), + } +} + +async fn delete_with_generation_fence( + pool_entry: PooledEtcdClient, + max_retries: u32, + key: String, +) -> anyhow::Result { + let Some(observed) = get_exact_generation( + pool_entry.clone(), + max_retries, + &key, + "delete_generation_read", + ) + .await? + else { + return Ok(false); + }; + + let txn = etcd::Txn::new() + .when(generation_compares(&key, Some(&observed))) + .and_then(vec![etcd::TxnOp::delete(key.clone(), None)]) + .or_else(vec![etcd::TxnOp::get(key.clone(), None)]); + let response = run_etcd_op( + pool_entry, + max_retries, + "delete", + format!("etcd conditional delete failed for key={}", key), + move |mut client| { + let txn = txn.clone(); + async move { client.txn(txn).await } + }, + ) + .await?; + if response.succeeded() { + return Ok(true); + } + + match exact_generation_from_txn_readback(&response, &key)? { + None => Ok(true), + Some(current) if current == observed => anyhow::bail!( + "etcd conditional delete for key={} failed while the observed generation still exists", + key + ), + Some(current) => anyhow::bail!( + "etcd conditional delete for key={} encountered a concurrent generation; preserving current mod_revision={} lease_id={}", + key, + current.mod_revision, + current.lease_id + ), + } } #[pyclass(name = "EtcdKvClient")] pub struct PyEtcdKvClient { rt: Arc, endpoints: Vec, + etcd_rpc_max_retries: u32, pool_entry: PooledEtcdClient, } #[pymethods] impl PyEtcdKvClient { #[new] - fn new(endpoints: Vec) -> PyResult { + #[pyo3(signature = (endpoints, etcd_rpc_max_retries=None))] + fn new(endpoints: Vec, etcd_rpc_max_retries: Option) -> PyResult { let endpoints = normalize_raw_endpoints(endpoints, "EtcdKvClient")?; let pool_entry = etcd_clients_pool().acquire(endpoints.clone()); Ok(Self { rt: crate::mpsc::get_global_runtime(), endpoints, + etcd_rpc_max_retries: etcd_rpc_max_retries.unwrap_or(DEFAULT_ETCD_RPC_MAX_RETRIES), pool_entry, }) } @@ -122,12 +300,15 @@ impl PyEtcdKvClient { } let pool_entry = self.pool_entry.clone(); + let etcd_rpc_max_retries = self.etcd_rpc_max_retries; let key_for_op = key.clone(); let value = py .allow_threads(|| { self.rt.run_async_from_sync(async move { let resp = run_etcd_op( pool_entry, + etcd_rpc_max_retries, + "get", format!("etcd get failed for key={}", key), move |mut client| { let key = key_for_op.clone(); @@ -155,12 +336,15 @@ impl PyEtcdKvClient { } let pool_entry = self.pool_entry.clone(); + let etcd_rpc_max_retries = self.etcd_rpc_max_retries; let prefix_for_op = prefix.clone(); let rows = py .allow_threads(|| { self.rt.run_async_from_sync(async move { let resp = run_etcd_op( pool_entry, + etcd_rpc_max_retries, + "get_prefix", format!("etcd get_prefix failed for prefix={}", prefix), move |mut client| { let prefix = prefix_for_op.clone(); @@ -221,23 +405,38 @@ impl PyEtcdKvClient { } let pool_entry = self.pool_entry.clone(); - let key_for_op = key.clone(); + let etcd_rpc_max_retries = self.etcd_rpc_max_retries; let value = value.as_ref().to_vec(); py.allow_threads(|| { self.rt.run_async_from_sync(async move { - run_etcd_op( - pool_entry, - format!("etcd put failed for key={}", key), - move |mut client| { - let key = key_for_op.clone(); - let value = value.clone(); - async move { - let opts = lease_id.map(|id| etcd::PutOptions::new().with_lease(id)); - client.put(key, value, opts).await.map(|_| ()) - } - }, - ) - .await?; + if etcd_rpc_max_retries == 0 { + let key_for_op = key.clone(); + run_etcd_op( + pool_entry, + 0, + "put", + format!("etcd put failed for key={}", key), + move |mut client| { + let key = key_for_op.clone(); + let value = value.clone(); + async move { + let opts = + lease_id.map(|id| etcd::PutOptions::new().with_lease(id)); + client.put(key, value, opts).await.map(|_| ()) + } + }, + ) + .await?; + } else { + put_with_generation_fence( + pool_entry, + etcd_rpc_max_retries, + key, + value, + lease_id, + ) + .await?; + } Ok::<(), anyhow::Error>(()) }) }) @@ -254,23 +453,30 @@ impl PyEtcdKvClient { } let pool_entry = self.pool_entry.clone(); - let key_for_op = key.clone(); + let etcd_rpc_max_retries = self.etcd_rpc_max_retries; py.allow_threads(|| { self.rt.run_async_from_sync(async move { - run_etcd_op( - pool_entry, - format!("etcd delete failed for key={}", key), - move |mut client| { - let key = key_for_op.clone(); - async move { - client - .delete(key, None) - .await - .map(|resp| resp.deleted() > 0) - } - }, - ) - .await + if etcd_rpc_max_retries == 0 { + let key_for_op = key.clone(); + run_etcd_op( + pool_entry, + 0, + "delete", + format!("etcd delete failed for key={}", key), + move |mut client| { + let key = key_for_op.clone(); + async move { + client + .delete(key, None) + .await + .map(|resp| resp.deleted() > 0) + } + }, + ) + .await + } else { + delete_with_generation_fence(pool_entry, etcd_rpc_max_retries, key).await + } }) }) .map_err(|e| anyhow::anyhow!("runtime bridge failed in EtcdKvClient.delete: {}", e)) @@ -289,8 +495,12 @@ impl PyEtcdKvClient { let prefix_for_op = prefix.clone(); py.allow_threads(|| { self.rt.run_async_from_sync(async move { + // A range delete cannot be replayed without risking deletion of a new + // generation and changing the returned count after an uncertain response. run_etcd_op( pool_entry, + 0, + "delete_prefix", format!("etcd delete_prefix failed for prefix={}", prefix), move |mut client| { let prefix = prefix_for_op.clone(); @@ -319,10 +529,13 @@ impl PyEtcdKvClient { } let pool_entry = self.pool_entry.clone(); + let etcd_rpc_max_retries = self.etcd_rpc_max_retries; py.allow_threads(|| { self.rt.run_async_from_sync(async move { run_etcd_op( pool_entry, + etcd_rpc_max_retries, + "lease_time_to_live", format!("etcd lease_ttl failed for lease_id={}", lease_id), move |mut client| async move { client @@ -348,12 +561,21 @@ impl PyEtcdKvClient { } let pool_entry = self.pool_entry.clone(); + let etcd_rpc_max_retries = self.etcd_rpc_max_retries; py.allow_threads(|| { self.rt.run_async_from_sync(async move { run_etcd_op( pool_entry, + etcd_rpc_max_retries, + "lease_revoke", format!("etcd revoke_lease failed for lease_id={}", lease_id), - move |mut client| async move { client.lease_revoke(lease_id).await.map(|_| ()) }, + move |mut client| async move { + match client.lease_revoke(lease_id).await { + Ok(_) => Ok(()), + Err(error) if is_lease_not_found_error(&error) => Ok(()), + Err(error) => Err(error), + } + }, ) .await }) @@ -364,7 +586,10 @@ impl PyEtcdKvClient { } fn __repr__(&self) -> String { - format!("", self.endpoints) + format!( + "", + self.endpoints, self.etcd_rpc_max_retries + ) } } @@ -675,8 +900,12 @@ mod tests { #[test] fn etcd_kv_client_constructor_normalizes_raw_endpoints() { - let client = PyEtcdKvClient::new(vec!["127.0.0.1:2379".to_string()]).unwrap(); + let client = PyEtcdKvClient::new(vec!["127.0.0.1:2379".to_string()], None).unwrap(); assert_eq!(client.endpoints, vec!["http://127.0.0.1:2379"]); + assert_eq!(client.etcd_rpc_max_retries, DEFAULT_ETCD_RPC_MAX_RETRIES); + + let no_retry = PyEtcdKvClient::new(vec!["127.0.0.1:2379".to_string()], Some(0)).unwrap(); + assert_eq!(no_retry.etcd_rpc_max_retries, 0); } #[test] @@ -706,20 +935,4 @@ mod tests { .is_err() ); } - - #[test] - fn reconnectable_error_text_matches_transient_transport_failures() { - assert!(is_reconnectable_etcd_error_text("StatusCode::UNAVAILABLE")); - assert!(is_reconnectable_etcd_error_text( - "etcdserver: request timed out" - )); - assert!(is_reconnectable_etcd_error_text("transport error")); - assert!(is_reconnectable_etcd_error_text("connection closed")); - assert!(is_reconnectable_etcd_error_text("broken pipe")); - - assert!(!is_reconnectable_etcd_error_text( - "requested lease not found" - )); - assert!(!is_reconnectable_etcd_error_text("permission denied")); - } } diff --git a/fluxon_rs/fluxon_pyo3/src/mpsc.rs b/fluxon_rs/fluxon_pyo3/src/mpsc.rs index 3ecf0d9..2f960d6 100644 --- a/fluxon_rs/fluxon_pyo3/src/mpsc.rs +++ b/fluxon_rs/fluxon_pyo3/src/mpsc.rs @@ -177,6 +177,7 @@ fn finalize_payload_result( #[pyclass] pub struct MpscContext { endpoints: Vec, + etcd_rpc_max_retries: u32, kv_backend_uid: LeaseBackendUid, kv_framework: Arc, kv_runtime: Handle, @@ -204,9 +205,11 @@ impl MpscContext { kv_client: Py, ) -> PyResult { let kv_client_ref = kv_client.borrow(py); + let etcd_rpc_max_retries = kv_client_ref.config.fluxonkv_spec.etcd_rpc_max_retries; let mq_context = crate::new_fluxon_mq_context(&kv_client_ref)?; Ok(Self { endpoints: etcd_endpoints, + etcd_rpc_max_retries, kv_backend_uid: mq_context.kv_lease_backend, kv_framework: mq_context.kv_framework.clone(), kv_runtime: mq_context.runtime.clone(), @@ -255,6 +258,7 @@ impl MpscContext { }; let endpoints = self.endpoints.clone(); + let etcd_rpc_max_retries = self.etcd_rpc_max_retries; let kv_backend_uid = self.kv_backend_uid.clone(); let self_info = self .kv_framework @@ -316,6 +320,7 @@ impl MpscContext { leases.global, leases.member, leases.payload, + etcd_rpc_max_retries, rth.clone(), shutdown_for_core.clone(), ), @@ -328,6 +333,7 @@ impl MpscContext { endpoints.clone(), kv_backend_uid.clone(), id, + etcd_rpc_max_retries, rth.clone(), ) .await @@ -344,6 +350,7 @@ impl MpscContext { capacity: cap, ttl_seconds, weight, + etcd_rpc_max_retries, override_global_lease_id, override_member_lease_id, override_payload_lease_id, @@ -444,6 +451,7 @@ impl MpscContext { }; let endpoints = self.endpoints.clone(); + let etcd_rpc_max_retries = self.etcd_rpc_max_retries; let kv_backend_uid = self.kv_backend_uid.clone(); let shutdown = ShutdownCtl::new(); let shutdown_for_core = shutdown.clone(); @@ -496,6 +504,7 @@ impl MpscContext { leases.global, leases.member, leases.payload, + etcd_rpc_max_retries, rth.clone(), shutdown_for_core.clone(), ) @@ -507,6 +516,7 @@ impl MpscContext { endpoints.clone(), kv_backend_uid.clone(), id, + etcd_rpc_max_retries, rth.clone(), ) .await @@ -525,6 +535,7 @@ impl MpscContext { // weight is producer-only; consumer path // does not configure or use it. weight: None, + etcd_rpc_max_retries, override_global_lease_id, override_member_lease_id, override_payload_lease_id, diff --git a/fluxon_rs/fluxon_util/Cargo.toml b/fluxon_rs/fluxon_util/Cargo.toml index cb7ef9b..1764e91 100644 --- a/fluxon_rs/fluxon_util/Cargo.toml +++ b/fluxon_rs/fluxon_util/Cargo.toml @@ -20,6 +20,7 @@ moka = { workspace = true } parking_lot = { workspace = true } futures = { workspace = true } tokio = { workspace = true, features = ["time", "rt", "macros"] } +tonic = { workspace = true } limit_thirdparty = { path = "../limit_thirdparty" } prometheus = "0.14" reqwest = { version = "0.12", default-features = false, features = ["rustls-tls"] } diff --git a/fluxon_rs/fluxon_util/src/etcd/cluster_lease.rs b/fluxon_rs/fluxon_util/src/etcd/cluster_lease.rs index d972933..037a34f 100755 --- a/fluxon_rs/fluxon_util/src/etcd/cluster_lease.rs +++ b/fluxon_rs/fluxon_util/src/etcd/cluster_lease.rs @@ -1,7 +1,9 @@ use anyhow::{Context, Result, anyhow}; -use etcd_client::{Client, Compare, CompareOp, GetOptions, PutOptions, Txn, TxnOp}; +use etcd_client::{Client, Compare, CompareOp, PutOptions, Txn, TxnOp, TxnOpResponse}; use tracing::debug; +use super::retry_etcd_rpc; + /// Get or create a shared cluster lease id for a given logical key. /// /// Port of Python `get_cluster_lease` in `fluxon_py.etcd`. @@ -12,20 +14,32 @@ pub async fn get_cluster_lease_id( client: &mut Client, lease_key: &str, ttl_seconds: i64, +) -> Result { + get_cluster_lease_id_with_retry(client, lease_key, ttl_seconds, 0).await +} + +/// Get or create a shared cluster lease with transient unary RPC retries. +/// +/// LeaseGrant remains a single call because blindly replaying it creates a +/// different lease. The safe Get and conditional publish use `max_retries`. +pub async fn get_cluster_lease_id_with_retry( + client: &mut Client, + lease_key: &str, + ttl_seconds: i64, + max_retries: u32, ) -> Result { let key = format!("cluster_lease/{}", lease_key); // Fast path: read existing lease id - let resp = client - .get(key.clone(), None) - .await - .with_context(|| format!("failed to get cluster lease key {key}"))?; + let resp = retry_etcd_rpc(max_retries, "get_cluster_lease", || { + let mut attempt_client = client.clone(); + let attempt_key = key.clone(); + async move { attempt_client.get(attempt_key, None).await } + }) + .await + .with_context(|| format!("failed to get cluster lease key {key}"))?; if let Some(kv) = resp.kvs().first() { - let txt = String::from_utf8(kv.value().to_vec()) - .with_context(|| format!("invalid lease id bytes for key {key}"))?; - let lease_id: i64 = txt - .parse() - .with_context(|| format!("invalid lease id '{}' for key {key}", txt))?; + let lease_id = parse_cluster_lease_id(kv.value(), &key, "")?; debug!( "reused existing cluster lease id {} for key {}", lease_id, key @@ -40,17 +54,22 @@ pub async fn get_cluster_lease_id( .with_context(|| format!("failed to grant lease for key {}", key))?; let lease_id = lease_resp.id(); - let compare = Compare::create_revision(key.clone(), CompareOp::Equal, 0); - let put_op = TxnOp::put( - key.clone(), - lease_id.to_string(), - Some(PutOptions::new().with_lease(lease_id)), - ); - let txn = Txn::new().when(vec![compare]).and_then(vec![put_op]); - let txn_res = client - .txn(txn) - .await - .with_context(|| format!("transaction failed when publishing cluster lease key {key}"))?; + let txn_res = retry_etcd_rpc(max_retries, "publish_cluster_lease", || { + let mut attempt_client = client.clone(); + let compare = Compare::create_revision(key.clone(), CompareOp::Equal, 0); + let put_op = TxnOp::put( + key.clone(), + lease_id.to_string(), + Some(PutOptions::new().with_lease(lease_id)), + ); + let txn = Txn::new() + .when(vec![compare]) + .and_then(vec![put_op]) + .or_else(vec![TxnOp::get(key.clone(), None)]); + async move { attempt_client.txn(txn).await } + }) + .await + .with_context(|| format!("transaction failed when publishing cluster lease key {key}"))?; if txn_res.succeeded() { debug!( "published new cluster lease id {} for key {}", @@ -59,26 +78,99 @@ pub async fn get_cluster_lease_id( return Ok(lease_id); } - // Another creator won the race; read back - let resp2 = client - .get(key.clone(), Some(GetOptions::new())) - .await - .with_context(|| format!("failed to re-get cluster lease key {key}"))?; - if let Some(kv) = resp2.kvs().first() { - let txt = String::from_utf8(kv.value().to_vec()) - .with_context(|| format!("invalid lease id bytes for key {key} after txn"))?; - let lease_id: i64 = txt - .parse() - .with_context(|| format!("invalid lease id '{}' for key {key} after txn", txt))?; - debug!( - "observed existing cluster lease id {} for key {} after txn", - lease_id, key + // A retry after an uncertain response lands here if the first publish + // committed. A concurrent winner has the same shape. In either case the + // atomic else-Get is the authoritative shared lease. + let observed = cluster_lease_publish_readback(&txn_res, &key)?; + let selected = reconcile_cluster_lease_publish(lease_id, observed, lease_key)?; + debug!( + "observed existing cluster lease id {} for key {} after txn", + selected, key + ); + Ok(selected) +} + +fn parse_cluster_lease_id(value: &[u8], key: &str, suffix: &str) -> Result { + let txt = String::from_utf8(value.to_vec()) + .with_context(|| format!("invalid lease id bytes for key {key}{suffix}"))?; + txt.parse() + .with_context(|| format!("invalid lease id '{}' for key {key}{suffix}", txt)) +} + +fn cluster_lease_publish_readback( + txn_res: &etcd_client::TxnResponse, + key: &str, +) -> Result> { + let responses = txn_res.op_responses(); + let [TxnOpResponse::Get(get)] = responses.as_slice() else { + return Err(anyhow!( + "cluster lease publish readback returned invalid response shape for key {}: operations={}", + key, + responses.len() + )); + }; + if get.kvs().len() > 1 { + return Err(anyhow!( + "cluster lease publish readback returned duplicate exact keys for {}", + key + )); + } + let Some(kv) = get.kvs().first() else { + return Ok(None); + }; + if kv.key() != key.as_bytes() { + return Err(anyhow!( + "cluster lease publish readback returned unexpected key for {}", + key + )); + } + parse_cluster_lease_id(kv.value(), key, " after txn").map(Some) +} + +fn reconcile_cluster_lease_publish( + candidate: i64, + observed: Option, + lease_key: &str, +) -> Result { + match observed { + // `candidate` may be our first uncertain publish; another value is the + // concurrent winner. Both are the authoritative shared identity. + Some(selected) => Ok(selected), + None => Err(anyhow!( + "failed to acquire cluster lease for key {}: key disappeared after publishing candidate {}", + lease_key, + candidate + )), + } +} + +#[cfg(test)] +mod tests { + use super::{parse_cluster_lease_id, reconcile_cluster_lease_publish}; + + #[test] + fn parses_atomic_cluster_lease_readback() { + assert_eq!( + parse_cluster_lease_id(b"42", "cluster_lease/test", " after txn").unwrap(), + 42 ); - return Ok(lease_id); } - Err(anyhow!( - "failed to acquire cluster lease for key {}: key disappeared after txn", - lease_key - )) + #[test] + fn rejects_invalid_cluster_lease_readback() { + assert!(parse_cluster_lease_id(b"not-an-id", "cluster_lease/test", "").is_err()); + } + + #[test] + fn publish_reconciliation_accepts_own_or_competing_winner() { + assert_eq!( + reconcile_cluster_lease_publish(41, Some(41), "test").unwrap(), + 41 + ); + assert_eq!( + reconcile_cluster_lease_publish(41, Some(52), "test").unwrap(), + 52 + ); + assert!(reconcile_cluster_lease_publish(41, None, "test").is_err()); + } } diff --git a/fluxon_rs/fluxon_util/src/etcd/id_allocator.rs b/fluxon_rs/fluxon_util/src/etcd/id_allocator.rs index e6120da..9c605d6 100755 --- a/fluxon_rs/fluxon_util/src/etcd/id_allocator.rs +++ b/fluxon_rs/fluxon_util/src/etcd/id_allocator.rs @@ -1,6 +1,38 @@ +use std::time::Duration; + use anyhow::{Context, Result, anyhow}; -use etcd_client::{Client, Compare, CompareOp, PutOptions, Txn, TxnOp}; -use tracing::debug; +use etcd_client::{ + Client, Compare, CompareOp, GetResponse, PutOptions, Txn, TxnOp, TxnOpResponse, TxnResponse, +}; +use tracing::{debug, warn}; + +use super::{is_transient_etcd_error, retry_etcd_rpc}; + +const MAX_CAS_CONFLICTS: u32 = 100; +const UNCERTAIN_RPC_RETRY_DELAY: Duration = Duration::from_millis(50); + +#[derive(Clone, Debug, PartialEq, Eq)] +struct CounterGeneration { + value: Vec, + lease_id: i64, + mod_revision: i64, +} + +impl CounterGeneration { + fn from_kv(kv: &etcd_client::KeyValue, expected_key: &str) -> Result { + if kv.key() != expected_key.as_bytes() { + return Err(anyhow!( + "dist_id readback returned unexpected key for {}", + expected_key + )); + } + Ok(Self { + value: kv.value().to_vec(), + lease_id: kv.lease(), + mod_revision: kv.mod_revision(), + }) + } +} /// Distributed ID allocator backed by etcd. /// @@ -15,109 +47,306 @@ pub struct DistributeIdAllocator { client: Client, prefix: String, lease_id: Option, + rpc_max_retries: u32, } impl DistributeIdAllocator { - /// Create a new allocator with the given etcd client, prefix and - /// associated lease id. + /// Create an allocator without automatic unary RPC retries. pub fn new(client: Client, prefix: impl Into, lease_id: i64) -> Self { + Self::new_with_retry(client, prefix, lease_id, 0) + } + + /// Create an allocator with transient unary RPC retries. + pub fn new_with_retry( + client: Client, + prefix: impl Into, + lease_id: i64, + rpc_max_retries: u32, + ) -> Self { Self { client, prefix: prefix.into(), lease_id: Some(lease_id), + rpc_max_retries, } } - /// Create a new allocator whose counter key is intentionally not bound to - /// any etcd lease. + /// Create an unleased allocator without automatic unary RPC retries. /// /// This mode is required for process-global monotonic counters such as the /// top-level MPSC `chan_id` allocator. Binding that counter to a short-lived /// lease would let the key disappear during an idle window and later restart /// from `1`, which can collide with still-existing historical metadata. pub fn new_without_lease(client: Client, prefix: impl Into) -> Self { + Self::new_without_lease_with_retry(client, prefix, 0) + } + + /// Create an unleased allocator with transient unary RPC retries. + pub fn new_without_lease_with_retry( + client: Client, + prefix: impl Into, + rpc_max_retries: u32, + ) -> Self { Self { client, prefix: prefix.into(), lease_id: None, + rpc_max_retries, } } - /// Allocate the next ID (starting from 1) using etcd transactions. + /// Allocate the next ID (starting from 1) using generation-fenced CAS. /// - /// This mirrors the Python logic: - /// - Read current value once. - /// - Try up to 100 times with compare-and-set on the value. + /// The 100-attempt bound applies only to ordinary CAS competition. A + /// transient response makes that candidate ambiguous, so it is permanently + /// skipped and the configured RPC retry budget is used to read a fresh + /// generation before attempting a larger candidate. pub async fn allocate_id(&self) -> Result { let key = format!("dist_id_allocator/{}", self.prefix); - let mut client = self.client.clone(); - - // Initial read of the current value (if any) - let resp = client - .get(key.clone(), None) - .await - .with_context(|| format!("failed to get dist_id key {key}"))?; - let mut old_value_v: Option> = resp.kvs().first().map(|kv| kv.value().to_vec()); - - for _ in 0..100 { - // Parse current value as integer; default to 0 on any error - let mut old_value_int: i64 = 0; - if let Some(ref v) = old_value_v { - if let Ok(s) = std::str::from_utf8(v) { - if let Ok(parsed) = s.parse::() { - old_value_int = parsed; - } - } - } + let client = self.client.clone(); + let mut generation = read_counter_generation(&client, &key, self.rpc_max_retries).await?; + let mut minimum_counter_value = 0_i64; + let mut cas_conflicts = 0_u32; + let mut rpc_retries_used = 0_u32; - if old_value_v.is_none() { - // First-time create: only succeed if key does not exist - let compare = Compare::create_revision(key.clone(), CompareOp::Equal, 0); - let put_op = match self.lease_id { - Some(lease_id) => { - let put_opts = PutOptions::new().with_lease(lease_id); - TxnOp::put(key.clone(), "1", Some(put_opts)) - } - None => TxnOp::put(key.clone(), "1", None), - }; - let txn = Txn::new().when(vec![compare]).and_then(vec![put_op]); - let txn_res = client.txn(txn).await.with_context(|| { - format!("transaction failed when creating dist_id key {key}") - })?; - if txn_res.succeeded() { - debug!("created dist_id key {} with value 1", key); - return Ok(1); + while cas_conflicts < MAX_CAS_CONFLICTS { + let candidate = next_candidate(generation.as_ref(), minimum_counter_value)?; + let txn = allocator_cas_txn(&key, generation.as_ref(), candidate, self.lease_id); + let mut attempt_client = client.clone(); + + match attempt_client.txn(txn).await { + Ok(txn_res) if txn_res.succeeded() => { + debug!("updated dist_id key {} to value {}", key, candidate); + return Ok(candidate); + } + Ok(txn_res) => { + // A false compare means this candidate was not allocated by + // this call. Keep it below the next candidate even if the + // competing generation disappears before readback. + minimum_counter_value = minimum_counter_value.max(candidate); + generation = allocator_cas_readback(&txn_res, &key)?; + cas_conflicts += 1; } - } else { - // Compare-and-set on existing value - let expected = old_value_v.clone().unwrap_or_default(); - let compare = Compare::value(key.clone(), CompareOp::Equal, expected); - let new_int = old_value_int + 1; - let put_op = match self.lease_id { - Some(lease_id) => { - let put_opts = PutOptions::new().with_lease(lease_id); - TxnOp::put(key.clone(), new_int.to_string(), Some(put_opts)) + Err(error) if is_transient_etcd_error(&error) => { + if rpc_retries_used >= self.rpc_max_retries { + return Err(anyhow!( + "dist_id CAS for key {} remained uncertain after {} retries: {}", + key, + rpc_retries_used, + error + )); } - None => TxnOp::put(key.clone(), new_int.to_string(), None), - }; - let txn = Txn::new().when(vec![compare]).and_then(vec![put_op]); - let txn_res = client.txn(txn).await.with_context(|| { - format!("transaction failed when updating dist_id key {key}") - })?; - if txn_res.succeeded() { - debug!("updated dist_id key {} to value {}", key, new_int); - return Ok(new_int); + + // The server may have committed `candidate`. Never return it + // after losing the response: another allocator may have won + // the same value. Recover an authoritative generation and + // require the next CAS to publish a strictly larger value. + minimum_counter_value = minimum_counter_value.max(candidate); + rpc_retries_used += 1; + warn!( + dist_id_key = %key, + candidate, + retry = rpc_retries_used, + max_retries = self.rpc_max_retries, + error = %error, + "dist_id CAS response was uncertain; abandoning candidate" + ); + tokio::time::sleep(UNCERTAIN_RPC_RETRY_DELAY).await; + generation = loop { + match read_counter_generation(&client, &key, 0).await { + Ok(generation) => break generation, + Err(read_error) + if is_transient_counter_read_error(&read_error) + && rpc_retries_used < self.rpc_max_retries => + { + rpc_retries_used += 1; + warn!( + dist_id_key = %key, + retry = rpc_retries_used, + max_retries = self.rpc_max_retries, + error = %read_error, + "dist_id uncertainty recovery Get failed; retrying" + ); + tokio::time::sleep(UNCERTAIN_RPC_RETRY_DELAY).await; + } + Err(read_error) => return Err(read_error), + } + }; + } + Err(error) => { + return Err(anyhow!( + "transaction failed when updating dist_id key {}: {}", + key, + error + )); } } - - // On failure, advance our local guess and try again, just like Python. - let next_int = old_value_int + 1; - old_value_v = Some(next_int.to_string().into_bytes()); } Err(anyhow!( - "DistributeIdAllocator with prefix {} failed to allocate id after 100 retries", - self.prefix + "DistributeIdAllocator with prefix {} failed after {} CAS conflicts", + self.prefix, + MAX_CAS_CONFLICTS )) } } + +async fn read_counter_generation( + client: &Client, + key: &str, + max_retries: u32, +) -> Result> { + let resp = retry_etcd_rpc(max_retries, "get_dist_id_counter", || { + let mut attempt_client = client.clone(); + let attempt_key = key.to_string(); + async move { attempt_client.get(attempt_key, None).await } + }) + .await + .with_context(|| format!("failed to get dist_id key {key}"))?; + counter_generation_from_get(&resp, key) +} + +fn counter_generation_from_get(resp: &GetResponse, key: &str) -> Result> { + if resp.kvs().len() > 1 { + return Err(anyhow!( + "dist_id Get returned duplicate exact keys for {}", + key + )); + } + resp.kvs() + .first() + .map(|kv| CounterGeneration::from_kv(kv, key)) + .transpose() +} + +fn allocator_cas_txn( + key: &str, + generation: Option<&CounterGeneration>, + candidate: i64, + lease_id: Option, +) -> Txn { + let compares = match generation { + Some(generation) => vec![ + Compare::mod_revision(key, CompareOp::Equal, generation.mod_revision), + Compare::lease(key, CompareOp::Equal, generation.lease_id), + Compare::value(key, CompareOp::Equal, generation.value.clone()), + ], + None => vec![Compare::create_revision(key, CompareOp::Equal, 0)], + }; + let put = match lease_id { + Some(lease_id) => TxnOp::put( + key, + candidate.to_string(), + Some(PutOptions::new().with_lease(lease_id)), + ), + None => TxnOp::put(key, candidate.to_string(), None), + }; + Txn::new() + .when(compares) + .and_then(vec![put]) + .or_else(vec![TxnOp::get(key, None)]) +} + +fn allocator_cas_readback(txn_res: &TxnResponse, key: &str) -> Result> { + let responses = txn_res.op_responses(); + let [TxnOpResponse::Get(get)] = responses.as_slice() else { + return Err(anyhow!( + "dist_id CAS readback returned invalid response shape for key {}: operations={}", + key, + responses.len() + )); + }; + if get.kvs().len() > 1 { + return Err(anyhow!( + "dist_id CAS readback returned duplicate exact keys for {}", + key + )); + } + get.kvs() + .first() + .map(|kv| CounterGeneration::from_kv(kv, key)) + .transpose() +} + +fn next_candidate( + generation: Option<&CounterGeneration>, + minimum_counter_value: i64, +) -> Result { + // Preserve the historical behavior of repairing a malformed counter from + // zero, while the exact generation fence still prevents overwriting a + // concurrently changed value. + let observed = generation + .and_then(|generation| std::str::from_utf8(&generation.value).ok()) + .and_then(|value| value.parse::().ok()) + .unwrap_or(0); + let base = observed.max(minimum_counter_value); + base.checked_add(1) + .ok_or_else(|| anyhow!("dist_id counter overflow at {}", base)) +} + +fn is_transient_counter_read_error(error: &anyhow::Error) -> bool { + error.chain().any(|source| { + source + .downcast_ref::() + .is_some_and(is_transient_etcd_error) + }) +} + +#[cfg(test)] +mod tests { + use anyhow::Context; + use tonic::Status; + + use super::{CounterGeneration, is_transient_counter_read_error, next_candidate}; + + fn generation(value: i64, mod_revision: i64) -> CounterGeneration { + CounterGeneration { + value: value.to_string().into_bytes(), + lease_id: 11, + mod_revision, + } + } + + #[test] + fn ordinary_candidate_advances_observed_counter() { + assert_eq!(next_candidate(Some(&generation(41, 7)), 0).unwrap(), 42); + assert_eq!(next_candidate(None, 0).unwrap(), 1); + } + + #[test] + fn uncertain_candidate_is_always_left_as_a_gap() { + let uncertain_candidate = 42; + assert_eq!(next_candidate(None, uncertain_candidate).unwrap(), 43); + assert_eq!( + next_candidate(Some(&generation(41, 7)), uncertain_candidate).unwrap(), + 43 + ); + assert_eq!( + next_candidate(Some(&generation(42, 8)), uncertain_candidate).unwrap(), + 43 + ); + assert_eq!( + next_candidate(Some(&generation(50, 9)), uncertain_candidate).unwrap(), + 51 + ); + } + + #[test] + fn malformed_counter_is_repaired_without_bypassing_generation_fence() { + let malformed = CounterGeneration { + value: b"invalid".to_vec(), + lease_id: 11, + mod_revision: 9, + }; + assert_eq!(next_candidate(Some(&malformed), 0).unwrap(), 1); + } + + #[test] + fn recovery_detects_transient_etcd_error_through_context() { + let result: Result<(), etcd_client::Error> = Err(etcd_client::Error::GRpcStatus( + Status::unavailable("temporary"), + )); + let error = result.context("recovery Get failed").unwrap_err(); + assert!(is_transient_counter_read_error(&error)); + } +} diff --git a/fluxon_rs/fluxon_util/src/etcd/mod.rs b/fluxon_rs/fluxon_util/src/etcd/mod.rs index 4950fab..159ed79 100755 --- a/fluxon_rs/fluxon_util/src/etcd/mod.rs +++ b/fluxon_rs/fluxon_util/src/etcd/mod.rs @@ -2,8 +2,9 @@ pub mod cluster_lease; pub mod etcd_clients_pool; pub mod id_allocator; pub mod prefix_watch_actor; +pub mod rpc_retry; -pub use cluster_lease::get_cluster_lease_id; +pub use cluster_lease::{get_cluster_lease_id, get_cluster_lease_id_with_retry}; pub use etcd_clients_pool::{ EtcdClientsPool, PooledEtcdClient, PooledEtcdClientSnapshot, etcd_clients_pool, }; @@ -12,3 +13,4 @@ pub use prefix_watch_actor::{ ETCD_PREFIX_WATCH_RESTART_SLEEP, EtcdPrefixWatchLoopControl, OwnedEtcdWatchEvent, OwnedEtcdWatchEventKind, run_prefix_watch_loop, }; +pub use rpc_retry::{DEFAULT_ETCD_RPC_MAX_RETRIES, is_transient_etcd_error, retry_etcd_rpc}; diff --git a/fluxon_rs/fluxon_util/src/etcd/rpc_retry.rs b/fluxon_rs/fluxon_util/src/etcd/rpc_retry.rs new file mode 100644 index 0000000..35468f6 --- /dev/null +++ b/fluxon_rs/fluxon_util/src/etcd/rpc_retry.rs @@ -0,0 +1,215 @@ +use std::future::Future; +use std::time::Duration; + +use etcd_client::Error; +use tonic::Code; + +/// Number of retries after the first etcd RPC attempt. +pub const DEFAULT_ETCD_RPC_MAX_RETRIES: u32 = 2; + +const INITIAL_RETRY_DELAY: Duration = Duration::from_millis(50); +const MAX_RETRY_DELAY: Duration = Duration::from_secs(1); + +/// Returns whether an etcd error is safe to retry as a transient RPC failure. +pub fn is_transient_etcd_error(error: &Error) -> bool { + match error { + Error::IoError(error) => matches!( + error.kind(), + std::io::ErrorKind::ConnectionRefused + | std::io::ErrorKind::ConnectionReset + | std::io::ErrorKind::ConnectionAborted + | std::io::ErrorKind::NotConnected + | std::io::ErrorKind::HostUnreachable + | std::io::ErrorKind::NetworkUnreachable + | std::io::ErrorKind::BrokenPipe + | std::io::ErrorKind::TimedOut + | std::io::ErrorKind::WouldBlock + | std::io::ErrorKind::Interrupted + | std::io::ErrorKind::UnexpectedEof + ), + Error::TransportError(_) => true, + Error::GRpcStatus(status) => matches!( + status.code(), + Code::DeadlineExceeded | Code::ResourceExhausted | Code::Unavailable + ), + _ => false, + } +} + +/// Runs one unary etcd RPC and retries transient failures. +/// +/// `max_retries` counts attempts after the initial call. Callers remain responsible for +/// choosing operations whose semantics permit replay. +pub async fn retry_etcd_rpc( + max_retries: u32, + operation: &str, + mut rpc: F, +) -> Result +where + F: FnMut() -> Fut, + Fut: Future>, +{ + let mut retries = 0; + + loop { + match rpc().await { + Ok(value) => return Ok(value), + Err(error) if is_transient_etcd_error(&error) && retries < max_retries => { + retries += 1; + let delay = retry_delay(retries); + tracing::warn!( + operation, + retry = retries, + max_retries, + delay_ms = delay.as_millis(), + error = %error, + "transient etcd RPC failure; retrying" + ); + tokio::time::sleep(delay).await; + } + Err(error) => { + if is_transient_etcd_error(&error) { + tracing::warn!( + operation, + attempts = retries.saturating_add(1), + max_retries, + error = %error, + "transient etcd RPC failure; retry budget exhausted" + ); + } else { + tracing::debug!( + operation, + attempts = retries.saturating_add(1), + error = %error, + "etcd RPC failure is not retryable" + ); + } + return Err(error); + } + } + } +} + +fn retry_delay(retry: u32) -> Duration { + let exponent = retry.saturating_sub(1).min(5); + (INITIAL_RETRY_DELAY * (1_u32 << exponent)).min(MAX_RETRY_DELAY) +} + +#[cfg(test)] +mod tests { + use std::io; + use std::sync::atomic::{AtomicUsize, Ordering}; + + use etcd_client::Error; + use tonic::Status; + + use super::{is_transient_etcd_error, retry_etcd_rpc}; + + fn unavailable() -> Error { + Error::GRpcStatus(Status::unavailable("temporary outage")) + } + + #[test] + fn classifies_only_explicit_transient_statuses() { + assert!(is_transient_etcd_error(&unavailable())); + assert!(is_transient_etcd_error(&Error::GRpcStatus( + Status::deadline_exceeded("deadline") + ))); + assert!(is_transient_etcd_error(&Error::GRpcStatus( + Status::resource_exhausted("overloaded") + ))); + assert!(!is_transient_etcd_error(&Error::GRpcStatus( + Status::aborted("transaction conflict") + ))); + assert!(!is_transient_etcd_error(&Error::GRpcStatus( + Status::cancelled("caller cancelled") + ))); + assert!(!is_transient_etcd_error(&Error::GRpcStatus( + Status::invalid_argument("invalid key") + ))); + assert!(is_transient_etcd_error(&Error::IoError(io::Error::new( + io::ErrorKind::ConnectionReset, + "connection reset", + )))); + assert!(is_transient_etcd_error(&Error::IoError(io::Error::new( + io::ErrorKind::ConnectionRefused, + "connection refused", + )))); + assert!(is_transient_etcd_error(&Error::IoError(io::Error::new( + io::ErrorKind::NotConnected, + "not connected", + )))); + assert!(is_transient_etcd_error(&Error::IoError(io::Error::new( + io::ErrorKind::HostUnreachable, + "host unreachable", + )))); + assert!(is_transient_etcd_error(&Error::IoError(io::Error::new( + io::ErrorKind::NetworkUnreachable, + "network unreachable", + )))); + assert!(!is_transient_etcd_error(&Error::IoError(io::Error::new( + io::ErrorKind::PermissionDenied, + "bad certificate permissions", + )))); + } + + #[tokio::test] + async fn zero_retries_makes_one_attempt() { + let attempts = AtomicUsize::new(0); + + let result = retry_etcd_rpc(0, "get", || async { + attempts.fetch_add(1, Ordering::SeqCst); + Err::<(), _>(unavailable()) + }) + .await; + + assert!(result.is_err()); + assert_eq!(attempts.load(Ordering::SeqCst), 1); + } + + #[tokio::test] + async fn two_retries_make_three_attempts() { + let attempts = AtomicUsize::new(0); + + let result = retry_etcd_rpc(2, "put", || async { + attempts.fetch_add(1, Ordering::SeqCst); + Err::<(), _>(unavailable()) + }) + .await; + + assert!(result.is_err()); + assert_eq!(attempts.load(Ordering::SeqCst), 3); + } + + #[tokio::test] + async fn non_transient_failure_is_not_retried() { + let attempts = AtomicUsize::new(0); + + let result = retry_etcd_rpc(2, "delete", || async { + attempts.fetch_add(1, Ordering::SeqCst); + Err::<(), _>(Error::InvalidArgs("invalid range".to_string())) + }) + .await; + + assert!(result.is_err()); + assert_eq!(attempts.load(Ordering::SeqCst), 1); + } + + #[tokio::test] + async fn success_stops_before_retry_budget_is_exhausted() { + let attempts = AtomicUsize::new(0); + + let result = retry_etcd_rpc(2, "get", || async { + let attempt = attempts.fetch_add(1, Ordering::SeqCst); + if attempt == 0 { + Err(unavailable()) + } else { + Ok("value") + } + }) + .await; + + assert_eq!(result.unwrap(), "value"); + assert_eq!(attempts.load(Ordering::SeqCst), 2); + } +} diff --git a/upgrade_version_and_release.md b/upgrade_version_and_release.md index 4118636..8093970 100644 --- a/upgrade_version_and_release.md +++ b/upgrade_version_and_release.md @@ -43,7 +43,7 @@ flowchart LR - `fluxon_rs/Cargo.toml` 的 workspace 版本 - `fluxon_rs/Cargo.toml` 中每个 workspace member 对应的 `Cargo.toml` -公开发布版本与闭源通信 SDK 契约版本相互独立。本轮 `sdk_version` 与 open-surface 契约均保留 `0.2.1`：`fluxon_rs/fluxon_commu_contract/src/lib.rs` 的 `FLUXON_COMMU_OPEN_SURFACE_VERSION` 是公开契约版本的唯一来源，release resolver 会把它与 SDK manifest 的 `required_open_surface_version` 对照，运行时再与 SDK 二进制导出的值对照。只有 open surface 本身发生不兼容变化时才升级该常量；升级前必须重建并验证 SDK 二进制，再更新随二进制生成的 manifest，不能只修改 JSON。 +公开发布版本与闭源通信 SDK 契约版本相互独立。本轮 `ClusterManagerNewArg` 的 bitcode payload 发生不兼容变化，因此 `sdk_version` 与 open-surface 契约同步升级为 `0.2.2`：`fluxon_rs/fluxon_commu_contract/src/lib.rs` 的 `FLUXON_COMMU_OPEN_SURFACE_VERSION` 是公开契约版本的唯一来源，release resolver 会把它与 SDK manifest 的 `required_open_surface_version` 对照，运行时再与 SDK 二进制导出的值对照。只有 open surface 本身发生不兼容变化时才升级该常量；升级前必须重建并验证 SDK 二进制，再更新随二进制生成的 manifest，不能只修改 JSON。 同时新增 `fluxon_release/release_notes/v.md`，并同步更新 `README.md`、`README_CN.md` 和 Quick Start 公开示例。仓库中的 YAML 默认是示例，不为版本升级直接改写。