diff --git a/CHANGELOG.md b/CHANGELOG.md index 7acc90a..37b2a12 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,83 @@ ## [Unreleased] +## [0.7.9] - 2026-07-21 + +### 收敛 + +- endpoint 路径补充低基数 `sharplink.client.attempts`、`retries`、`endpoint_admission.rejected`、`breaker.open` 与 `selection.failures` metrics；默认标签不含 endpoint ID、地址、authority 或 transport 名称。 +- 完成 static/dynamic topology、selector、Retry、custom admission 与 generation-scoped circuit breaker 的本地组合验证；物理 Ready/Draining 状态不因 admission 拒绝或 breaker Open 被伪装成断线。 +- 文档补全迁移路径、传输限制、Retry/Breaker 语义与 0.7.x API freeze 审核说明。 + +## [0.7.8] - 2026-07-21 + +### 新增 + +- `ISharpLinkEndpointAdmissionPolicy`、`SharpLinkEndpointAdmissionDecision` 和 `SharpLinkEndpointOutcome` 提供 endpoint 级 TryAcquire/Report SPI；`UseEndpointAdmission` 显式启用自定义策略。 +- `UseCircuitBreaker` 和 `SharpLinkCircuitBreakerOptions` 提供按 endpoint generation 隔离的 Closed/Open/HalfOpen breaker，默认关闭。 + +### 变更 + +- endpoint selection 在连接选择之前执行 admission；拒绝候选会继续选择其他 Ready endpoint，实际获得许可的 attempt 沿用 PendingCall 单一终结路径恰好 Report 一次。 +- breaker 使用 monotonic time、惰性状态推进、有限采样 ring 和 HalfOpen 原子 permit，不创建每 endpoint timer；连接、拓扑与 `CheckHealthAsync` 的物理语义保持不变。 + +## [0.7.7] - 2026-07-21 + +### 新增 + +- `UseRetry()`、`UseRetry(Action)` 和 `UseRetry(ISharpLinkRetryPolicy)` 为显式标记 `[Idempotent]` 的 Unary 提供可选重试；默认最多三次、50/100/200 ms 指数退避和 ±20% jitter。 +- `ISharpLinkRetryPolicy`、`SharpLinkRetryContext` 与 `SharpLinkRetryDecision` 提供同步、无 I/O 的自定义决策 SPI；非法 delay 或 policy 异常只失败当前 logical call，不影响 Client 健康。 +- Retry attempt 复用既有 PendingCall 的单一完成仲裁，记录 endpoint/generation、connection、完成原因、响应是否已观测和耗时；无第二套 pending-request 表。 + +### 变更 + +- Client interceptor 仍只对一次 logical call 执行；Retry 位于其 terminal 内，每次 attempt 重新选择 endpoint，并共享入口冻结的绝对 deadline。 +- 默认策略仅对 `[Idempotent]` Unary 的 endpoint 不可用、连接关闭/切换、发送失败与远端 `Unavailable` 重试；业务错误和 `ResourceExhausted`、OneWay 及所有 Streaming 不自动重试。 +- 多 endpoint retry 使用调用内 `ulong` exclusion mask，优先尝试不同 Ready endpoint；尝试完当前 snapshot 后才复用候选，动态 generation 更新自动形成新候选集。 +- telemetry 保持 logical call 指标一调用一次，并在 listener 存在时额外产生 `sharplink.rpc.attempt` Activity。 + +### 兼容性与验证 + +- Retry 默认关闭，因此固定单 endpoint 的既有 Unary 继续直接走原有路径；Protocol v2 wire format 和握手 capability 未改变。 +- 覆盖远端 `Unavailable`、`ResponseObserved`、非幂等与 `ResourceExhausted` 拒绝重试、绝对 deadline、delay 中取消、custom policy、interceptor 一次性和 endpoint exclusion/reset。 + +## [0.7.6] - 2026-07-21 + +### 新增 + +- `SharpLinkEndpointSnapshot`、`ISharpLinkEndpointResolver` 与 `UseEndpointResolver` 提供版本化的动态 endpoint 拓扑；Client 对 Resolver 拥有明确的 Stop/Dispose 生命周期。 +- `DelegateSharpLinkEndpointResolver` 支持连续 Watch 或单 worker 轮询，可适配 Consul、Nacos、Etcd 等应用已有 SDK，而 SharpLink 核心不引入其依赖。 +- `UseDnsEndpoints` 与 `SharpLinkDnsEndpointResolver` 提供 A/AAAA Discovery、地址族筛选、规范化稳定 ID、hostname Authority、last-good 保留和可配置 refresh/jitter。 + +### 变更 + +- 动态快照以单 writer 原子协调：新增 ID 建立 generation；同 ID 的 Address/Authority 变化替换 generation；Attributes-only 更新保留连接；删除 endpoint 立即停止新调用并排空已有 Unary/Streaming。 +- Resolver Watch 结束或异常后以 100 ms–30 s 的指数退避和 ±20% jitter 重启；空拓扑可恢复且继续遵守 WaitForReady、deadline、cancel 与 Stop。 +- retired connection 使用独立预算。预算超出时抑制 replacement 而不强杀用户 stream，归零后 factory 恰好释放一次。 + +### 兼容性与验证 + +- Protocol v2 wire format、握手 capability、固定单 endpoint 与静态 cluster 的调用路径未改变；无新 NuGet 或第三方服务发现 SDK。 +- 覆盖 add/remove/replace、属性更新、DNS、watch/retry、流排空、PackageSmoke、NativeAOT 及动态稳态矩阵；固定 TCP 五轮 A/B 的 QPS 中位数为 0.7.5 的 100.44%，P99 中位数保持 72 µs。 + +## [0.7.5] - 2026-07-21 + +### 新增 + +- Client 新增不可变的 `SharpLinkEndpoint`、显式传输地址和 transport factory 注册模型；`UseEndpoint`、`UseEndpoints`、`UseCluster` 及四种内置负载均衡策略可在不影响旧单端点用法的前提下构建静态端点集群。 +- 静态集群支持 TCP（hostname/IPv4/IPv6）、Unix Domain Socket、Named Pipe、Shared Memory 和既有 Anonymous Pipe；端点属性可传给自定义选择器。 +- 集群按端点独立维护连接、重连和健康状态，初始连接受 `MaxConnections` 与并行度上限约束；支持最少就绪端点、LeastPending、P2C、Random、RoundRobin 与自定义选择器。 + +### 变更 + +- `GoAway`、Stop 和 Dispose 进入排空流程后，新调用立即选择仍健康的端点；长 Unary 和流式调用在预算内继续完成，超出独立 retiring budget 时才被定点终止。 +- 单个静态端点继续折叠为既有固定连接快速路径；多端点的成员快照仅在就绪成员增减时重建，调用路径仅读取原子快照和实时计数。 +- Protocol v2 wire format、默认传输语义和现有公共配置保持兼容，未引入新的 NuGet 依赖。 + +### 测试与性能 + +- 覆盖多传输、地址族、故障/重连、GoAway 排空、所有选择策略、TLS、包使用与 NativeAOT；固定 TCP Unary 的五轮本地 A/B 中，吞吐中位数为基线的 100.27%，P99 为 104.48%，BenchmarkDotNet 分配保持 352 B/op。 + ## [0.7.4] - 2026-07-20 ### 新增 diff --git a/Directory.Build.props b/Directory.Build.props index c2d4592..cf8e406 100644 --- a/Directory.Build.props +++ b/Directory.Build.props @@ -7,7 +7,7 @@ - 0.7.4 + 0.7.9 sunsi MIT false diff --git a/README.md b/README.md index c7b2dbd..f6c8afc 100644 --- a/README.md +++ b/README.md @@ -517,6 +517,13 @@ if (health.Status != SharpLinkHealthStatus.Ready) - 0.7.1 迁移：`doc/migration-0.7.1.md` - 0.7.2 性能与迁移：`doc/performance-0.7.2.md`、`doc/migration-0.7.2.md` - 0.7.4 压缩、接入控制与性能：`doc/migration-0.7.4.md`、`doc/performance-0.7.4.md` +- 0.7.5 静态多 endpoint：`doc/architecture-0.7.5.md`、`doc/performance-0.7.5.md` +- 0.7.6 动态 endpoint、Resolver 与 DNS Discovery：`doc/architecture-0.7.6.md` +- 0.7.6 本地性能证据：`doc/performance-0.7.6.md` +- 0.7.7 logical call、attempt 与 retry：`doc/architecture-0.7.7.md` +- 0.7.8 endpoint admission 与 circuit breaker：`doc/architecture-0.7.8.md` +- 0.7.9 迁移、组合验证与 API freeze：`doc/migration-0.7.9.md` +- 0.7.9 本地性能与组合 smoke：`doc/performance-0.7.9.md` - 0.6.10 性能与 Chaos：`doc/performance-0.6.10.md`、`doc/chaos-0.6.10.md` - 贡献指南：`CONTRIBUTING.md` - 更新日志：`CHANGELOG.md` diff --git a/doc/architecture-0.7.5.md b/doc/architecture-0.7.5.md new file mode 100644 index 0000000..8832d13 --- /dev/null +++ b/doc/architecture-0.7.5.md @@ -0,0 +1,57 @@ +# SharpLink 0.7.5 静态多端点设计 + +本文档是 0.7.5 的实现设计。它补充 `architecture.md`，在 0.7.5 全部验收前不描述 0.7.6 的动态 Resolver、0.7.7 的 Retry 或后续韧性策略。 + +## 目标与非目标 + +- 保留既有 `UseTransport` 和所有 `UseTcp`、`UseUds`、`UseNamedPipe`、`UseAnonymousPipe`、`UseSharedMemory` 路径的固定单端点快路径。 +- 通过 `UseEndpoint` 或 `UseEndpoints` 显式启用静态 endpoint 配置；单个 endpoint 在 `Build()` 时折叠为固定快路径，两个或更多 endpoint 才构造 Cluster 状态。 +- 一个 Cluster Client 继续只拥有一套 proxy、manifest、interceptor、runtime context 和调用管线。endpoint 不是子 Client。 +- 动态成员变更、Resolver、Retry、Admission、Circuit Breaker 和新增 wire capability 均不在 0.7.5 范围内。 + +## 公共模型 + +`SharpLink.Abstractions` 定义不可变的 `SharpLinkTransportAddress` 记录层次、`SharpLinkEndpoint`、endpoint transport factory 委托和 selector SPI。地址和值对象在构造时验证自身不变量；endpoint 的 ID、属性个数和属性键值长度在 Builder/Cluster snapshot 边界验证。 + +endpoint 进入 Client 时将被深复制：`Id`、`Address` 和 `Authority` 是不可变值，Attributes 复制为只读字典。Client 不保留调用方提供的 collection 或 dictionary 引用。`Address + Authority` 是连接 generation 的身份；0.7.5 的静态拓扑只有 generation `1`，但候选和诊断保留 generation 字段，使该事实不泄漏到调用路径。 + +`SharpLinkEndpointSelectionContext` 为只读 `ref struct`，只携带当前不可变 Ready candidate snapshot、数量和 `ulong` exclusion mask。selector 返回该 snapshot 的索引；越界、被排除或失去 Ready 状态的索引使当前调用以 `FailedPrecondition` 失败，selector 异常只失败当前调用。一个候选时不调用随机数或用户 selector。 + +## Builder 运行模式 + +Builder 在 `Build()` 时一次性冻结为下列模式之一： + +```text +FixedTransport + UseTransport / legacy transport helpers + UseEndpoint(s) with exactly one endpoint + +StaticEndpoints + UseEndpoint(s) with two to 64 endpoints +``` + +`UseTransport` 与 `UseEndpoint(s)` 互斥。固定模式允许 `UseConnectionPool`，不允许 `UseCluster`；集群模式要求 `UseCluster` 或使用其默认快照，不允许 `UseConnectionPool`。Cluster 的连接总量和 endpoint 内连接数分别由 `MaxConnections`、`MaxConnectionsPerEndpoint` 约束；Connecting 和 Ready 都计入总预算，retiring 连接使用独立预算。 + +每次调用 endpoint transport factory 仅由 Client 所有。每个 factory 只会在该 endpoint 的停止/清理路径调用一次 `DisposeAsync`。内置 factory helper 位于 `SharpLink.Client`，将 TCP、UDS、NamedPipe 和 SharedMemory 地址映射到现有 transport；AnonymousPipe 的一次性 handle offer 不支持内置多 endpoint 配置。 + +## Cluster 状态机与连接所有权 + +Static Cluster 的 Client 拥有一个 `StaticClusterRuntime`：冻结的 `EndpointState[]`、不可变 Ready candidate 数组、全局连接 reservation、retiring connection 预算、一次性 topology signal 和已跟踪 worker 集合。每个 `EndpointState` 拥有一个 endpoint generation 的 factory、Ready connection snapshot、Connecting/Ready/Draining 计数、active-call 计数、reconnect 信号和该 endpoint 的 worker。 + +连接建立采用 endpoint-first：在全局预算内先尝试让各 endpoint 各有一条连接，之后才扩展同一 endpoint。最多四个内部连接操作并发。`ConnectAsync` 在任意 endpoint 完成 RPC handshake 后成功；后台继续填充实际 `min(MinReadyEndpoints, endpointCount)`。任一 endpoint 的失败只启动自身退避重连，不阻塞其余 Ready endpoint。Stop 获胜后取消所有 endpoint worker，等待其退出，释放 connection 与 factory，且不会再建立连接。 + +Ready candidate snapshot 只包含至少一条 Ready connection 的 endpoint。它仅在启动完成、连接数在零与非零间转换、或 endpoint 固定成员初始化时重建，并以 `Volatile.Write` 作为同一 endpoint/candidate 对发布。调用路径只读取这一快照与原子计数，不取得 topology writer lock，也不重建数组。Ready/active 计数通过 endpoint-owned provider 实时读取，因此连接扩缩容或 in-flight call 变化不会迫使快照重建。 + +## 选择与调用 + +调用先从 Cluster Ready candidate snapshot 选择 endpoint，再用该 endpoint 内已有的连接 P2C 选择 connection。P2C 用两个不同候选的 `ActiveCallCount / ReadyConnectionCount` 作 64 位交叉相乘比较；Random、RoundRobin 和 LeastPending 也只在静态 snapshot 上运行。RoundRobin cursor 是 Client 实例字段，LeastPending 使用旋转起点消除固定并列偏差。 + +endpoint 在选择后失去最后一个 Ready connection 时，该调用在同一 snapshot 上设置 exclusion bit 并重新选择，最多候选数次；无需 `HashSet`。所有 endpoint 不可用时保留既有 `WaitForReady`、deadline、cancellation 与 Stop 语义。Streaming/OneWay 在开始时选择一个 connection 后一直绑定它；GoAway 仅排空所属 endpoint 中的对应 connection。Retiring connection 不消耗 Ready/Connecting budget，最多保留 `MaxRetiringConnections` 条；超出该独立预算的连接会被关闭并让当前调用得到连接关闭结果。 + +## 验收映射 + +- 公共 address/endpoint/selector/options API 由 Unit 与 PackageSmoke 覆盖。 +- Builder 模式、冻结和 factory ownership 由 Unit 覆盖。 +- P2C、Random、RoundRobin、LeastPending 与 selector failure 由无网络 Unit 覆盖。 +- TCP、UDS、NamedPipe、SharedMemory 多 endpoint 分布、局部故障、独立重连、GoAway、Stop/Dispose 并发由 Integration 覆盖。 +- fixed single 与 one-static-endpoint 继续使用原路径；性能结论只在完整 0.7.5 矩阵完成后写入性能报告。 diff --git a/doc/architecture-0.7.6.md b/doc/architecture-0.7.6.md new file mode 100644 index 0000000..2e85844 --- /dev/null +++ b/doc/architecture-0.7.6.md @@ -0,0 +1,47 @@ +# SharpLink 0.7.6 动态端点与 Resolver 设计 + +本文档描述 0.7.6 在 0.7.5 静态 endpoint cluster 之上增加的动态拓扑能力。它不改变 Protocol v2，不新增握手 capability，也不改变固定单端点或静态 cluster 的调用路径。 + +## 启用方式与边界 + +动态模式必须显式配置，且与 `UseTransport`、`UseEndpoint`、`UseEndpoints` 互斥： + +```csharp +var client = SharpClientBuilder.Create() + .UseEndpointResolver(resolver, SharpLinkTransportFactories.Sockets()) + .UseCluster(options => options.MaxConnections = 4) + .Build(); +``` + +`ISharpLinkEndpointResolver` 位于 `SharpLink.Abstractions`；它只返回完整的 `SharpLinkEndpointSnapshot`，不创建或持有 transport factory。Client 拥有 resolver，并在 Stop/Dispose 中恰好调用一次 `DisposeAsync`。静态与固定模式不会创建 resolver worker、候选数组或额外调用路径。 + +`SharpLinkEndpointSnapshot.Version` 必须严格递增。Client 对 `version <= lastAcceptedVersion` 的快照直接忽略；新快照先完整复制、冻结并校验，再做 reconciliation。重复 ID、非法地址、超出 `MaxEndpoints` 或属性限制会使整份快照被拒绝，最后一个成功拓扑继续服务。空快照合法，它会移除全部新调用候选，但仍允许 `WaitForReady` 等待后续恢复。 + +## Resolver 生命周期 + +`ConnectAsync` 先调用一次 `ResolveAsync`，接受初始拓扑后启动一个有界 watch worker。Worker 正常结束、抛出异常或 Resolve 失败时保留 last-good topology，并按 100 ms 起步、最大 30 s、±20% jitter 的退避重试。任一次成功 Resolve 或 Watch 更新都会复位退避。 + +Stop 先取消 Client 生命周期 token；watch、resolver retry 和 endpoint reconnect 都以该 token 为退出条件。Stop 获胜后不再接受快照或创建连接，随后等待 worker、释放 connection/factory 并 dispose resolver。`DelegateSharpLinkEndpointResolver` 可桥接任意 Consul、Nacos 或 Etcd SDK：提供 watch delegate 时直接转发；没有 watch 时只以一个有界 polling delay 调用 resolve delegate。 + +## 拓扑协调与 generation + +动态 runtime 维护一个单 writer 的 current topology 和独立的 retired generation 集合： + +- 新 ID：创建新的、Client 所有的 factory 与单调递增 generation。 +- 相同 ID 且 `Address + Authority` 相同：复用连接和 factory，仅替换冻结后的 Attributes。 +- 相同 ID 但地址或 Authority 改变：先发布新 generation，再使旧 generation 退出候选并进入 draining。 +- 删除 ID：在同一次原子发布中从候选移除，已有调用与 stream 继续绑定旧 connection 至完成。 + +Ready endpoint/candidate 对以一次 `Volatile.Write` 发布。属性更新也强制重建控制面 candidate snapshot，使自定义 selector 立刻读取新的属性；连接数、active calls 和选择过程仍不获取 topology writer lock。稳定调用不因 resolver 无更新而分配 topology collection。 + +Retiring connection 不计入 active Ready/Connecting budget。超过 `MaxRetiringConnections` 时，拓扑仍然接受更新，但抑制新的 replacement connection，而不会强杀用户 stream；当 draining connection 和 active call 归零后，旧 factory 恰好释放一次。 + +## 内置 DNS Discovery + +`UseDnsEndpoints(host, port, factory, configure)` 使用 `SharpLinkDnsEndpointResolver`。它查询 A/AAAA（可按 `AddressFamily` 过滤），对规范化 IP 去重排序，并由 host、port、address family 和 IP 构造稳定 endpoint ID；原始 host 保留为 Authority，因此 TLS 默认 SNI/证书主机名不随 IP 变化。地址排列变化不会发布 snapshot；地址增加/消失分别表现为 add/remove。 + +BCL 无法提供可移植的 DNS TTL，因此 resolver 使用 `RefreshInterval`（由最小/最大 interval 约束）与可选 jitter，不伪造 TTL。查询失败保留 last-good 结果。DNS 查询器在内部可替换，测试不依赖公网 DNS；一个 resolver 只运行一个 refresh loop，不创建每 endpoint timer。 + +## 验证范围 + +0.7.6 的 Unit/Integration/PackageSmoke 覆盖 DNS 规范化和 last-good、resolver 所有权、初始空拓扑恢复、add/remove、同 ID 地址 generation 替换、Attributes-only 更新、正常 Watch 结束、resolver failure/retry、流排空和新调用迁移。所有行为都是客户端本地行为，继续与 0.7.4 Server 互操作，且保持 NativeAOT 可用。 diff --git a/doc/architecture-0.7.7.md b/doc/architecture-0.7.7.md new file mode 100644 index 0000000..8014629 --- /dev/null +++ b/doc/architecture-0.7.7.md @@ -0,0 +1,46 @@ +# SharpLink 0.7.7 Logical Call、Attempt 与 Retry 设计 + +0.7.7 在 0.7.6 的固定和动态 endpoint topology 之上增加显式、受限的客户端 Retry。它不改变 Protocol v2 wire format 或握手 capability，也不改变未启用 Retry 的固定单 endpoint 调用路径。 + +## 启用与安全边界 + +Retry 默认关闭。通过 Builder 显式启用内置策略或自定义策略： + +```csharp +var client = SharpClientBuilder.Create() + .UseTransport(transport) + .UseRetry(options => + { + options.MaxAttempts = 3; // 包括首次调用 + options.InitialBackoff = TimeSpan.FromMilliseconds(50); + }) + .Build(); +``` + +无论配置了哪个 policy，Core 都只允许 `RpcMethodKind.Unary && IsIdempotent` 自动重试。也就是说，契约方法必须显式标记 `[Idempotent]`；OneWay、客户端/服务端/双向 Streaming 和未标记的 Unary 永远只发起一次调用。此限制不能由自定义 policy 绕过，因为重试包装器只在 generated descriptor 已满足该条件时进入。 + +`MaxAttempts` 是总 attempt 数，范围 1–10，默认值是 3。内置策略只接受 `Unavailable` 或 `ConnectionClosed`，涵盖 endpoint 选择失败、断连、发送失败、连接切换和远端 `Unavailable`。`ResourceExhausted`、认证/授权、参数或业务错误、用户取消和 deadline 都不会被默认策略重试。远端 `Unavailable` 仍可能意味着服务端已实际执行，因此 `[Idempotent]` 是调用方的安全承诺。 + +## Logical call 与 attempt + +Unary 调用先一次性解析 CallOptions、metadata 和 absolute deadline，再执行一次 Client Interceptor 链。Retry 位于 interceptor terminal 内部；每一个 attempt 只运行 endpoint/connection selection、PendingCall 注册、发送和完成等待，绝不重新执行 interceptor。 + +所有 attempt 共用第一次进入 logical call 时计算的绝对 deadline。退避 delay 在 deadline 外会直接以 `DeadlineExceeded` 结束；调用方的取消 token 会同时取消 delay 和后续 attempt。请求 payload 不会为未来的 Retry 提前复制，每个真的 attempt 仍可直接按 codec 序列化。 + +未配置 Retry、或方法不符合安全边界时，Unary 继续调用原来的 `InvokeUnaryCoreAsync` 直接快速路径，不创建 attempt state、mask 或 delay。 + +## 完成结果与 endpoint 选择 + +Retry attempt 使用现有 `PendingRequestTable` 的 compare/exchange 终结仲裁。一个内部 completion observer 随同该 pending entry 生命周期存在，因而 Response、RemoteError、ConnectionClosed、SendFailure、deadline 与 cancel 的竞争仍只会有一个完成者；observer 在 PendingCall 归还对象池前清除。每次 attempt 的内部结果包含 endpoint ID/generation、connection ID、完成原因、是否收到合法 Response/Error、错误码和 elapsed。 + +多 endpoint 调用在 logical call 内维护一个 `ulong` exclusion mask。每次 attempt 重新读取当前 immutable ready snapshot，并优先排除已选 endpoint。64 个候选均已尝试后，若 policy 与预算仍允许，mask 才清零并复用候选；动态 resolver 发布了新 generation 时 snapshot 引用改变，mask 自动从新候选状态重新开始。单 endpoint 场景可以在连接恢复后重试同一 endpoint。 + +## 自定义 Retry policy 与遥测 + +`ISharpLinkRetryPolicy.Evaluate(in SharpLinkRetryContext)` 同步返回 `SharpLinkRetryDecision`。policy 不应阻塞、执行 I/O 或直接发起 attempt。负 delay 或 policy 抛异常将以 `FailedPrecondition` 终止当前 logical call，Client 和连接保持可用。 + +正常 telemetry 仍以 logical call 为单位记录一次。存在 `SharpLink.Client` Activity listener 时，每个网络 attempt 另产生一个 `sharplink.rpc.attempt` Activity，并带固定的 contract、method、kind、attempt number 标签；逻辑调用 counters 不会因重试而重复计数。endpoint ID 和 connection ID 保留在内部 outcome/诊断路径，不进入默认 Meter 标签。 + +## 验证范围 + +0.7.7 Unit 覆盖 remote `Unavailable`、response observation、non-idempotent/`ResourceExhausted` 拒绝、共享 absolute deadline、delay cancellation、custom policy 的非法 delay、interceptor 一次执行以及两 endpoint exclusion/reset。全量验证继续覆盖 PackageSmoke、NativeAOT 和未启用 Retry 的性能门禁。 diff --git a/doc/architecture-0.7.8.md b/doc/architecture-0.7.8.md new file mode 100644 index 0000000..2e1204f --- /dev/null +++ b/doc/architecture-0.7.8.md @@ -0,0 +1,40 @@ +# SharpLink 0.7.8 Endpoint Admission 与 Circuit Breaker 设计 + +0.7.8 为 static 与 resolver-backed endpoint topology 增加客户端 endpoint 级准入。它不同于服务端 Admission Control：服务端策略决定某个 RPC 是否进入服务实现；endpoint admission 决定客户端是否应向某个 Ready endpoint 发起一次网络 attempt。固定 `UseTransport` 模式没有 endpoint topology，不创建 admission state。 + +## 自定义 admission + +使用 `UseEndpointAdmission(ISharpLinkEndpointAdmissionPolicy)` 注册策略： + +```csharp +builder.UseEndpointAdmission(new ZoneAndHealthPolicy()); +``` + +`TryAcquire(in SharpLinkEndpointCandidate, in RpcMethodDescriptor)` 是同步、非阻塞的选择路径。它可返回拒绝、opaque token 和可选 `RetryAfter`。拒绝会把当前候选排除并继续 selector 的剩余候选；所有候选都拒绝时调用以 `Unavailable` 结束，`WaitForReady` retry 路径会遵守最早 RetryAfter、absolute deadline、cancel 和 Stop。 + +Policy 获准的 attempt 将在原有 PendingCall 的唯一终结路径中一次性调用 `Report(in SharpLinkEndpointOutcome, token)`。Response、RemoteError、SendFailure、ConnectionClosed、GoAway、deadline、cancel 与 one-way 本地完成都得到固定分类。TryAcquire 异常或负 RetryAfter 以 `FailedPrecondition` 结束当前调用；Report 异常只记录日志，绝不覆盖已经获得的 RPC 结果或关闭连接。 + +## 内置 Circuit Breaker + +```csharp +builder.UseCircuitBreaker(options => +{ + options.MinimumThroughput = 20; + options.FailureRatio = 0.5; + options.SamplingDuration = TimeSpan.FromSeconds(30); + options.BreakDuration = TimeSpan.FromSeconds(10); + options.HalfOpenMaxCalls = 1; +}); +``` + +内置 breaker 是 admission policy 的另一种实现，因此不能与自定义 admission 同时注册。每个 `{endpoint ID, generation}` 拥有独立状态：地址或 authority 替换形成新 generation，并从 Closed 开始；动态 retired generation 排空时会释放其 breaker state。 + +- **Closed**：在有界滚动时间窗记录 endpoint 可用性样本；达到 MinimumThroughput 且失败比例不低于 FailureRatio 时 Open。 +- **Open**：在 BreakDuration 内拒绝新 attempt，但不会修改物理连接状态或从 topology 删除 endpoint。 +- **HalfOpen**：到期后按原子 permit 放行最多 HalfOpenMaxCalls 个 probe；成功清空旧窗口并回到 Closed，基础设施失败重新 Open。 + +Breaker 只计入连接关闭、GoAway、发送故障、远端/本地 Unavailable、远端 ResourceExhausted、DataLoss 和 Internal 等基础设施故障。用户取消、deadline、认证/授权、参数/业务错误与本地 Pending/queue ResourceExhausted 不会打开 breaker。状态推进使用 `Stopwatch` 单调时间，没有每 endpoint timer 或 topology writer lock。 + +## 与 Retry 的顺序 + +每个 logical Retry attempt 依次执行：Retry budget → endpoint selector → admission/breaker → connection selection → PendingCall/send → terminal Report → retry decision。admission 拒绝不会发送网络请求；Retry 仍优先选择尚未尝试的 endpoint。breaker 全 Open 不改变 Client 的物理 Ready/Reconnecting 状态，`CheckHealthAsync` 依旧只查询真实 Ready connection。 diff --git a/doc/architecture.md b/doc/architecture.md index 73b9da2..a56fc0a 100644 --- a/doc/architecture.md +++ b/doc/architecture.md @@ -95,6 +95,15 @@ SharpLink.Serializer.MemoryPack - `GoAway` 将单条 session 标为 draining 并立即从选择快照移除；已有请求完成后释放该连接，池在后台恢复最小连接数。 - Client Stop 取消并等待 connect、reconnect、expand、heartbeat 与 read-loop worker，再释放所有 session 和 transport factory。 +## 0.7.x endpoint 拓扑与韧性 + +- 固定单 endpoint 仍是默认快路径；只有显式 `UseEndpoints` 或 `UseEndpointResolver` 才会创建 endpoint candidate、selector 和后台 topology worker。单个 static endpoint 在 Build 时折叠回固定快路径。 +- static 和 dynamic cluster 都以不可变 Ready candidate snapshot 供调用路径读取；端点增减或 Ready 边界变化由单 writer 发布，选择路径不获取 topology writer lock。多 endpoint 默认 P2C，可显式选择 Random、RoundRobin、LeastPending 或同步自定义 selector。 +- Resolver snapshot 按版本验证并原子 reconcile：新 ID 创建 generation，Address/Authority 变化替换 generation 并排空旧连接，仅 Attributes 更新保留连接。空 snapshot 合法；resolver 故障或 Watch 结束保留 last-good topology 并退避恢复。 +- Retry 默认关闭，只对显式 `[Idempotent]` Unary 生效；拦截器按 logical call 执行一次，每次 attempt 重新选择 endpoint 并共享入口冻结的绝对 deadline。任何 Streaming 或 OneWay 不会被自动重试。 +- Endpoint admission 和 Circuit Breaker 只决定是否发起新 attempt，不会修改物理 connection 的 Ready 语义。Breaker 状态按 endpoint generation 隔离，以 monotonic time 惰性推进，HalfOpen 使用原子 probe permit。 +- `SharpLinkTelemetry` 无 listener 时不创建 TagList、Activity 或动态字符串。endpoint 路径提供 active/ready/draining endpoint、resolver update/failure、active/retiring connection、attempt、retry、admission rejection、breaker open 的低基数指标；endpoint ID、address 和 authority 只出现在 Activity 或结构化日志中。 + ## 取消与超时 1. 调用侧 `CancellationToken`、monotonic deadline 或 stream consumer early-break 通过客户端 PendingCall 的单一 CAS 终态仲裁。 diff --git a/doc/loadtest.md b/doc/loadtest.md index 3b8b6dd..8701dc9 100644 --- a/doc/loadtest.md +++ b/doc/loadtest.md @@ -181,6 +181,16 @@ StreamLoadTest 专有： `eng/run-v074-performance-evidence.sh` 额外运行 0.7.3/0.7.4 关闭扩展的交替 A/B、admission 立即/排队/拒绝，以及压缩算法、level、收益阈值和 payload pattern 矩阵。完整 JSON 和 BDN 报告可由 `0.7.4 Performance Evidence` workflow 上传为 artifact。 +## 0.7.5 静态 endpoint 矩阵 + +`eng/run-v075-static-performance-matrix.sh` 使用真实本地 TCP 服务实例和 `UseEndpoint(s)` API 覆盖静态 endpoint 路径：一个 endpoint 走 Builder 折叠后的固定路径，多个 endpoint 走 static cluster。`smoke` tier 覆盖 1/2 endpoint、并发 1/8、payload 0/256 B 和 P2C/LeastPending；`full` tier 扩展为 endpoint 1/2/8/32、并发 1/8/32/128、payload 0/32/256/4096/65536 B，以及 P2C、Random、RoundRobin、LeastPending。默认运行 JIT；设置 `SHARPLINK_V075_MATRIX_RUNTIMES=jit,aot` 会同时发布并运行本机 RID 的 NativeAOT 负载程序。所有原始 JSON 写入 `artifacts/performance/v0.7.5/static/`。 + +## 0.7.6 动态 Resolver 矩阵 + +`eng/run-v076-dynamic-performance-matrix.sh` 使用 `UseEndpointResolver` 和固定不变的 Delegate snapshot，在真实本地 TCP 服务上测量 resolver 模式的稳态调用。它不把 resolver 更新混入测量窗口，因此可验证动态 cluster 在没有新 snapshot 时不会周期性重建候选数组。`smoke` tier 覆盖 1/2 endpoint、并发 1/8、payload 0/256 B 与 P2C/LeastPending；`full` tier 覆盖 1/2/8/32 endpoint、并发 1/8/32/128、payload 0/32/256/4096/65536 B 和全部内置策略。默认 JIT；设置 `SHARPLINK_V076_MATRIX_RUNTIMES=jit,aot` 同时验证 NativeAOT。原始 JSON 写入 `artifacts/performance/v0.7.6/dynamic/`。 + +正式性能结论仍须以相同硬件上的 0.7.4 fixed-single 基线交替多轮比较，检查 QPS 中位数、P99 和 alloc/op；短时 smoke 只验证矩阵可运行且无请求错误。 + 运行矩阵或 trace 前必须确认同机没有其他 LoadTest、StreamLoadTest、Chaos 或诊断采集进程。存在资源竞争时，整批吞吐、延迟、分配和 trace 均标记无效并从头重跑；错误数与资源归零仍可单独作为正确性线索，但不得转化为性能结论。 `LoadTest` 额外输出： diff --git a/doc/migration-0.7.9.md b/doc/migration-0.7.9.md new file mode 100644 index 0000000..33a805b --- /dev/null +++ b/doc/migration-0.7.9.md @@ -0,0 +1,40 @@ +# SharpLink 0.7.x endpoint、Retry 与 resiliency 迁移指南 + +## 从固定单 endpoint 迁移 + +现有 `UseTransport(...)` 保持原样，是默认且最低开销的模式。它不会自动启动 resolver、构造 endpoint candidate、启用 Retry 或 breaker： + +```csharp +var client = SharpClientBuilder.Create() + .UseTransport(SharpLinkTransportFactories.Sockets(/* ... */)) + .Build(); +``` + +需要静态多 endpoint 时，改用 `UseEndpoints` 和每 endpoint 独立 factory。可选的 P2C、Random、RoundRobin、LeastPending 或 custom selector 只影响多 endpoint： + +```csharp +builder.UseEndpoints(endpoints, SharpLinkTransportFactories.Sockets()) + .UseLoadBalancing(SharpLinkLoadBalancingStrategy.PowerOfTwoChoices); +``` + +需要动态服务发现时使用 `UseEndpointResolver`；Client owns resolver，Stop/Dispose 时会取消 watch 并精确释放它。TCP DNS 使用 `UseDnsEndpoints`。Consul、Etcd、Nacos、Kubernetes 等 SDK 由应用封装为 resolver，不会引入 SharpLink 核心依赖。 + +## 传输注意事项 + +TCP、UDS、NamedPipe 与 SharedMemory 可作为 static/dynamic endpoint transport factory。AnonymousPipe 的 handles 只适合固定单 endpoint；多 endpoint 只能由应用提供能为每个 endpoint generation 独立拥有资源的自定义 factory。endpoint Attributes 仅用于 selector/admission 诊断，不能决定连接关键配置；Address 或 Authority 改变会建立新 generation。 + +## Retry 与幂等性 + +Retry 默认关闭。只有显式 `[Idempotent]` Unary 才能被 `UseRetry` 或自定义 retry policy 自动重试。`MaxAttempts` 包括首次 attempt；所有 attempts 共用原始 absolute deadline，用户 cancel 会停止 delay 和后续 attempt。不要把非幂等写入、OneWay 或任何 Streaming 方法标记为可自动重试。 + +默认策略仅重试 endpoint/connection unavailable 与远端 `Unavailable`，不重试 `ResourceExhausted` 或业务错误。服务端可能在响应丢失前已经执行请求，因此业务必须自行保证 `[Idempotent]` 的语义。 + +## Breaker 与服务端 Admission 的区别 + +`UseCircuitBreaker` 是客户端对某个 endpoint generation 是否发起新 attempt 的保护；它不会拒绝服务端已经收到的调用，也不会改变物理 connection 的 Ready 状态。服务端 `UseAdmissionControl` 则限制服务实现的并发、速率与队列。二者可同时使用：Breaker 保护端点可用性，Server Admission 保护服务资源。 + +使用 `UseEndpointAdmission` 可接入外部健康/zone 策略。策略必须同步且非阻塞；不应执行 I/O、发起 RPC 或持有 connection/session。拒绝可以返回 RetryAfter；Report 异常会被记录而不会篡改业务结果。 + +## API freeze 与可观测性 + +0.7.9 固定了 endpoint/address、resolver、selector、Retry 和 admission/breaker 的公共 API 形状。Builder 只接受显式实例，不进行程序集扫描，保持 NativeAOT 可达性。默认 metrics 使用固定低基数标签，绝不包含 endpoint ID、host、authority 或 pipe name；需要逐 endpoint 诊断时使用 Activity/结构化日志，而不是 Meter 标签。 diff --git a/doc/performance-0.7.5.md b/doc/performance-0.7.5.md new file mode 100644 index 0000000..e3376b6 --- /dev/null +++ b/doc/performance-0.7.5.md @@ -0,0 +1,26 @@ +# SharpLink 0.7.5 本地性能报告 + +本报告记录静态 endpoint 功能完成前的本地开发证据。它不是跨平台发布声明；完整的 full matrix 由 `eng/run-v075-static-performance-matrix.sh` 在固定 runner 上重跑。 + +## 固定单 endpoint 门禁 + +- 基线：0.7.4 `faac6833fce06b9464f67f7a977ca63b592395c2`。 +- 候选：静态 endpoint 实现完成后的本地提交 `8ea307296757880d4738af648a17b1783ad3edd2`。 +- 环境：macOS Arm64、Apple M4、.NET 10.0.2、Release、TCP、本地 server/client 同进程、连接池 1/1、Unary `Add`、并发 8。 +- 方法：基线与候选交替五轮；每轮 warmup 1 秒、采样 3 秒。原始 JSON 位于本任务 checkout 的 `artifacts/performance/v0.7.5/`。 + +| Build | QPS r1..r5 | Median QPS | P99 us r1..r5 | Median P99 | +|---|---|---:|---|---:| +| 0.7.4 baseline | 170060.44, 171314.82, 168318.39, 168209.54, 169653.35 | 169653.35 | 64, 64, 70, 67, 70 | 67 | +| 0.7.5 fixed path | 170109.60, 172012.05, 168039.38, 165756.18, 170589.72 | 170109.60 | 71, 67, 71, 70, 65 | 70 | + +结论：QPS 为基线的 100.27%，P99 为基线的 104.48%，满足固定路径 QPS 不低于 99%、P99 不高于 105% 的门槛。 + +BenchmarkDotNet `UnaryBenchmarks.Rpc_Add` 的 `MemoryDiagnoser` 在 payload 16 与 256 下均为 **352 B/op**，与 0.7.4 报告中的固定路径分配相同。LoadTest 的进程级 allocated-bytes 仅用于辅助诊断，不用于取代该每操作分配结论。 + +## 静态 endpoint smoke + +- JIT：`eng/run-v075-static-performance-matrix.sh` 的 smoke tier 已运行，覆盖折叠的 1 endpoint、2 endpoint P2C/LeastPending、并发 1/8、payload 0/256 B；全部请求零失败。 +- NativeAOT：本机 `osx-arm64` 发布的 `SharpLink.LoadTest` 已以 2 endpoint、RoundRobin、Unary `Add`、并发 1 运行成功，QPS 35671.15、P99 57 us、零失败。 + +full tier 额外覆盖 1/2/8/32 endpoint、并发 1/8/32/128、payload 0/32/256/4096/65536 B、四种内置策略，以及 JIT/NativeAOT。它应在没有竞争负载的固定 runner 上运行，作为发布前的完整矩阵证据。 diff --git a/doc/performance-0.7.6.md b/doc/performance-0.7.6.md new file mode 100644 index 0000000..102e45b --- /dev/null +++ b/doc/performance-0.7.6.md @@ -0,0 +1,26 @@ +# SharpLink 0.7.6 本地性能报告 + +本报告记录动态 endpoint/Resolver 完成后的本地开发证据。结果用于回归判断，不替代固定硬件、跨平台发布前的 full matrix。 + +## 固定单端点门禁 + +- 基线：本地 0.7.5 提交 `8b1a2da`。 +- 候选：0.7.6 动态 Resolver 实现完成后的本地工作树。 +- 方法：本机 TCP Unary `Add`、连接池 `1/1`、并发 8；基线和候选交替五轮，每轮 warmup 1 秒、采样 2 秒。原始 JSON 保存在本任务 checkout 的 `artifacts/performance/v0.7.6/`。 + +| 路径 | QPS（五轮） | QPS 中位数 | P99（µs，五轮） | P99 中位数 | +| --- | --- | ---: | --- | ---: | +| 0.7.5 基线 | 163903.11, 162824.50, 164070.70, 165479.71, 163517.55 | 163903.11 | 72, 75, 77, 72, 72 | 72 | +| 0.7.6 当前 | 165734.46, 167550.81, 163221.92, 164619.21, 163226.10 | 164619.21 | 70, 72, 72, 73, 74 | 72 | + +固定路径 QPS 中位数为基线的 **100.44%**，P99 中位数保持 **72 µs**，满足 0.7.6 的固定路径门禁。 + +BenchmarkDotNet `UnaryBenchmarks.Rpc_Add`（Payload 16/256，`MemoryDiagnoser`）均为 **352 B/op**，与 0.7.5 记录一致。报告位于 `artifacts/benchmarks-v076/results/`。 + +## 动态 Resolver 稳态 smoke + +`eng/run-v076-dynamic-performance-matrix.sh` 已在 JIT smoke tier 运行：1/2 endpoint、P2C/LeastPending、并发 1/8、payload 0/256 B，共 16 个真实本地 TCP case，全部零失败。它使用不变的 Delegate snapshot，确认 resolver 模式在没有新快照时不会周期性重建候选 topology。原始 JSON 位于 `artifacts/performance/v0.7.6/dynamic/`。 + +NativeAOT 已发布并运行 `SharpLink.AotSmoke` 的动态 TCP Resolver 路径，输出 `AOT_SMOKE_PASS transport=tcp`。 + +完整发布前矩阵可设置 `SHARPLINK_V076_MATRIX_TIER=full`；它扩展为 1/2/8/32 endpoint、并发 1/8/32/128、payload 0/32/256/4096/65536 B、四种内置策略，并可通过 `SHARPLINK_V076_MATRIX_RUNTIMES=jit,aot` 同时运行 NativeAOT。 diff --git a/doc/performance-0.7.9.md b/doc/performance-0.7.9.md new file mode 100644 index 0000000..e2919e9 --- /dev/null +++ b/doc/performance-0.7.9.md @@ -0,0 +1,18 @@ +# SharpLink 0.7.9 本地性能与组合 smoke + +本记录是本地开发证据，不替代固定硬件、跨平台与 release soak。 + +## endpoint 稳态矩阵 + +本地 JIT smoke 以真实 TCP 服务运行两套 16-case 矩阵，warmup 0 秒、采样 1 秒： + +- static：1/2 endpoint、P2C/LeastPending、并发 1/8、payload 0/256 B；16/16 通过、零失败，原始 JSON 在 `artifacts/performance/v0.7.9/static/`。 +- dynamic Delegate Resolver：同一矩阵；16/16 通过、零失败，原始 JSON 在 `artifacts/performance/v0.7.9/dynamic/`。 + +代表性 endpoint=2、P2C、并发 8 的结果为：0 B **138,682 QPS / P99 84 µs**，256 B **131,127 QPS / P99 87 µs**。这些短窗口用于确认 0.7.7–0.7.9 的 Retry/admission/breaker 默认关闭时，static/dynamic 常规路径没有失败或明显的延迟异常；正式 release 应以五轮交替基准和 full matrix 复核。 + +## 默认路径与 enabled 路径 + +未配置 Retry/Admission/Breaker 时，fixed single endpoint 不创建 resolver、candidate、retry state、admission token 或 breaker state。Retry 仅在显式标记 `[Idempotent]` Unary 且 Builder 启用后创建 attempt state；admission/breaker 仅在 endpoint topology 且显式启用时创建 policy state。 + +`SharpLink.AotSmoke` 已以 `osx-arm64` NativeAOT 发布并运行 TCP 路径，输出 `AOT_SMOKE_PASS transport=tcp`。PackageSmoke 从本地生成的 0.7.9 packages 还原并运行通过。 diff --git a/doc/plan.md b/doc/plan.md index 9c66aa5..987da40 100644 --- a/doc/plan.md +++ b/doc/plan.md @@ -9,7 +9,7 @@ - 默认单连接复用，可配置连接池；默认 `Balanced`，提供 `LowLatency` 与 `Throughput`。 - SDK 内置无反射 DTO Source Generator Codec；MemoryPack 保持可选插件。 - 核心提供 TLS、认证、Interceptor、deadline、背压、健康检查、遥测与优雅停机。 -- Discovery、Load Balancing、Retry 与 Circuit Breaker 位于官方扩展包。 +- Discovery、Load Balancing、Retry 与 Circuit Breaker 内置于现有程序集；第三方注册中心仅通过显式 Resolver SPI 接入。 ## 0.4.0：Runtime 与 Protocol v2 基线（已发布） @@ -49,7 +49,11 @@ 1. `0.7.0`（已完成）：实验性 SharedMemory 传输。 2. `0.7.1`（已完成）：Generator Manifest 自动服务注册、`Singleton/Connection/Call` 生命周期与运行时程序集安全注册/注销。 3. `0.7.2`（已完成）：静态 Singleton Unary 性能恢复、稳态分配削减与完整回归归因。 -4. 后续版本：Discovery、Load Balancing 与只针对 `[Idempotent]` Unary 的 Resilience 扩展。 +4. `0.7.5`（已完成）：静态 endpoint topology、内置负载均衡与 selector SPI。 +5. `0.7.6`（已完成）：动态 Resolver topology、DNS Discovery 与 generation 排空。 +6. `0.7.7`（已完成）：Logical Call/Attempt、仅 `[Idempotent]` Unary 的 Retry。 +7. `0.7.8`（已完成）：endpoint admission SPI 与 generation-scoped Circuit Breaker。 +8. `0.7.9`（已完成）：组合验证、低基数 telemetry、迁移文档与 API freeze。 ## 0.8.0-rc：发布门禁 diff --git a/doc/todo.md b/doc/todo.md index 25eba18..b59206c 100644 --- a/doc/todo.md +++ b/doc/todo.md @@ -38,7 +38,11 @@ - `0.7.0`（已完成）：实验性 SharedMemory 传输。 - `0.7.1`（已完成）：自动服务注册、三种根服务生命周期、运行时程序集原子注册、安全排空与 collectible ALC 验证。 - `0.7.2`（已完成）：静态 Singleton Unary 快路径、请求池分配削减、历史性能二分与五传输回归矩阵。 -- 后续 `0.7.x`：Discovery、Load Balancing 与 Resilience 官方扩展包。 +- `0.7.5`（已完成）：静态 endpoint topology、P2C/Random/RoundRobin/LeastPending 与 custom selector。 +- `0.7.6`（已完成）：动态 Resolver/DNS Discovery、snapshot generation 与 draining。 +- `0.7.7`（已完成）：Logical Call/Attempt、仅 `[Idempotent]` Unary 的 Retry。 +- `0.7.8`（已完成）：endpoint admission SPI、Circuit Breaker、HalfOpen probe 与 generation isolation。 +- `0.7.9`（已完成）：组合验证、低基数 telemetry、迁移文档与 API freeze。 - `0.8.0-rc`：Chaos、长稳、跨平台/AOT/Package/性能 Release Gate，只修复正确性、稳定性、性能回退和文档问题。 - `1.0.0`：通过 RC 接受标准后发布稳定 API 与 Protocol v2。 diff --git a/eng/run-v075-static-performance-matrix.sh b/eng/run-v075-static-performance-matrix.sh new file mode 100755 index 0000000..9419976 --- /dev/null +++ b/eng/run-v075-static-performance-matrix.sh @@ -0,0 +1,86 @@ +#!/usr/bin/env bash +set -euo pipefail + +SCRIPT_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +ROOT="${SHARPLINK_V075_MATRIX_ROOT:-$SCRIPT_ROOT}" +TIER="${SHARPLINK_V075_MATRIX_TIER:-smoke}" +RUNTIMES="${SHARPLINK_V075_MATRIX_RUNTIMES:-jit}" +OUTPUT_ROOT="${SHARPLINK_V075_MATRIX_OUTPUT:-$ROOT/artifacts/performance/v0.7.5/static}" +WARMUP="${SHARPLINK_V075_MATRIX_WARMUP:-1}" +DURATION="${SHARPLINK_V075_MATRIX_DURATION:-3}" + +case "$TIER" in + smoke) + ENDPOINTS="1 2" + CONCURRENCY="1 8" + PAYLOADS="0 256" + STRATEGIES="p2c leastpending" + ;; + full) + ENDPOINTS="1 2 8 32" + CONCURRENCY="1 8 32 128" + PAYLOADS="0 32 256 4096 65536" + STRATEGIES="p2c random roundrobin leastpending" + ;; + *) + echo "Unsupported SHARPLINK_V075_MATRIX_TIER: $TIER" >&2 + exit 2 + ;; +esac + +mkdir -p "$OUTPUT_ROOT" +cd "$ROOT" +dotnet build test/SharpLink.LoadTest/SharpLink.LoadTest.csproj -c Release -v minimal + +run_case() { + local runtime="$1" + local endpoint_count="$2" + local strategy="$3" + local concurrency="$4" + local payload="$5" + local output="$6" + local args=( + --mode local --transport tcp --static-endpoints "$endpoint_count" + --load-balancing "$strategy" --operation echo --payload-size "$payload" + --concurrency "$concurrency" --warmup "$WARMUP" --duration "$DURATION" + --metrics-port 0 --json-output "$output") + + if [[ "$runtime" == "jit" ]]; then + dotnet run -c Release --no-build --project test/SharpLink.LoadTest/SharpLink.LoadTest.csproj -- "${args[@]}" + else + "$OUTPUT_ROOT/aot/SharpLink.LoadTest" "${args[@]}" + fi +} + +IFS=',' read -r -a RUNTIME_LIST <<< "$RUNTIMES" +for runtime in "${RUNTIME_LIST[@]}"; do + case "$runtime" in + jit) ;; + aot) + RID="" + case "$(uname -s)-$(uname -m)" in + Darwin-arm64) RID=osx-arm64 ;; + Darwin-x86_64) RID=osx-x64 ;; + Linux-x86_64) RID=linux-x64 ;; + Linux-aarch64|Linux-arm64) RID=linux-arm64 ;; + *) echo "Unsupported host for NativeAOT: $(uname -s)-$(uname -m)" >&2; exit 2 ;; + esac + dotnet publish test/SharpLink.LoadTest/SharpLink.LoadTest.csproj -c Release -r "$RID" \ + -p:PublishAot=true -o "$OUTPUT_ROOT/aot" + ;; + *) echo "Unsupported runtime: $runtime" >&2; exit 2 ;; + esac + + for endpoint_count in $ENDPOINTS; do + for strategy in $STRATEGIES; do + for concurrency in $CONCURRENCY; do + for payload in $PAYLOADS; do + output="$OUTPUT_ROOT/${runtime}-e${endpoint_count}-${strategy}-c${concurrency}-p${payload}.json" + run_case "$runtime" "$endpoint_count" "$strategy" "$concurrency" "$payload" "$output" + done + done + done + done +done + +echo "0.7.5 static endpoint performance matrix complete: $OUTPUT_ROOT" diff --git a/eng/run-v076-dynamic-performance-matrix.sh b/eng/run-v076-dynamic-performance-matrix.sh new file mode 100755 index 0000000..2583d66 --- /dev/null +++ b/eng/run-v076-dynamic-performance-matrix.sh @@ -0,0 +1,87 @@ +#!/usr/bin/env bash +set -euo pipefail + +SCRIPT_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +ROOT="${SHARPLINK_V076_MATRIX_ROOT:-$SCRIPT_ROOT}" +TIER="${SHARPLINK_V076_MATRIX_TIER:-smoke}" +RUNTIMES="${SHARPLINK_V076_MATRIX_RUNTIMES:-jit}" +OUTPUT_ROOT="${SHARPLINK_V076_MATRIX_OUTPUT:-$ROOT/artifacts/performance/v0.7.6/dynamic}" +WARMUP="${SHARPLINK_V076_MATRIX_WARMUP:-1}" +DURATION="${SHARPLINK_V076_MATRIX_DURATION:-3}" + +case "$TIER" in + smoke) + ENDPOINTS="1 2" + CONCURRENCY="1 8" + PAYLOADS="0 256" + STRATEGIES="p2c leastpending" + ;; + full) + ENDPOINTS="1 2 8 32" + CONCURRENCY="1 8 32 128" + PAYLOADS="0 32 256 4096 65536" + STRATEGIES="p2c random roundrobin leastpending" + ;; + *) + echo "Unsupported SHARPLINK_V076_MATRIX_TIER: $TIER" >&2 + exit 2 + ;; +esac + +mkdir -p "$OUTPUT_ROOT" +cd "$ROOT" +dotnet build test/SharpLink.LoadTest/SharpLink.LoadTest.csproj -c Release -v minimal + +RID="" +case "$(uname -s)-$(uname -m)" in + Darwin-arm64) RID=osx-arm64 ;; + Darwin-x86_64) RID=osx-x64 ;; + Linux-x86_64) RID=linux-x64 ;; + Linux-aarch64|Linux-arm64) RID=linux-arm64 ;; + *) echo "Unsupported host for NativeAOT: $(uname -s)-$(uname -m)" >&2; exit 2 ;; +esac + +run_case() { + local runtime="$1" + local endpoint_count="$2" + local strategy="$3" + local concurrency="$4" + local payload="$5" + local output="$6" + local args=( + --mode local --transport tcp --dynamic-endpoints "$endpoint_count" + --load-balancing "$strategy" --operation echo --payload-size "$payload" + --concurrency "$concurrency" --warmup "$WARMUP" --duration "$DURATION" + --metrics-port 0 --json-output "$output") + + if [[ "$runtime" == "jit" ]]; then + dotnet run -c Release --no-build --project test/SharpLink.LoadTest/SharpLink.LoadTest.csproj -- "${args[@]}" + else + "$OUTPUT_ROOT/aot/SharpLink.LoadTest" "${args[@]}" + fi +} + +IFS=',' read -r -a RUNTIME_LIST <<< "$RUNTIMES" +for runtime in "${RUNTIME_LIST[@]}"; do + case "$runtime" in + jit) ;; + aot) + dotnet publish test/SharpLink.LoadTest/SharpLink.LoadTest.csproj -c Release -r "$RID" \ + -p:PublishAot=true -o "$OUTPUT_ROOT/aot" + ;; + *) echo "Unsupported runtime: $runtime" >&2; exit 2 ;; + esac + + for endpoint_count in $ENDPOINTS; do + for strategy in $STRATEGIES; do + for concurrency in $CONCURRENCY; do + for payload in $PAYLOADS; do + output="$OUTPUT_ROOT/${runtime}-e${endpoint_count}-${strategy}-c${concurrency}-p${payload}.json" + run_case "$runtime" "$endpoint_count" "$strategy" "$concurrency" "$payload" "$output" + done + done + done + done +done + +echo "0.7.6 dynamic endpoint performance matrix complete: $OUTPUT_ROOT" diff --git a/src/SharpLink.Abstractions/SharpLinkEndpointAdmission.cs b/src/SharpLink.Abstractions/SharpLinkEndpointAdmission.cs new file mode 100644 index 0000000..15ffb28 --- /dev/null +++ b/src/SharpLink.Abstractions/SharpLinkEndpointAdmission.cs @@ -0,0 +1,61 @@ +namespace SharpLink.Abstractions; + +/// Describes whether an endpoint may accept one client attempt. +/// Whether the endpoint may receive the attempt. +/// An opaque policy-owned token returned unchanged to . +/// An optional non-negative wait before a rejected endpoint may be considered again. +public readonly record struct SharpLinkEndpointAdmissionDecision( + bool IsAllowed, + long Token, + TimeSpan? RetryAfter); + +/// Classifies the terminal result of one endpoint-bound client attempt. +public enum SharpLinkEndpointOutcomeKind : byte +{ + /// The endpoint returned a successful response or locally completed a one-way call. + Success, + /// The endpoint returned a structured RPC error. + RemoteError, + /// The request could not be sent. + SendFailure, + /// The connection closed before the attempt completed. + ConnectionClosed, + /// The endpoint asked the client to stop accepting calls on this connection. + GoAway, + /// The caller cancelled the attempt. + Cancelled, + /// The effective deadline expired. + DeadlineExceeded +} + +/// Contains one completed endpoint-bound client attempt for admission policy reporting. +/// The immutable endpoint candidate selected for the attempt. +/// The generated RPC method descriptor. +/// The terminal attempt classification. +/// The structured RPC error code when available. +/// Whether a valid Response or Error frame matched the pending call. +/// The time from endpoint selection to terminal completion. +public readonly record struct SharpLinkEndpointOutcome( + SharpLinkEndpointCandidate Endpoint, + RpcMethodDescriptor Method, + SharpLinkEndpointOutcomeKind Kind, + SharpLinkErrorCode? ErrorCode, + bool ResponseObserved, + TimeSpan Elapsed); + +/// Controls whether a specific endpoint may receive a client RPC attempt. +/// +/// Implementations run synchronously on the client attempt path. They must not block, perform I/O, +/// retain client connection/session objects, or initiate RPC calls. may be called +/// from an I/O completion path and must not throw; thrown report exceptions are logged and ignored. +/// +public interface ISharpLinkEndpointAdmissionPolicy +{ + /// Attempts to acquire permission to send one RPC attempt to an endpoint. + SharpLinkEndpointAdmissionDecision TryAcquire( + in SharpLinkEndpointCandidate endpoint, + in RpcMethodDescriptor method); + + /// Reports the single terminal outcome for a successfully acquired endpoint attempt. + void Report(in SharpLinkEndpointOutcome outcome, long token); +} diff --git a/src/SharpLink.Abstractions/SharpLinkEndpointResolver.cs b/src/SharpLink.Abstractions/SharpLinkEndpointResolver.cs new file mode 100644 index 0000000..ccd48a5 --- /dev/null +++ b/src/SharpLink.Abstractions/SharpLinkEndpointResolver.cs @@ -0,0 +1,53 @@ +namespace SharpLink.Abstractions; + +/// Represents one versioned endpoint topology supplied by a resolver. +/// +/// The client copies and validates before accepting the snapshot. Versions are +/// compared by the client and must increase strictly for a topology update to take effect. +/// +public sealed class SharpLinkEndpointSnapshot +{ + private readonly SharpLinkEndpoint[] _endpoints; + + /// Initializes a versioned endpoint snapshot. + /// A resolver-assigned, non-negative topology version. + /// The endpoints in this complete topology snapshot. + /// is negative. + /// is null. + public SharpLinkEndpointSnapshot(long version, IReadOnlyList endpoints) + { + if (version < 0) + throw new ArgumentOutOfRangeException(nameof(version)); + ArgumentNullException.ThrowIfNull(endpoints); + + Version = version; + _endpoints = new SharpLinkEndpoint[endpoints.Count]; + for (var index = 0; index < endpoints.Count; index++) + _endpoints[index] = endpoints[index] ?? throw new ArgumentException( + "Endpoint snapshots cannot contain null endpoints.", nameof(endpoints)); + } + + /// Gets the resolver-assigned version for this complete topology. + public long Version { get; } + + /// Gets the endpoint collection supplied by the resolver. + public IReadOnlyList Endpoints => _endpoints; +} + +/// Supplies complete endpoint topology snapshots to a dynamic SharpLink client. +/// +/// Resolvers must not create or retain client transport factories. A client owns the resolver passed to +/// its builder and disposes it exactly once when the client stops or is disposed. +/// +public interface ISharpLinkEndpointResolver : IAsyncDisposable +{ + /// Resolves the initial complete topology. + /// Cancels the resolution attempt. + /// The latest complete topology snapshot. + ValueTask ResolveAsync(CancellationToken cancellationToken); + + /// Watches for complete topology snapshots after the initial resolution. + /// Stops the watch when the owning client stops. + /// An asynchronous sequence of complete topology snapshots. + IAsyncEnumerable WatchAsync(CancellationToken cancellationToken); +} diff --git a/src/SharpLink.Abstractions/SharpLinkEndpoints.cs b/src/SharpLink.Abstractions/SharpLinkEndpoints.cs new file mode 100644 index 0000000..9a39865 --- /dev/null +++ b/src/SharpLink.Abstractions/SharpLinkEndpoints.cs @@ -0,0 +1,242 @@ +namespace SharpLink.Abstractions; + +/// Represents an immutable transport address used by a SharpLink endpoint. +/// +/// Custom address types must be immutable and provide stable value equality. Transport factories +/// interpret custom address types; the SharpLink core does not switch on every possible subtype. +/// +public abstract record SharpLinkTransportAddress; + +/// Represents a TCP host and port. +/// A non-empty host name, IPv4 address, or IPv6 address. +/// A TCP port from 1 through 65535. +public sealed record SharpLinkTcpAddress : SharpLinkTransportAddress +{ + /// Initializes a TCP address. + /// is null, empty, or whitespace. + /// is outside the TCP port range. + public SharpLinkTcpAddress(string host, int port) + { + ArgumentException.ThrowIfNullOrWhiteSpace(host); + if (port is < 1 or > 65535) + throw new ArgumentOutOfRangeException(nameof(port)); + + Host = host; + Port = port; + } + + /// Gets the host name or IP address. + public string Host { get; } + + /// Gets the TCP port. + public int Port { get; } +} + +/// Represents a Unix-domain socket path. +/// A non-empty socket path. +public sealed record SharpLinkUnixDomainSocketAddress : SharpLinkTransportAddress +{ + /// Initializes a Unix-domain socket address. + /// is null, empty, or whitespace. + public SharpLinkUnixDomainSocketAddress(string path) + { + ArgumentException.ThrowIfNullOrWhiteSpace(path); + Path = path; + } + + /// Gets the socket path. + public string Path { get; } +} + +/// Represents a named-pipe server and pipe name. +/// A non-empty pipe name. +/// The pipe server name. The default is the local server. +public sealed record SharpLinkNamedPipeAddress : SharpLinkTransportAddress +{ + /// Initializes a named-pipe address. + /// or is null, empty, or whitespace. + public SharpLinkNamedPipeAddress(string pipeName, string serverName = ".") + { + ArgumentException.ThrowIfNullOrWhiteSpace(pipeName); + ArgumentException.ThrowIfNullOrWhiteSpace(serverName); + PipeName = pipeName; + ServerName = serverName; + } + + /// Gets the pipe name. + public string PipeName { get; } + + /// Gets the pipe server name. + public string ServerName { get; } +} + +/// Represents a same-user, same-machine shared-memory transport name. +/// A non-empty logical shared-memory name. +public sealed record SharpLinkSharedMemoryAddress : SharpLinkTransportAddress +{ + /// Initializes a shared-memory address. + /// is null, empty, or whitespace. + public SharpLinkSharedMemoryAddress(string name) + { + ArgumentException.ThrowIfNullOrWhiteSpace(name); + Name = name; + } + + /// Gets the logical shared-memory name. + public string Name { get; } +} + +/// Represents a one-time anonymous-pipe handle offer. +/// +/// The handle values are intentionally never included in the string representation, diagnostics, or exceptions. +/// A single offer cannot be reused to reconnect or create an automatic multi-endpoint pool. +/// +/// A non-empty inbound anonymous-pipe handle. +/// A non-empty outbound anonymous-pipe handle. +public sealed record SharpLinkAnonymousPipeAddress : SharpLinkTransportAddress +{ + /// Initializes an anonymous-pipe address. + /// or is null, empty, or whitespace. + public SharpLinkAnonymousPipeAddress(string inHandle, string outHandle) + { + ArgumentException.ThrowIfNullOrWhiteSpace(inHandle); + ArgumentException.ThrowIfNullOrWhiteSpace(outHandle); + InHandle = inHandle; + OutHandle = outHandle; + } + + /// Gets the inbound handle. Do not log this value. + public string InHandle { get; } + + /// Gets the outbound handle. Do not log this value. + public string OutHandle { get; } + + /// + public override string ToString() => "SharpLinkAnonymousPipeAddress { Handles = [redacted] }"; +} + +/// Describes one logical endpoint in a static or dynamic SharpLink topology. +/// +/// The client copies and freezes endpoints during Build() or snapshot acceptance. Attributes +/// are intended for selection and diagnostics only and must not determine connection-critical factory settings. +/// +public sealed class SharpLinkEndpoint +{ + /// Gets or initializes the unique logical endpoint identifier. + public required string Id { get; init; } + + /// Gets or initializes the immutable transport address. + public required SharpLinkTransportAddress Address { get; init; } + + /// Gets or initializes the optional logical authority used by the transport, such as TLS SNI. + public string? Authority { get; init; } + + /// Gets or initializes immutable selection and diagnostics attributes. + public IReadOnlyDictionary Attributes { get; init; } = new Dictionary(); +} + +/// Creates a transport factory owned by a SharpLink client for one endpoint generation. +/// The frozen endpoint for which to create a factory. +/// An independently disposable factory for the endpoint generation. +public delegate IClientTransportFactory SharpLinkEndpointTransportFactory(SharpLinkEndpoint endpoint); + +/// Selects an endpoint from one immutable Ready candidate snapshot. +/// +/// Implementations must be synchronous, non-blocking, allocation-free on their normal path, and must not +/// modify topology. Return an index in ; invalid, excluded, or unavailable results +/// fail only the current call with . +/// +public interface ISharpLinkEndpointSelector +{ + /// Selects an index from the current Ready candidate snapshot. + /// The current candidate snapshot and exclusion mask. + /// An index from zero through context.Count - 1. + int Select(in SharpLinkEndpointSelectionContext context); +} + +/// Provides a zero-allocation read-only view of one endpoint selection snapshot. +public readonly ref struct SharpLinkEndpointSelectionContext +{ + private readonly ReadOnlySpan _candidates; + + /// Initializes a selection context over one candidate snapshot. + /// The immutable candidates available to the current call. + /// Bits identifying candidates already excluded by this call. + public SharpLinkEndpointSelectionContext( + ReadOnlySpan candidates, + ulong excludedMask) + { + _candidates = candidates; + ExcludedMask = excludedMask; + } + + /// Gets the candidate count. + public int Count => _candidates.Length; + + /// Gets bits for candidates excluded by the current call. + public ulong ExcludedMask { get; } + + /// Gets a candidate by its snapshot index. + /// An index from zero through minus one. + /// is outside the snapshot. + public SharpLinkEndpointCandidate this[int index] => _candidates[index]; +} + +/// Describes a Ready endpoint visible to one selector invocation. +/// +/// The endpoint identity and generation belong to the immutable candidate snapshot. Connection and +/// active-call counts are read from endpoint-owned atomic state, so a selector never needs a rebuilt +/// candidate array merely because in-flight work changed. +/// +public readonly record struct SharpLinkEndpointCandidate +{ + private readonly int _readyConnectionCount; + private readonly int _activeCallCount; + private readonly Func? _readyConnectionCountProvider; + private readonly Func? _activeCallCountProvider; + + /// Initializes a candidate with fixed diagnostic counts. + /// The frozen endpoint. + /// The Ready connection count. + /// The active-call count. + /// The client-assigned endpoint generation. + public SharpLinkEndpointCandidate( + SharpLinkEndpoint endpoint, + int readyConnectionCount, + int activeCallCount, + long generation) + { + Endpoint = endpoint ?? throw new ArgumentNullException(nameof(endpoint)); + _readyConnectionCount = readyConnectionCount; + _activeCallCount = activeCallCount; + _readyConnectionCountProvider = null; + _activeCallCountProvider = null; + Generation = generation; + } + + internal SharpLinkEndpointCandidate( + SharpLinkEndpoint endpoint, + Func readyConnectionCountProvider, + Func activeCallCountProvider, + long generation) + { + Endpoint = endpoint ?? throw new ArgumentNullException(nameof(endpoint)); + _readyConnectionCount = 0; + _activeCallCount = 0; + _readyConnectionCountProvider = readyConnectionCountProvider ?? throw new ArgumentNullException(nameof(readyConnectionCountProvider)); + _activeCallCountProvider = activeCallCountProvider ?? throw new ArgumentNullException(nameof(activeCallCountProvider)); + Generation = generation; + } + + /// Gets the frozen endpoint identity and attributes. + public SharpLinkEndpoint Endpoint { get; } + + /// Gets the current number of Ready connections for this endpoint. + public int ReadyConnectionCount => _readyConnectionCountProvider?.Invoke() ?? _readyConnectionCount; + + /// Gets the current active-call count for this endpoint. + public int ActiveCallCount => _activeCallCountProvider?.Invoke() ?? _activeCallCount; + + /// Gets the client-assigned endpoint generation. + public long Generation { get; } +} diff --git a/src/SharpLink.Abstractions/SharpLinkRetry.cs b/src/SharpLink.Abstractions/SharpLinkRetry.cs new file mode 100644 index 0000000..3338e06 --- /dev/null +++ b/src/SharpLink.Abstractions/SharpLinkRetry.cs @@ -0,0 +1,32 @@ +namespace SharpLink.Abstractions; + +/// Describes the completed outcome of one retryable unary attempt. +/// The immutable generated RPC method descriptor. +/// The one-based attempt number. +/// The structured error code, or null when the failure was not a SharpLink error. +/// Whether a successful response was observed for this attempt. +/// The elapsed duration of this attempt. +public readonly record struct SharpLinkRetryContext( + RpcMethodDescriptor Method, + int Attempt, + SharpLinkErrorCode? ErrorCode, + bool ResponseObserved, + TimeSpan Elapsed); + +/// Specifies whether a failed idempotent unary attempt should be retried. +/// Whether to schedule another attempt. +/// The non-negative delay before the next attempt. +public readonly record struct SharpLinkRetryDecision(bool ShouldRetry, TimeSpan Delay); + +/// Evaluates retry decisions for explicitly idempotent unary RPC calls. +/// +/// The client invokes this policy only after a failed attempt of a method marked [Idempotent]. +/// Policies are synchronous and must not execute RPC attempts, block, or mutate client topology. +/// +public interface ISharpLinkRetryPolicy +{ + /// Evaluates the outcome of one completed unary attempt. + /// The immutable method metadata and attempt outcome. + /// A decision for the next attempt. + SharpLinkRetryDecision Evaluate(in SharpLinkRetryContext context); +} diff --git a/src/SharpLink.Abstractions/SharpLinkTelemetry.cs b/src/SharpLink.Abstractions/SharpLinkTelemetry.cs index 115a0c1..ddd862f 100644 --- a/src/SharpLink.Abstractions/SharpLinkTelemetry.cs +++ b/src/SharpLink.Abstractions/SharpLinkTelemetry.cs @@ -1,5 +1,6 @@ using System.Diagnostics; using System.Diagnostics.Metrics; +using System.Threading; namespace SharpLink.Abstractions; @@ -13,8 +14,43 @@ public static class SharpLinkTelemetry /// Gets the process-wide immutable meter named SharpLink. public static Meter Meter { get; } = new("SharpLink"); + private static long _clientActiveEndpointCount; + private static long _clientReadyEndpointCount; + private static long _clientDrainingEndpointCount; + private static long _clientActiveConnectionCount; + private static long _clientRetiringConnectionCount; + private static readonly UpDownCounter ActiveConnections = Meter.CreateUpDownCounter("sharplink.connections.active", unit: "{connection}"); + private static readonly ObservableUpDownCounter ClientActiveEndpoints = + Meter.CreateObservableUpDownCounter( + "sharplink.client.endpoints.active", + static () => Volatile.Read(ref _clientActiveEndpointCount), + unit: "{endpoint}"); + private static readonly ObservableUpDownCounter ClientReadyEndpoints = + Meter.CreateObservableUpDownCounter( + "sharplink.client.endpoints.ready", + static () => Volatile.Read(ref _clientReadyEndpointCount), + unit: "{endpoint}"); + private static readonly ObservableUpDownCounter ClientDrainingEndpoints = + Meter.CreateObservableUpDownCounter( + "sharplink.client.endpoints.draining", + static () => Volatile.Read(ref _clientDrainingEndpointCount), + unit: "{endpoint}"); + private static readonly Counter ClientResolverUpdates = + Meter.CreateCounter("sharplink.client.resolver.updates", unit: "{update}"); + private static readonly Counter ClientResolverFailures = + Meter.CreateCounter("sharplink.client.resolver.failures", unit: "{failure}"); + private static readonly ObservableUpDownCounter ClientActiveConnections = + Meter.CreateObservableUpDownCounter( + "sharplink.client.connections.active", + static () => Volatile.Read(ref _clientActiveConnectionCount), + unit: "{connection}"); + private static readonly ObservableUpDownCounter ClientRetiringConnections = + Meter.CreateObservableUpDownCounter( + "sharplink.client.connections.retiring", + static () => Volatile.Read(ref _clientRetiringConnectionCount), + unit: "{connection}"); private static readonly Counter Reconnects = Meter.CreateCounter("sharplink.connections.reconnects", unit: "{attempt}"); private static readonly Counter StartedCalls = @@ -83,18 +119,89 @@ public static class SharpLinkTelemetry Meter.CreateCounter("sharplink.admission.oneway.dropped", unit: "{call}"); private static readonly UpDownCounter AdmissionActivePartitions = Meter.CreateUpDownCounter("sharplink.admission.partitions.active", unit: "{partition}"); + private static readonly Counter ClientAttempts = + Meter.CreateCounter("sharplink.client.attempts", unit: "{attempt}"); + private static readonly Counter ClientRetries = + Meter.CreateCounter("sharplink.client.retries", unit: "{retry}"); + private static readonly Counter EndpointAdmissionRejected = + Meter.CreateCounter("sharplink.client.endpoint_admission.rejected", unit: "{attempt}"); + private static readonly Counter SelectionFailures = + Meter.CreateCounter("sharplink.client.selection.failures", unit: "{failure}"); + private static readonly Counter BreakerOpen = + Meter.CreateCounter("sharplink.client.breaker.open", unit: "{rejection}"); internal static CallScope StartClientCall(RpcMethodDescriptor method) => StartCall(ClientActivitySource, ActivityKind.Client, "client", method, requestId: 0); + /// Starts telemetry for one physical attempt within a logical client call. + /// + /// Attempt activities deliberately do not affect the logical call counters. This lets retry + /// diagnostics show every network attempt while the normal call metrics remain one-per-call. + /// + internal static AttemptScope StartClientAttempt(RpcMethodDescriptor method, int attempt) + { + if (!ClientActivitySource.HasListeners()) + return default; + + var activity = ClientActivitySource.StartActivity("sharplink.rpc.attempt", ActivityKind.Client); + if (activity is null) + return default; + activity.SetTag("rpc.system", "sharplink"); + activity.SetTag("rpc.sharplink.contract_id", method.ContractId); + activity.SetTag("rpc.sharplink.method_id", method.MethodId); + activity.SetTag("rpc.sharplink.method_kind", method.Kind.ToString()); + activity.SetTag("rpc.sharplink.attempt", attempt); + return new AttemptScope(activity); + } + internal static CallScope StartServerCall(RpcMethodDescriptor method, long requestId) => StartCall(ServerActivitySource, ActivityKind.Server, "server", method, requestId); internal static bool ClientCallsEnabled => ClientActivitySource.HasListeners() || CallMetricsEnabled; - internal static void ConnectionOpened(string side) => RecordDelta(ActiveConnections, 1, side); - internal static void ConnectionClosed(string side) => RecordDelta(ActiveConnections, -1, side); + internal static void ConnectionOpened(string side) + { + RecordDelta(ActiveConnections, 1, side); + if (side == "client") + Interlocked.Increment(ref _clientActiveConnectionCount); + } + internal static void ConnectionClosed(string side) + { + RecordDelta(ActiveConnections, -1, side); + if (side == "client") + Interlocked.Decrement(ref _clientActiveConnectionCount); + } + internal static void AddClientActiveEndpoints(long count) + { + if (count != 0) + Interlocked.Add(ref _clientActiveEndpointCount, count); + } + internal static void AddClientReadyEndpoints(long count) + { + if (count != 0) + Interlocked.Add(ref _clientReadyEndpointCount, count); + } + internal static void AddClientDrainingEndpoints(long count) + { + if (count != 0) + Interlocked.Add(ref _clientDrainingEndpointCount, count); + } + internal static void RecordClientResolverUpdate() + { + if (ClientResolverUpdates.Enabled) + ClientResolverUpdates.Add(1); + } + internal static void RecordClientResolverFailure() + { + if (ClientResolverFailures.Enabled) + ClientResolverFailures.Add(1); + } + internal static void AddClientRetiringConnections(long count) + { + if (count != 0) + Interlocked.Add(ref _clientRetiringConnectionCount, count); + } internal static void ReconnectAttempt() => Record(Reconnects, 1, "client"); internal static void RecordSentBytes(long bytes) => RecordPositive(SentBytes, bytes); internal static void RecordReceivedBytes(long bytes) => RecordPositive(ReceivedBytes, bytes); @@ -147,6 +254,31 @@ internal static void RecordAdmissionOneWayDropped(string scope, string reason) new KeyValuePair("sharplink.admission.scope", scope), new KeyValuePair("sharplink.admission.reason", reason)); } + internal static void RecordClientAttempt() + { + if (ClientAttempts.Enabled) + ClientAttempts.Add(1); + } + internal static void RecordClientRetry() + { + if (ClientRetries.Enabled) + ClientRetries.Add(1); + } + internal static void RecordEndpointAdmissionRejected(string reason) + { + if (EndpointAdmissionRejected.Enabled) + EndpointAdmissionRejected.Add(1, new KeyValuePair("sharplink.admission.reason", reason)); + } + internal static void RecordSelectionFailure(string reason) + { + if (SelectionFailures.Enabled) + SelectionFailures.Add(1, new KeyValuePair("sharplink.selection.reason", reason)); + } + internal static void RecordBreakerOpen() + { + if (BreakerOpen.Enabled) + BreakerOpen.Add(1); + } internal static void RecordSharedMemoryConnection(string side, int capacity) { if (!SharedMemoryConnections.Enabled) @@ -359,4 +491,38 @@ internal void Complete(Exception? exception = null) _activity?.Dispose(); } } + + internal struct AttemptScope + { + private readonly Activity? _activity; + private bool _completed; + + internal AttemptScope(Activity activity) + { + _activity = activity; + } + + internal void Complete(Exception? exception = null) + { + if (_completed) + return; + _completed = true; + if (exception is null) + { + _activity?.SetStatus(ActivityStatusCode.Ok); + } + else + { + var status = exception switch + { + SharpLinkException sharpLinkException => sharpLinkException.Code.ToString(), + OperationCanceledException => SharpLinkErrorCode.Cancelled.ToString(), + _ => SharpLinkErrorCode.Internal.ToString() + }; + _activity?.SetStatus(ActivityStatusCode.Error, status); + _activity?.SetTag("error.type", exception.GetType().FullName); + } + _activity?.Dispose(); + } + } } diff --git a/src/SharpLink.Client/ClientConnection.cs b/src/SharpLink.Client/ClientConnection.cs index 96629b0..df9e36e 100644 --- a/src/SharpLink.Client/ClientConnection.cs +++ b/src/SharpLink.Client/ClientConnection.cs @@ -28,19 +28,29 @@ public ClientConnection( RpcSession session, CancellationTokenSource cancellation, int maxPendingCalls, - IRpcCodecProvider codecs) + IRpcCodecProvider codecs, + string? endpointId = null, + long endpointGeneration = 0) { _client = client ?? throw new ArgumentNullException(nameof(client)); Session = session ?? throw new ArgumentNullException(nameof(session)); _cancellation = cancellation ?? throw new ArgumentNullException(nameof(cancellation)); _consumerAbandonedCallback = OnConsumerAbandoned; PendingCalls = new PendingRequestTable(maxPendingCalls, codecs, this); + EndpointId = endpointId; + EndpointGeneration = endpointGeneration; } public RpcSession Session { get; } public PendingRequestTable PendingCalls { get; } + /// Gets the owning endpoint identity when this connection belongs to a cluster. + public string? EndpointId { get; } + + /// Gets the owning endpoint generation when this connection belongs to a dynamic cluster. + public long EndpointGeneration { get; } + public ClientConnectionState State => (ClientConnectionState)Volatile.Read(ref _state); @@ -56,7 +66,7 @@ public bool CanAcceptCalls internal bool ShouldLogLateResponse(out int suppressedCount) => _lateResponseLogLimiter.ShouldLog(Stopwatch.GetTimestamp(), out suppressedCount); - public void MarkDraining() + public bool MarkDraining() { if (Interlocked.CompareExchange( ref _state, @@ -64,17 +74,23 @@ public void MarkDraining() (int)ClientConnectionState.Ready) == (int)ClientConnectionState.Ready) { Session.MarkDraining(); + SharpLinkTelemetry.AddClientRetiringConnections(1); + return true; } + + return false; } public void Fail(Exception exception) { ArgumentNullException.ThrowIfNull(exception); - if (Interlocked.Exchange(ref _state, (int)ClientConnectionState.Closed) == - (int)ClientConnectionState.Closed) + var previousState = Interlocked.Exchange(ref _state, (int)ClientConnectionState.Closed); + if (previousState == (int)ClientConnectionState.Closed) { return; } + if (previousState == (int)ClientConnectionState.Draining) + SharpLinkTelemetry.AddClientRetiringConnections(-1); try { diff --git a/src/SharpLink.Client/PendingRequestTable.cs b/src/SharpLink.Client/PendingRequestTable.cs index 5bdb59d..003d4d4 100644 --- a/src/SharpLink.Client/PendingRequestTable.cs +++ b/src/SharpLink.Client/PendingRequestTable.cs @@ -37,6 +37,17 @@ internal interface IPendingCallOwner void OnPendingCallCompleted(in PendingCallCompletion completion); } +/// +/// Receives the single terminal outcome selected by the pending-call completion race. +/// This stays attached to the existing pending entry, rather than creating a second +/// request lookup for retry or endpoint-admission bookkeeping. +/// +internal interface IPendingCallCompletionObserver +{ + void OnResponseObserved(); + void OnPendingCallCompleted(in PendingCallCompletion completion); +} + /// /// Stores all pending calls for one physical connection in a bounded power-of-two table. /// @@ -110,11 +121,14 @@ public RpcRequestOperation Rent( PendingCallKind kind, long deadlineTimestamp, CancellationToken cancellationToken, - out long id) + out long id, + IPendingCallCompletionObserver? completionObserver = null) { ArgumentNullException.ThrowIfNull(responseCodec); ObjectDisposedException.ThrowIf(Volatile.Read(ref _disposed) != 0, this); - if (TryRent(responseCodec, kind, deadlineTimestamp, cancellationToken, out id, out var operation)) + if (TryRent( + responseCodec, kind, deadlineTimestamp, cancellationToken, + completionObserver, out id, out var operation)) return operation; throw CreateResourceExhaustedException(); @@ -151,11 +165,14 @@ public async ValueTask> RentAsync( long deadlineTimestamp, bool waitForSlot, DateTimeOffset? deadline, - CancellationToken cancellationToken) + CancellationToken cancellationToken, + IPendingCallCompletionObserver? completionObserver = null) { ArgumentNullException.ThrowIfNull(responseCodec); ObjectDisposedException.ThrowIf(Volatile.Read(ref _disposed) != 0, this); - if (TryRent(responseCodec, kind, deadlineTimestamp, cancellationToken, out var id, out var operation)) + if (TryRent( + responseCodec, kind, deadlineTimestamp, cancellationToken, + completionObserver, out var id, out var operation)) return new PendingRequestLease(id, operation); if (!waitForSlot) throw CreateResourceExhaustedException(); @@ -168,7 +185,9 @@ public async ValueTask> RentAsync( Interlocked.Increment(ref _waiterCount); try { - if (TryRent(responseCodec, kind, deadlineTimestamp, cancellationToken, out id, out operation)) + if (TryRent( + responseCodec, kind, deadlineTimestamp, cancellationToken, + completionObserver, out id, out operation)) return new PendingRequestLease(id, operation); if (deadline is not { } absoluteDeadline) @@ -195,7 +214,9 @@ public async ValueTask> RentAsync( Interlocked.Decrement(ref _waiterCount); } - if (TryRent(responseCodec, kind, deadlineTimestamp, cancellationToken, out id, out operation)) + if (TryRent( + responseCodec, kind, deadlineTimestamp, cancellationToken, + completionObserver, out id, out operation)) return new PendingRequestLease(id, operation); } } @@ -204,7 +225,8 @@ public long RegisterStream( PendingCallKind kind, IStreamDispatcher dispatcher, long deadlineTimestamp, - CancellationToken cancellationToken) + CancellationToken cancellationToken, + IPendingCallCompletionObserver? completionObserver = null) { ArgumentNullException.ThrowIfNull(dispatcher); if (kind is not (PendingCallKind.ServerStreaming or PendingCallKind.DuplexStreaming)) @@ -216,7 +238,8 @@ public long RegisterStream( dispatcher, deadlineTimestamp, cancellationToken, - out var id)) + out var id, + completionObserver)) { return id; } @@ -226,7 +249,8 @@ public long RegisterStream( public PendingRequestLease RegisterOneWayClientStream( long deadlineTimestamp, - CancellationToken cancellationToken) + CancellationToken cancellationToken, + IPendingCallCompletionObserver? completionObserver = null) { var operation = RpcOperationPool.Rent(); if (TryRegister( @@ -236,7 +260,8 @@ public PendingRequestLease RegisterOneWayClientStream( deadlineTimestamp, cancellationToken, out var id, - RpcEmptyRequestCodec.Instance)) + RpcEmptyRequestCodec.Instance, + completionObserver)) { return new PendingRequestLease(id, operation); } @@ -247,13 +272,27 @@ public PendingRequestLease RegisterOneWayClientStream( public bool Dispatch(long id, ref ReadOnlySequence payload) { - var current = Volatile.Read(ref _slots[(int)(id & _indexMask)]); + var index = (int)(id & _indexMask); + var current = Volatile.Read(ref _slots[index]); if (current is not null && current.Id == id && current.Kind is PendingCallKind.ServerStreaming or PendingCallKind.DuplexStreaming) { // A successful Response is only the server's acknowledgement; StreamComplete owns - // the terminal transition for server and duplex streams. - return true; + // the terminal transition for server and duplex streams. The callback shares the + // per-call completion gate with terminal removal, so a matching acknowledgement is + // observed before cancellation, deadline, or disconnect can report the terminal result. + lock (current.CompletionGate) + { + if (!ReferenceEquals(Volatile.Read(ref _slots[index]), current) || + current.Id != id || + current.Kind is not (PendingCallKind.ServerStreaming or PendingCallKind.DuplexStreaming)) + { + return false; + } + + current.CompletionObserver?.OnResponseObserved(); + return true; + } } if (!TryTakeMatchingCall(id, out var call)) @@ -304,15 +343,12 @@ public void FailAllPendingRequests(Exception exception) ArgumentNullException.ThrowIfNull(exception); for (var index = 0; index < _slots.Length; index++) { - var call = Interlocked.Exchange(ref _slots[index], null); - if (call is null) + if (!TryTakeCallAtIndex(index, out var call)) continue; - call.WaitUntilRegistered(); - ReleaseSlot(); var payload = ReadOnlySequence.Empty; CompleteTakenCall( - call, + call!, PendingCallCompletionReason.ConnectionClosed, exception, ref payload); @@ -337,6 +373,7 @@ private bool TryRent( PendingCallKind kind, long deadlineTimestamp, CancellationToken cancellationToken, + IPendingCallCompletionObserver? completionObserver, out long id, out RpcRequestOperation operation) { @@ -348,7 +385,8 @@ private bool TryRent( deadlineTimestamp, cancellationToken, out id, - responseCodec)) + responseCodec, + completionObserver)) { return true; } @@ -365,7 +403,8 @@ private bool TryRegister( long deadlineTimestamp, CancellationToken cancellationToken, out long id, - IRpcCodec responseCodec) + IRpcCodec responseCodec, + IPendingCallCompletionObserver? completionObserver) { for (var attempt = 0; attempt < _slots.Length; attempt++) { @@ -378,7 +417,8 @@ private bool TryRegister( operation, dispatcher, deadlineTimestamp, - cancellationToken); + cancellationToken, + completionObserver); var index = (int)(id & _indexMask); if (Interlocked.CompareExchange(ref _slots[index], call, null) is null) { @@ -399,7 +439,8 @@ private bool TryRegister( IStreamDispatcher? dispatcher, long deadlineTimestamp, CancellationToken cancellationToken, - out long id) + out long id, + IPendingCallCompletionObserver? completionObserver = null) { for (var attempt = 0; attempt < _slots.Length; attempt++) { @@ -411,7 +452,8 @@ private bool TryRegister( operation, dispatcher, deadlineTimestamp, - cancellationToken); + cancellationToken, + completionObserver); var index = (int)(id & _indexMask); if (Interlocked.CompareExchange(ref _slots[index], call, null) is null) { @@ -449,14 +491,47 @@ private bool TryTakeMatchingCall(long id, out PendingCall? call) return false; } - var exchanged = Interlocked.CompareExchange(ref _slots[index], null, current); - if (!ReferenceEquals(exchanged, current)) - continue; + lock (current.CompletionGate) + { + if (!ReferenceEquals(Volatile.Read(ref _slots[index]), current) || current.Id != id) + continue; - current.WaitUntilRegistered(); - call = current; - ReleaseSlot(); - return true; + var exchanged = Interlocked.CompareExchange(ref _slots[index], null, current); + if (!ReferenceEquals(exchanged, current)) + continue; + + current.WaitUntilRegistered(); + call = current; + ReleaseSlot(); + return true; + } + } + } + + private bool TryTakeCallAtIndex(int index, out PendingCall? call) + { + while (true) + { + var current = Volatile.Read(ref _slots[index]); + if (current is null) + { + call = null; + return false; + } + + lock (current.CompletionGate) + { + if (!ReferenceEquals(Volatile.Read(ref _slots[index]), current)) + continue; + + if (!ReferenceEquals(Interlocked.CompareExchange(ref _slots[index], null, current), current)) + continue; + + current.WaitUntilRegistered(); + call = current; + ReleaseSlot(); + return true; + } } } @@ -468,24 +543,35 @@ private void CompleteTakenCall( { call.DisposeCancellationRegistration(); call.CancelProducer(reason); + var isResponse = reason is PendingCallCompletionReason.Response or PendingCallCompletionReason.LocalStreamComplete; + if (isResponse && call.Operation is { } responseOperation) + { + exception = responseOperation.TryDeserializeResponse(ref payload); + if (exception is not null) + reason = PendingCallCompletionReason.RemoteError; + } exception ??= CreateCompletionException(call, reason); + var completion = new PendingCallCompletion( + call.Id, + call.Kind, + reason, + call.Dispatcher, + exception); + // Decode response payloads before reporting the terminal admission outcome so malformed + // endpoint responses are not published as successful attempts. + call.CompletionObserver?.OnPendingCallCompleted(in completion); + if (call.Operation is { } operation) { - if (reason is PendingCallCompletionReason.Response or PendingCallCompletionReason.LocalStreamComplete) - operation.SetResult(ref payload); + if (isResponse) + operation.CompleteResponse(exception); else operation.SetError(exception ?? new SharpLinkException( SharpLinkErrorCode.Internal, "A pending request completed without a result.")); } - var completion = new PendingCallCompletion( - call.Id, - call.Kind, - reason, - call.Dispatcher, - exception); _owner?.OnPendingCallCompleted(in completion); call.ReturnCompleted(); } @@ -647,8 +733,11 @@ private sealed class PendingCall private PendingRequestTable? _table; private CancellationTokenRegistration _cancellationRegistration; private CancellationTokenSource? _producerCancellation; + private IPendingCallCompletionObserver? _completionObserver; private int _registered; + public object CompletionGate { get; } = new(); + public long Id { get; private set; } public PendingCallKind Kind { get; private set; } public IRpcOperation? Operation { get; private set; } @@ -657,6 +746,7 @@ private sealed class PendingCall public CancellationToken CancellationToken { get; private set; } public CancellationToken ProducerCancellationToken => _producerCancellation?.Token ?? CancellationToken.None; + public IPendingCallCompletionObserver? CompletionObserver => _completionObserver; public static PendingCall Rent( PendingRequestTable table, @@ -665,7 +755,8 @@ public static PendingCall Rent( IRpcOperation? operation, IStreamDispatcher? dispatcher, long deadlineTimestamp, - CancellationToken cancellationToken) + CancellationToken cancellationToken, + IPendingCallCompletionObserver? completionObserver) { if (!Pool.TryDequeue(out var call)) call = new PendingCall(); @@ -680,6 +771,7 @@ public static PendingCall Rent( call.Dispatcher = dispatcher; call.DeadlineTimestamp = deadlineTimestamp; call.CancellationToken = cancellationToken; + call._completionObserver = completionObserver; call._producerCancellation = kind is PendingCallKind.ClientStreaming or PendingCallKind.DuplexStreaming or @@ -740,6 +832,7 @@ private void ReturnCore() DeadlineTimestamp = 0; CancellationToken = CancellationToken.None; _producerCancellation = null; + _completionObserver = null; _cancellationRegistration = default; Volatile.Write(ref _registered, 0); diff --git a/src/SharpLink.Client/RpcRequestOperation.cs b/src/SharpLink.Client/RpcRequestOperation.cs index 41ba2dc..6c19266 100644 --- a/src/SharpLink.Client/RpcRequestOperation.cs +++ b/src/SharpLink.Client/RpcRequestOperation.cs @@ -7,7 +7,8 @@ internal interface IRpcOperation { // 在 IO 线程被调用 public long Id { get; } - void SetResult(ref ReadOnlySequence payload); + Exception? TryDeserializeResponse(ref ReadOnlySequence payload); + void CompleteResponse(Exception? exception); void SetError(Exception ex); } @@ -16,6 +17,7 @@ internal sealed class RpcRequestOperation : IValueTaskSource, IRpcOperatio { private ManualResetValueTaskSourceCore _core; private IRpcCodec? _codec; + private T? _response; private readonly Action> _returnAction; @@ -56,26 +58,35 @@ public T GetResult(short token) public void OnCompleted(Action continuation, object? state, short token, ValueTaskSourceOnCompletedFlags flags) => _core.OnCompleted(continuation, state, token, flags); - public void SetResult(ref ReadOnlySequence payload) + public Exception? TryDeserializeResponse(ref ReadOnlySequence payload) { try { if (payload.Length == 0) { - _core.SetResult(default!); - return; + _response = default; + return null; } // 【IO线程反序列化】 // 此时 payload 有效，直接转为 T 对象，不拷贝 bytes - var result = (_codec ?? throw new InvalidOperationException("Request operation has no response codec.")) + _response = (_codec ?? throw new InvalidOperationException("Request operation has no response codec.")) .Deserialize(payload); - _core.SetResult(result!); + return null; } catch (Exception ex) { - _core.SetException(ex); + return ex; } } + + public void CompleteResponse(Exception? exception) + { + if (exception is null) + _core.SetResult(_response!); + else + _core.SetException(exception); + } + public void SetError(Exception ex) { _core.SetException(ex); @@ -85,6 +96,7 @@ public void SetError(Exception ex) private void ReturnToPool() { _codec = null; + _response = default; // Clear the continuation and its state before this operation enters the // process-wide generic pool. A continuation can close over request types // from a collectible AssemblyLoadContext even when T itself is static. diff --git a/src/SharpLink.Client/SharpClientBuilder.cs b/src/SharpLink.Client/SharpClientBuilder.cs index 42a5022..e6fad22 100644 --- a/src/SharpLink.Client/SharpClientBuilder.cs +++ b/src/SharpLink.Client/SharpClientBuilder.cs @@ -6,6 +6,10 @@ public class SharpClientBuilder private IClientTransportFactory? _transport; + private IEnumerable? _endpoints; + private SharpLinkEndpointTransportFactory? _endpointTransportFactory; + private ISharpLinkEndpointResolver? _endpointResolver; + private SharpLinkEndpointTransportFactory? _resolverTransportFactory; private ILoggerFactory? _loggerFactory; private ISharpLinkClientAuthenticator? _authenticator; private readonly List _interceptors = []; @@ -39,6 +43,17 @@ public SharpClientBuilder AddInterceptor(ISharpLinkClientInterceptor interceptor private RpcSessionFlushOptions? _rpcSessionFlushOptions; private readonly SharpLinkConnectionPoolOptions _connectionPool = new(); private bool _connectionPoolConfigured; + private readonly SharpLinkClusterOptions _cluster = new(); + private bool _clusterConfigured; + private SharpLinkLoadBalancingStrategy _loadBalancingStrategy = SharpLinkLoadBalancingStrategy.PowerOfTwoChoices; + private bool _loadBalancingConfigured; + private ISharpLinkEndpointSelector? _endpointSelector; + private readonly SharpLinkRetryOptions _retry = new(); + private bool _retryConfigured; + private ISharpLinkRetryPolicy? _retryPolicy; + private ISharpLinkEndpointAdmissionPolicy? _endpointAdmissionPolicy; + private readonly SharpLinkCircuitBreakerOptions _circuitBreaker = new(); + private bool _circuitBreakerConfigured; /// Configures instance-scoped runtime behavior. public SharpClientBuilder UseRuntime(Action configure) @@ -159,25 +174,278 @@ public SharpClientBuilder UseConnectionPool(ActionUses one static endpoint and an endpoint-specific transport factory. + /// The endpoint copied and frozen during . + /// Creates the client-owned transport factory for the frozen endpoint. + public SharpClientBuilder UseEndpoint( + SharpLinkEndpoint endpoint, + SharpLinkEndpointTransportFactory transportFactory) + { + ArgumentNullException.ThrowIfNull(endpoint); + ArgumentNullException.ThrowIfNull(transportFactory); + _endpoints = [endpoint]; + _endpointTransportFactory = transportFactory; + return this; + } + + /// Uses a static endpoint collection and an endpoint-specific transport factory. + /// Endpoints enumerated once and frozen during . + /// Creates one client-owned transport factory per frozen endpoint. + public SharpClientBuilder UseEndpoints( + IEnumerable endpoints, + SharpLinkEndpointTransportFactory transportFactory) + { + _endpoints = endpoints ?? throw new ArgumentNullException(nameof(endpoints)); + _endpointTransportFactory = transportFactory ?? throw new ArgumentNullException(nameof(transportFactory)); + return this; + } + + /// Uses a client-owned resolver to maintain a dynamic endpoint topology. + /// The resolver disposed by the built client. + /// Creates one client-owned transport factory for each endpoint generation. + /// + /// This mode is mutually exclusive with and . + /// The resolver supplies complete snapshots; its initial resolution and watch execute only after + /// is called. + /// + public SharpClientBuilder UseEndpointResolver( + ISharpLinkEndpointResolver resolver, + SharpLinkEndpointTransportFactory transportFactory) + { + _endpointResolver = resolver ?? throw new ArgumentNullException(nameof(resolver)); + _resolverTransportFactory = transportFactory ?? throw new ArgumentNullException(nameof(transportFactory)); + return this; + } + + /// Uses the built-in DNS resolver for a dynamic TCP endpoint topology. + /// The DNS host name used as the default endpoint authority. + /// The TCP port from 1 through 65535. + /// Creates a client-owned transport factory for every discovered endpoint generation. + /// Optionally configures refresh and address-family behavior. + /// This builder. + public SharpClientBuilder UseDnsEndpoints( + string host, + int port, + SharpLinkEndpointTransportFactory transportFactory, + Action? configure = null) + { + ArgumentException.ThrowIfNullOrWhiteSpace(host); + if (port is < 1 or > 65535) + throw new ArgumentOutOfRangeException(nameof(port)); + ArgumentNullException.ThrowIfNull(transportFactory); + + var options = new SharpLinkDnsResolverOptions(); + configure?.Invoke(options); + return UseEndpointResolver( + new SharpLinkDnsEndpointResolver(host, port, options), + transportFactory); + } + + /// Configures the bounded resources used only by a multi-endpoint static cluster. + /// Mutates builder-owned options frozen by . + public SharpClientBuilder UseCluster(Action configure) + { + ArgumentNullException.ThrowIfNull(configure); + configure(_cluster); + _clusterConfigured = true; + return this; + } + + /// Selects a built-in static endpoint load-balancing strategy. + /// The strategy used only by a multi-endpoint static cluster. + /// A custom selector has already been configured. + public SharpClientBuilder UseLoadBalancing(SharpLinkLoadBalancingStrategy strategy) + { + if (_endpointSelector is not null) + throw new InvalidOperationException("A custom endpoint selector is already configured."); + if (!Enum.IsDefined(strategy)) + throw new ArgumentOutOfRangeException(nameof(strategy)); + _loadBalancingStrategy = strategy; + _loadBalancingConfigured = true; + return this; + } + + /// Uses a custom static endpoint selector. + /// A synchronous selector that returns a current candidate index. + /// A built-in strategy was explicitly configured. + public SharpClientBuilder UseEndpointSelector(ISharpLinkEndpointSelector selector) + { + ArgumentNullException.ThrowIfNull(selector); + if (_loadBalancingConfigured) + throw new InvalidOperationException("A built-in endpoint load-balancing strategy is already configured."); + _endpointSelector = selector; + return this; + } + + /// Enables the built-in retry policy for explicitly idempotent unary calls. + /// Retry is disabled by default. Streaming, one-way, and non-idempotent unary calls never retry. + public SharpClientBuilder UseRetry() + { + _retryConfigured = true; + _retryPolicy = null; + return this; + } + + /// Enables and configures the built-in retry policy for explicitly idempotent unary calls. + /// Mutates builder-owned options frozen during . + public SharpClientBuilder UseRetry(Action configure) + { + ArgumentNullException.ThrowIfNull(configure); + configure(_retry); + _retryConfigured = true; + _retryPolicy = null; + return this; + } + + /// Enables a custom retry policy for explicitly idempotent unary calls. + /// A synchronous policy that returns only a decision and delay. + public SharpClientBuilder UseRetry(ISharpLinkRetryPolicy policy) + { + _retryPolicy = policy ?? throw new ArgumentNullException(nameof(policy)); + _retryConfigured = true; + return this; + } + + /// Uses a synchronous custom endpoint admission policy for cluster attempts. + /// + /// Endpoint admission and the built-in circuit breaker are alternative policies. Neither affects + /// fixed mode because it has no endpoint topology. + /// + public SharpClientBuilder UseEndpointAdmission(ISharpLinkEndpointAdmissionPolicy policy) + { + ArgumentNullException.ThrowIfNull(policy); + if (_circuitBreakerConfigured) + throw new InvalidOperationException("UseEndpointAdmission and UseCircuitBreaker are mutually exclusive."); + _endpointAdmissionPolicy = policy; + return this; + } + + /// Enables the built-in endpoint-generation circuit breaker for cluster attempts. + public SharpClientBuilder UseCircuitBreaker(Action configure) + { + ArgumentNullException.ThrowIfNull(configure); + if (_endpointAdmissionPolicy is not null) + throw new InvalidOperationException("UseEndpointAdmission and UseCircuitBreaker are mutually exclusive."); + configure(_circuitBreaker); + _circuitBreakerConfigured = true; + return this; + } public ISharpLinkClient Build() { - if (_transport == null) - throw new InvalidOperationException("Transport must be set before building the client."); + var modeCount = (_transport is null ? 0 : 1) + (_endpoints is null ? 0 : 1) + (_endpointResolver is null ? 0 : 1); + if (modeCount > 1) + throw new InvalidOperationException("UseTransport, UseEndpoint(s), and UseEndpointResolver are mutually exclusive."); + if (modeCount == 0) + throw new InvalidOperationException("Transport, endpoint(s), or an endpoint resolver must be set before building the client."); var runtimeContext = _runtimeContextBuilder.Build(); - if (_transport is IPerformanceProfileAwareTransport profileAwareTransport) - profileAwareTransport.BindPerformanceProfile(runtimeContext.Options.PerformanceProfile); var protocolOptions = runtimeContext.Protocol; - var connectionPool = CreateConnectionPoolSnapshot(runtimeContext); - if (_transport is AnonymousPipeClientTransportFactory && connectionPool.MaxConnections != 1) + if (_endpointResolver is not null) + { + if (_connectionPoolConfigured) + throw new InvalidOperationException("UseConnectionPool is only available for a fixed single endpoint."); + var cluster = _cluster.CloneValidatedForDynamicResolver(); + return CreateDynamicClusterClient( + _endpointResolver, + _resolverTransportFactory!, + cluster, + runtimeContext, + protocolOptions); + } + + if (_endpoints is not null) { - throw new InvalidOperationException( - "Anonymous-pipe handle offers support exactly one client connection."); + var endpoints = CreateEndpointSnapshot(_endpoints, allowEmpty: false); + if (endpoints.Length == 1) + { + if (_clusterConfigured) + throw new InvalidOperationException("UseCluster requires two or more endpoints."); + var transport = CreateTransportFactory(endpoints[0], _endpointTransportFactory!, runtimeContext); + try + { + var singleEndpointPool = CreateConnectionPoolSnapshot(runtimeContext); + if (transport is AnonymousPipeClientTransportFactory && singleEndpointPool.MaxConnections != 1) + { + throw new InvalidOperationException( + "Anonymous-pipe handle offers support exactly one client connection."); + } + return CreateFixedClient( + transport, + runtimeContext, + protocolOptions, + singleEndpointPool, + fixedEndpoint: endpoints[0]); + } + catch + { + try { transport.DisposeAsync().AsTask().GetAwaiter().GetResult(); } + catch { } + throw; + } + } + + if (_connectionPoolConfigured) + throw new InvalidOperationException("UseConnectionPool is only available for a fixed single endpoint."); + var cluster = _cluster.CloneValidated(endpoints.Length); + var configurations = new StaticEndpointConfiguration[endpoints.Length]; + var ownedFactories = new HashSet(ReferenceEqualityComparer.Instance); + try + { + for (var index = 0; index < endpoints.Length; index++) + { + var factory = CreateTransportFactory(endpoints[index], _endpointTransportFactory!, runtimeContext); + if (!ownedFactories.Add(factory)) + { + throw new InvalidOperationException( + "Each static endpoint must receive an independently owned transport factory."); + } + if (factory is AnonymousPipeClientTransportFactory) + { + throw new InvalidOperationException( + "Anonymous-pipe handle offers cannot be used by endpoint clusters."); + } + configurations[index] = new StaticEndpointConfiguration( + endpoints[index], + factory); + } + return CreateClusterClient(configurations, cluster, runtimeContext, protocolOptions); + } + catch (Exception buildException) + { + List? cleanupFailures = null; + foreach (var factory in ownedFactories) + { + try { factory.DisposeAsync().AsTask().GetAwaiter().GetResult(); } + catch (Exception exception) { (cleanupFailures ??= []).Add(exception); } + } + if (cleanupFailures is null) + throw; + cleanupFailures.Insert(0, buildException); + throw new AggregateException(cleanupFailures); + } } + var fixedTransport = _transport!; + if (fixedTransport is IPerformanceProfileAwareTransport profileAwareTransport) + profileAwareTransport.BindPerformanceProfile(runtimeContext.Options.PerformanceProfile); + var connectionPool = CreateConnectionPoolSnapshot(runtimeContext); + if (fixedTransport is AnonymousPipeClientTransportFactory && connectionPool.MaxConnections != 1) + throw new InvalidOperationException("Anonymous-pipe handle offers support exactly one client connection."); + + return CreateFixedClient(fixedTransport, runtimeContext, protocolOptions, connectionPool); + } + + private ISharpLinkClient CreateFixedClient( + IClientTransportFactory transport, + SharpLinkRuntimeContext runtimeContext, + SharpLinkProtocolOptions protocolOptions, + SharpLinkConnectionPoolOptions? connectionPool = null, + SharpLinkEndpoint? fixedEndpoint = null) + { return new SharpLinkClient( - _transport, + transport, _heartbeatInterval, _heartbeatTimeout, _loggerFactory ?? NullLoggerFactory.Instance, @@ -186,11 +454,140 @@ public ISharpLinkClient Build() protocolOptions, runtimeContext, _rpcSessionFlushOptions, - connectionPool, - _interceptors.ToArray() + connectionPool ?? CreateConnectionPoolSnapshot(runtimeContext), + _interceptors.ToArray(), + fixedEndpoint: fixedEndpoint, + retryOptions: CreateRetryOptions(), + retryPolicy: _retryPolicy, + endpointAdmissionPolicy: CreateEndpointAdmissionPolicy() ); } + private ISharpLinkClient CreateClusterClient( + StaticEndpointConfiguration[] configurations, + SharpLinkClusterOptions cluster, + SharpLinkRuntimeContext runtimeContext, + SharpLinkProtocolOptions protocolOptions) + => new SharpLinkClient( + configurations[0].TransportFactory, + _heartbeatInterval, + _heartbeatTimeout, + _loggerFactory ?? NullLoggerFactory.Instance, + _requestTimeout, + _authenticator, + protocolOptions, + runtimeContext, + _rpcSessionFlushOptions, + new SharpLinkConnectionPoolOptions(), + _interceptors.ToArray(), + configurations, + cluster, + _loadBalancingStrategy, + _endpointSelector, + retryOptions: CreateRetryOptions(), + retryPolicy: _retryPolicy, + endpointAdmissionPolicy: CreateEndpointAdmissionPolicy()); + + private ISharpLinkClient CreateDynamicClusterClient( + ISharpLinkEndpointResolver resolver, + SharpLinkEndpointTransportFactory transportFactory, + SharpLinkClusterOptions cluster, + SharpLinkRuntimeContext runtimeContext, + SharpLinkProtocolOptions protocolOptions) + => new SharpLinkClient( + DynamicClusterTransportPlaceholder.Instance, + _heartbeatInterval, + _heartbeatTimeout, + _loggerFactory ?? NullLoggerFactory.Instance, + _requestTimeout, + _authenticator, + protocolOptions, + runtimeContext, + _rpcSessionFlushOptions, + new SharpLinkConnectionPoolOptions(), + _interceptors.ToArray(), + dynamicResolver: resolver, + dynamicTransportFactory: transportFactory, + clusterOptions: cluster, + loadBalancingStrategy: _loadBalancingStrategy, + endpointSelector: _endpointSelector, + retryOptions: CreateRetryOptions(), + retryPolicy: _retryPolicy, + endpointAdmissionPolicy: CreateEndpointAdmissionPolicy()); + + private SharpLinkRetryOptions? CreateRetryOptions() + => _retryConfigured ? _retry.CloneValidated() : null; + + private ISharpLinkEndpointAdmissionPolicy? CreateEndpointAdmissionPolicy() + => _circuitBreakerConfigured + ? new SharpLinkCircuitBreaker(_circuitBreaker.CloneValidated()) + : _endpointAdmissionPolicy; + + internal static IClientTransportFactory CreateTransportFactory( + SharpLinkEndpoint endpoint, + SharpLinkEndpointTransportFactory factory, + SharpLinkRuntimeContext runtimeContext) + { + var transport = factory(endpoint) ?? throw new InvalidOperationException("Endpoint transport factory returned null."); + if (transport is IPerformanceProfileAwareTransport profileAware) + { + try + { + profileAware.BindPerformanceProfile(runtimeContext.Options.PerformanceProfile); + } + catch + { + try { transport.DisposeAsync().AsTask().GetAwaiter().GetResult(); } + catch { } + throw; + } + } + return transport; + } + + internal static SharpLinkEndpoint[] CreateEndpointSnapshot( + IEnumerable source, + bool allowEmpty) + { + var endpoints = new List(); + var ids = new HashSet(StringComparer.Ordinal); + foreach (var endpoint in source) + { + ArgumentNullException.ThrowIfNull(endpoint); + ArgumentException.ThrowIfNullOrWhiteSpace(endpoint.Id); + if (endpoint.Id.Length > 256 || !StringComparer.Ordinal.Equals(endpoint.Id, endpoint.Id.Trim())) + throw new ArgumentException("Endpoint IDs must be trimmed and at most 256 characters.", nameof(source)); + ArgumentNullException.ThrowIfNull(endpoint.Address); + if (endpoint.Attributes is null) + throw new ArgumentException("Endpoint attributes cannot be null.", nameof(source)); + if (endpoint.Attributes.Count > 32) + throw new ArgumentException("An endpoint supports at most 32 attributes.", nameof(source)); + var attributes = new Dictionary(endpoint.Attributes.Count, StringComparer.Ordinal); + foreach (var attribute in endpoint.Attributes) + { + ArgumentException.ThrowIfNullOrWhiteSpace(attribute.Key); + ArgumentNullException.ThrowIfNull(attribute.Value); + if (attribute.Key.Length > 128 || attribute.Value.Length > 1024) + throw new ArgumentException("Endpoint attribute limits were exceeded.", nameof(source)); + attributes.Add(attribute.Key, attribute.Value); + } + if (!ids.Add(endpoint.Id)) + throw new ArgumentException("Endpoint IDs must be unique.", nameof(source)); + endpoints.Add(new SharpLinkEndpoint + { + Id = endpoint.Id, + Address = endpoint.Address, + Authority = endpoint.Authority, + Attributes = attributes.ToFrozenDictionary(StringComparer.Ordinal) + }); + if (endpoints.Count > SharpLinkClusterOptions.MaximumEndpoints) + throw new ArgumentException("A static topology supports at most 64 endpoints.", nameof(source)); + } + if (!allowEmpty && endpoints.Count == 0) + throw new ArgumentException("At least one endpoint is required.", nameof(source)); + return [.. endpoints]; + } + private SharpLinkConnectionPoolOptions CreateConnectionPoolSnapshot(SharpLinkRuntimeContext runtimeContext) { if (_connectionPoolConfigured) diff --git a/src/SharpLink.Client/SharpLinkCircuitBreaker.cs b/src/SharpLink.Client/SharpLinkCircuitBreaker.cs new file mode 100644 index 0000000..9bdac6b --- /dev/null +++ b/src/SharpLink.Client/SharpLinkCircuitBreaker.cs @@ -0,0 +1,305 @@ +using System.Collections.Concurrent; + +namespace SharpLink.Client; + +/// Allows dynamic topology owners to release per-generation policy state deterministically. +internal interface ISharpLinkEndpointAdmissionLifecycle +{ + void Retire(in SharpLinkEndpointCandidate endpoint); +} + +/// +/// Built-in, lazy endpoint-generation breaker. It uses monotonic timestamps and has no timer or +/// topology writer lock on its Closed path. The bounded sample ring is allocated once per active +/// endpoint generation and released when that generation retires. +/// +internal sealed class SharpLinkCircuitBreaker : ISharpLinkEndpointAdmissionPolicy, ISharpLinkEndpointAdmissionLifecycle +{ + private readonly SharpLinkCircuitBreakerOptions _options; + private readonly ConcurrentDictionary _states = new(); + + public SharpLinkCircuitBreaker(SharpLinkCircuitBreakerOptions options) + { + _options = options ?? throw new ArgumentNullException(nameof(options)); + } + + public SharpLinkEndpointAdmissionDecision TryAcquire( + in SharpLinkEndpointCandidate endpoint, + in RpcMethodDescriptor method) + { + var key = new CircuitKey(endpoint.Endpoint.Id, endpoint.Generation); + var state = _states.GetOrAdd(key, static (_, options) => new CircuitState(options), _options); + var decision = state.TryAcquire(Stopwatch.GetTimestamp()); + if (!decision.IsAllowed) + { + SharpLinkTelemetry.RecordEndpointAdmissionRejected("breaker_open"); + SharpLinkTelemetry.RecordBreakerOpen(); + } + return decision; + } + + public void Report(in SharpLinkEndpointOutcome outcome, long token) + { + var key = new CircuitKey(outcome.Endpoint.Endpoint.Id, outcome.Endpoint.Generation); + if (!_states.TryGetValue(key, out var state)) + return; + state.Report(Stopwatch.GetTimestamp(), Classify(outcome), token); + } + + public void Retire(in SharpLinkEndpointCandidate endpoint) + => _states.TryRemove(new CircuitKey(endpoint.Endpoint.Id, endpoint.Generation), out _); + + /// Records an endpoint-level infrastructure failure that has no call admission token. + internal void ReportInfrastructureFailure(in SharpLinkEndpointCandidate endpoint) + { + var key = new CircuitKey(endpoint.Endpoint.Id, endpoint.Generation); + var state = _states.GetOrAdd(key, static (_, options) => new CircuitState(options), _options); + state.ReportInfrastructureFailure(Stopwatch.GetTimestamp()); + } + + private static CircuitSample Classify(in SharpLinkEndpointOutcome outcome) + { + if (outcome.Kind is SharpLinkEndpointOutcomeKind.Cancelled or SharpLinkEndpointOutcomeKind.DeadlineExceeded) + return CircuitSample.Ignore; + if (outcome.Kind is SharpLinkEndpointOutcomeKind.ConnectionClosed or + SharpLinkEndpointOutcomeKind.GoAway) + { + return CircuitSample.Failure; + } + if (outcome.Kind == SharpLinkEndpointOutcomeKind.SendFailure) + { + return outcome.ErrorCode is SharpLinkErrorCode.Unavailable or + SharpLinkErrorCode.ConnectionClosed or + SharpLinkErrorCode.DataLoss or + SharpLinkErrorCode.Internal + ? CircuitSample.Failure + : CircuitSample.Ignore; + } + if (outcome.Kind == SharpLinkEndpointOutcomeKind.RemoteError) + { + return outcome.ErrorCode is SharpLinkErrorCode.Unavailable or + SharpLinkErrorCode.ConnectionClosed or + SharpLinkErrorCode.ResourceExhausted or + SharpLinkErrorCode.DataLoss or + SharpLinkErrorCode.Internal + ? CircuitSample.Failure + : CircuitSample.Success; + } + return CircuitSample.Success; + } + + private readonly record struct CircuitKey(string EndpointId, long Generation); + + private enum CircuitSample : byte + { + Ignore, + Success, + Failure + } + + private sealed class CircuitState + { + private const int Closed = 0; + private const int Open = 1; + private const int HalfOpen = 2; + + private readonly SharpLinkCircuitBreakerOptions _options; + private readonly object _samplesGate = new(); + private readonly long[] _timestamps; + private readonly bool[] _failures; + private readonly long _samplingTicks; + private readonly long _breakTicks; + private int _state; + private long _openUntil; + private int _halfOpenInFlight; + private long _halfOpenEpoch; + private int _head; + private int _count; + private int _failureCount; + + public CircuitState(SharpLinkCircuitBreakerOptions options) + { + _options = options; + var capacity = Math.Max(options.MinimumThroughput * 4, 64); + _timestamps = new long[capacity]; + _failures = new bool[capacity]; + _samplingTicks = ToStopwatchTicks(options.SamplingDuration); + _breakTicks = ToStopwatchTicks(options.BreakDuration); + } + + public SharpLinkEndpointAdmissionDecision TryAcquire(long now) + { + while (true) + { + var state = Volatile.Read(ref _state); + if (state == Closed) + return new SharpLinkEndpointAdmissionDecision(true, Token: 0, RetryAfter: null); + + if (state == Open) + { + TimeSpan? retryAfter = null; + lock (_samplesGate) + { + if (Volatile.Read(ref _state) != Open) + continue; + var openUntil = _openUntil; + if (now < openUntil) + { + retryAfter = TimeSpan.FromSeconds((double)(openUntil - now) / Stopwatch.Frequency); + } + else + { + BeginHalfOpenLocked(); + } + } + if (retryAfter is { } remaining) + return new SharpLinkEndpointAdmissionDecision(false, Token: 0, remaining); + continue; + } + + lock (_samplesGate) + { + if (Volatile.Read(ref _state) != HalfOpen) + continue; + if (_halfOpenInFlight >= _options.HalfOpenMaxCalls) + return new SharpLinkEndpointAdmissionDecision(false, Token: 0, TimeSpan.Zero); + _halfOpenInFlight++; + return new SharpLinkEndpointAdmissionDecision(true, _halfOpenEpoch, RetryAfter: null); + } + } + } + + public void Report(long now, CircuitSample sample, long token) + { + if (token != 0) + { + ReportHalfOpen(now, sample, token); + return; + } + + if (sample == CircuitSample.Ignore || Volatile.Read(ref _state) != Closed) + return; + + lock (_samplesGate) + { + if (Volatile.Read(ref _state) != Closed) + return; + Prune(now); + Add(now, sample == CircuitSample.Failure); + if (_count >= _options.MinimumThroughput && + (double)_failureCount / _count >= _options.FailureRatio) + { + OpenCircuitLocked(now); + } + } + } + + public void ReportInfrastructureFailure(long now) + { + lock (_samplesGate) + { + var state = Volatile.Read(ref _state); + if (state == Open) + return; + if (state == HalfOpen) + { + OpenCircuitLocked(now); + return; + } + + Prune(now); + Add(now, failure: true); + if (_count >= _options.MinimumThroughput && + (double)_failureCount / _count >= _options.FailureRatio) + { + OpenCircuitLocked(now); + } + } + } + + private void ReportHalfOpen(long now, CircuitSample sample, long token) + { + lock (_samplesGate) + { + if (Volatile.Read(ref _state) != HalfOpen || token != _halfOpenEpoch || _halfOpenInFlight == 0) + return; + + _halfOpenInFlight--; + if (sample == CircuitSample.Failure) + { + OpenCircuitLocked(now); + return; + } + + if (sample == CircuitSample.Success && _halfOpenInFlight == 0) + { + _head = 0; + _count = 0; + _failureCount = 0; + Volatile.Write(ref _state, Closed); + } + } + } + + private void BeginHalfOpenLocked() + { + _halfOpenEpoch = NextHalfOpenEpoch(); + _halfOpenInFlight = 0; + Volatile.Write(ref _state, HalfOpen); + } + + private void OpenCircuitLocked(long now) + { + _openUntil = SaturatingAdd(now, _breakTicks); + _halfOpenInFlight = 0; + _halfOpenEpoch = NextHalfOpenEpoch(); + Volatile.Write(ref _state, Open); + } + + private long NextHalfOpenEpoch() + => _halfOpenEpoch == long.MaxValue ? 1 : _halfOpenEpoch + 1; + + private void Prune(long now) + { + var minimum = now - _samplingTicks; + while (_count != 0 && _timestamps[_head] < minimum) + { + if (_failures[_head]) + _failureCount--; + _head = (_head + 1) % _timestamps.Length; + _count--; + } + } + + private void Add(long timestamp, bool failure) + { + if (_count == _timestamps.Length) + { + if (_failures[_head]) + _failureCount--; + _timestamps[_head] = timestamp; + _failures[_head] = failure; + if (failure) + _failureCount++; + _head = (_head + 1) % _timestamps.Length; + return; + } + + var index = (_head + _count) % _timestamps.Length; + _timestamps[index] = timestamp; + _failures[index] = failure; + _count++; + if (failure) + _failureCount++; + } + + private static long ToStopwatchTicks(TimeSpan value) + { + var ticks = value.TotalSeconds * Stopwatch.Frequency; + return ticks >= long.MaxValue ? long.MaxValue : Math.Max(1, (long)Math.Ceiling(ticks)); + } + + private static long SaturatingAdd(long value, long add) + => add >= long.MaxValue - value ? long.MaxValue : value + add; + } +} diff --git a/src/SharpLink.Client/SharpLinkCircuitBreakerOptions.cs b/src/SharpLink.Client/SharpLinkCircuitBreakerOptions.cs new file mode 100644 index 0000000..0db6a97 --- /dev/null +++ b/src/SharpLink.Client/SharpLinkCircuitBreakerOptions.cs @@ -0,0 +1,43 @@ +namespace SharpLink.Client; + +/// Configures the built-in endpoint circuit breaker. +public sealed class SharpLinkCircuitBreakerOptions +{ + /// Gets or sets the minimum samples required before a Closed breaker can open. The default is 20. + public int MinimumThroughput { get; set; } = 20; + + /// Gets or sets the inclusive infrastructure-failure ratio that opens a Closed breaker. The default is 0.5. + public double FailureRatio { get; set; } = 0.5; + + /// Gets or sets the rolling sample window. The default is 30 seconds. + public TimeSpan SamplingDuration { get; set; } = TimeSpan.FromSeconds(30); + + /// Gets or sets how long an Open breaker rejects attempts before HalfOpen probing. The default is 10 seconds. + public TimeSpan BreakDuration { get; set; } = TimeSpan.FromSeconds(10); + + /// Gets or sets the maximum concurrent HalfOpen probes. The default is one. + public int HalfOpenMaxCalls { get; set; } = 1; + + internal SharpLinkCircuitBreakerOptions CloneValidated() + { + if (MinimumThroughput is < 1 or > 1024) + throw new ArgumentOutOfRangeException(nameof(MinimumThroughput)); + if (FailureRatio is <= 0 or > 1 || double.IsNaN(FailureRatio)) + throw new ArgumentOutOfRangeException(nameof(FailureRatio)); + if (SamplingDuration <= TimeSpan.Zero) + throw new ArgumentOutOfRangeException(nameof(SamplingDuration)); + if (BreakDuration <= TimeSpan.Zero) + throw new ArgumentOutOfRangeException(nameof(BreakDuration)); + if (HalfOpenMaxCalls <= 0) + throw new ArgumentOutOfRangeException(nameof(HalfOpenMaxCalls)); + + return new SharpLinkCircuitBreakerOptions + { + MinimumThroughput = MinimumThroughput, + FailureRatio = FailureRatio, + SamplingDuration = SamplingDuration, + BreakDuration = BreakDuration, + HalfOpenMaxCalls = HalfOpenMaxCalls + }; + } +} diff --git a/src/SharpLink.Client/SharpLinkClient.Attempts.cs b/src/SharpLink.Client/SharpLinkClient.Attempts.cs new file mode 100644 index 0000000..ab71b51 --- /dev/null +++ b/src/SharpLink.Client/SharpLinkClient.Attempts.cs @@ -0,0 +1,240 @@ +namespace SharpLink.Client; + +internal sealed partial class SharpLinkClient +{ + /// + /// Owns the outcome of one endpoint-bound attempt. When admission is enabled the same state is + /// registered as the PendingCall completion observer, preserving the existing one-winner race. + /// + private sealed class AttemptOutcomeState : IPendingCallCompletionObserver + { + private readonly SharpLinkClient _client; + private readonly RpcMethodDescriptor _method; + private readonly long _attemptStarted; + private long _endpointStarted; + private PendingCallCompletionReason? _completionReason; + private int _responseObserved; + private SharpLinkErrorCode? _localErrorCode; + private SharpLinkEndpointCandidate _admissionEndpoint; + private long _admissionToken; + private int _hasAdmissionLease; + private int _admissionGranted; + private int _reported; + private int _admissionLeaseVersion; + private int _completionLeaseVersion; + private int _admissionRejected; + private int _admissionDecisionSequence; + private int _lastAdmissionGrantSequence; + private int _lastAdmissionRejectionSequence; + private TimeSpan? _retryAfter; + + public AttemptOutcomeState(SharpLinkClient client, RpcMethodDescriptor method) + { + _client = client; + _method = method; + _attemptStarted = Stopwatch.GetTimestamp(); + SharpLinkTelemetry.RecordClientAttempt(); + } + + public string? EndpointId { get; private set; } + public long EndpointGeneration { get; private set; } + public string? ConnectionId { get; private set; } + + public TimeSpan? RetryAfter => _retryAfter; + + public bool HasAdmissionRejection => Volatile.Read(ref _admissionRejected) != 0; + + public bool HasAdmissionGrant => Volatile.Read(ref _admissionGranted) != 0; + + public bool HasCompletion => Volatile.Read(ref _admissionLeaseVersion) != 0 && + Volatile.Read(ref _completionLeaseVersion) == Volatile.Read(ref _admissionLeaseVersion); + + public bool ShouldHonorAdmissionRetryAfter + => Volatile.Read(ref _lastAdmissionRejectionSequence) > + Volatile.Read(ref _lastAdmissionGrantSequence); + + public void BeginAdmissionSelection() + { + Volatile.Write(ref _admissionRejected, 0); + Volatile.Write(ref _admissionGranted, 0); + Volatile.Write(ref _admissionDecisionSequence, 0); + Volatile.Write(ref _lastAdmissionGrantSequence, 0); + Volatile.Write(ref _lastAdmissionRejectionSequence, 0); + _retryAfter = null; + } + + public bool TryAcquire(in SharpLinkEndpointCandidate endpoint) + { + var policy = _client._endpointAdmissionPolicy; + if (policy is null) + return true; + + SharpLinkEndpointAdmissionDecision decision; + try + { + decision = policy.TryAcquire(endpoint, _method); + } + catch (Exception exception) + { + throw new SharpLinkException( + SharpLinkErrorCode.FailedPrecondition, + "The endpoint admission policy failed while acquiring an attempt.", + exception); + } + + if (decision.RetryAfter is { } retryAfter && retryAfter < TimeSpan.Zero) + { + throw new SharpLinkException( + SharpLinkErrorCode.FailedPrecondition, + "The endpoint admission policy returned a negative retry delay."); + } + if (!decision.IsAllowed) + { + Volatile.Write(ref _admissionRejected, 1); + Volatile.Write(ref _lastAdmissionRejectionSequence, Interlocked.Increment(ref _admissionDecisionSequence)); + if (decision.RetryAfter is { } delay && (_retryAfter is null || delay < _retryAfter.Value)) + _retryAfter = delay; + if (policy is not SharpLinkCircuitBreaker) + SharpLinkTelemetry.RecordEndpointAdmissionRejected("policy"); + return false; + } + + _admissionEndpoint = endpoint; + _admissionToken = decision.Token; + Volatile.Write(ref _lastAdmissionGrantSequence, Interlocked.Increment(ref _admissionDecisionSequence)); + _retryAfter = null; + Interlocked.Increment(ref _admissionLeaseVersion); + Volatile.Write(ref _completionLeaseVersion, 0); + _completionReason = null; + _localErrorCode = null; + Volatile.Write(ref _responseObserved, 0); + Volatile.Write(ref _endpointStarted, Stopwatch.GetTimestamp()); + Volatile.Write(ref _admissionGranted, 1); + Volatile.Write(ref _hasAdmissionLease, 1); + Volatile.Write(ref _reported, 0); + return true; + } + + public void SetConnection(ClientConnection connection) + { + EndpointId = connection.EndpointId; + EndpointGeneration = connection.EndpointGeneration; + ConnectionId = connection.Session.Id; + } + + public void SetLocalFailure(Exception exception) + => _localErrorCode = GetErrorCode(exception); + + public void CompleteWithoutPending(PendingCallCompletionReason reason, Exception? exception = null) + { + SetLocalFailureIfPresent(exception); + _completionReason = reason; + Volatile.Write(ref _completionLeaseVersion, Volatile.Read(ref _admissionLeaseVersion)); + if (reason is PendingCallCompletionReason.Response or PendingCallCompletionReason.RemoteError) + Volatile.Write(ref _responseObserved, 1); + Report(reason, exception); + } + + public void CompleteLocalFailure(Exception exception) + { + var reason = exception switch + { + OperationCanceledException => PendingCallCompletionReason.UserCancellation, + SharpLinkException { Code: SharpLinkErrorCode.DeadlineExceeded } => PendingCallCompletionReason.DeadlineExceeded, + SharpLinkException { Code: SharpLinkErrorCode.ConnectionClosed } => PendingCallCompletionReason.ConnectionClosed, + _ => PendingCallCompletionReason.SendFailure + }; + CompleteWithoutPending(reason, exception); + } + + public void OnResponseObserved() => Volatile.Write(ref _responseObserved, 1); + + public void OnPendingCallCompleted(in PendingCallCompletion completion) + { + _completionReason = completion.Reason; + Volatile.Write(ref _completionLeaseVersion, Volatile.Read(ref _admissionLeaseVersion)); + if (completion.Reason is PendingCallCompletionReason.Response or PendingCallCompletionReason.RemoteError) + Volatile.Write(ref _responseObserved, 1); + SetLocalFailureIfPresent(completion.Exception); + Report(completion.Reason, completion.Exception); + } + + public RetryAttemptOutcome CreateRetryOutcome(Exception exception) + => new( + EndpointId, + EndpointGeneration, + ConnectionId, + _completionReason, + Volatile.Read(ref _responseObserved) != 0, + _localErrorCode ?? GetErrorCode(exception), + Stopwatch.GetElapsedTime(_attemptStarted)); + + private void Report(PendingCallCompletionReason reason, Exception? exception) + { + if (Volatile.Read(ref _hasAdmissionLease) == 0 || Interlocked.Exchange(ref _reported, 1) != 0) + return; + + var policy = _client._endpointAdmissionPolicy; + if (policy is null) + return; + var outcome = new SharpLinkEndpointOutcome( + _admissionEndpoint, + _method, + ToOutcomeKind(reason, exception), + _localErrorCode ?? (exception is null ? null : GetErrorCode(exception)), + Volatile.Read(ref _responseObserved) != 0, + Stopwatch.GetElapsedTime(Volatile.Read(ref _endpointStarted))); + try + { + policy.Report(outcome, _admissionToken); + } + catch (Exception reportException) + { + _client._logger.LogError(reportException, "SharpLink endpoint admission policy report failed."); + } + finally + { + Volatile.Write(ref _hasAdmissionLease, 0); + } + } + + private void SetLocalFailureIfPresent(Exception? exception) + { + if (exception is not null) + SetLocalFailure(exception); + } + } + + private readonly record struct RetryAttemptOutcome( + string? EndpointId, + long EndpointGeneration, + string? ConnectionId, + PendingCallCompletionReason? CompletionReason, + bool ResponseObserved, + SharpLinkErrorCode? ErrorCode, + TimeSpan Elapsed); + + private static SharpLinkEndpointOutcomeKind ToOutcomeKind( + PendingCallCompletionReason reason, + Exception? exception) + => reason switch + { + PendingCallCompletionReason.Response or PendingCallCompletionReason.LocalStreamComplete => SharpLinkEndpointOutcomeKind.Success, + PendingCallCompletionReason.RemoteError => SharpLinkEndpointOutcomeKind.RemoteError, + PendingCallCompletionReason.RemoteStreamComplete when exception is null => SharpLinkEndpointOutcomeKind.Success, + PendingCallCompletionReason.RemoteStreamComplete => SharpLinkEndpointOutcomeKind.RemoteError, + PendingCallCompletionReason.SendFailure => SharpLinkEndpointOutcomeKind.SendFailure, + PendingCallCompletionReason.ConnectionClosed => SharpLinkEndpointOutcomeKind.ConnectionClosed, + PendingCallCompletionReason.GoAway => SharpLinkEndpointOutcomeKind.GoAway, + PendingCallCompletionReason.DeadlineExceeded => SharpLinkEndpointOutcomeKind.DeadlineExceeded, + _ => SharpLinkEndpointOutcomeKind.Cancelled + }; + + private static SharpLinkErrorCode? GetErrorCode(Exception exception) + => exception switch + { + SharpLinkException sharpLinkException => sharpLinkException.Code, + IOException or ObjectDisposedException => SharpLinkErrorCode.ConnectionClosed, + _ => null + }; +} diff --git a/src/SharpLink.Client/SharpLinkClient.CallOptions.cs b/src/SharpLink.Client/SharpLinkClient.CallOptions.cs index 6215de5..3671205 100644 --- a/src/SharpLink.Client/SharpLinkClient.CallOptions.cs +++ b/src/SharpLink.Client/SharpLinkClient.CallOptions.cs @@ -2,6 +2,10 @@ namespace SharpLink.Client; internal sealed partial class SharpLinkClient { + // This duration is accepted by Task.Delay on every supported runtime. Longer public retry + // and admission delays are awaited in cancellable slices rather than rejected by the timer. + private static readonly TimeSpan MaximumRetryOrAdmissionDelay = TimeSpan.FromMilliseconds(int.MaxValue); + private ResolvedCallControl ResolveCallControl( SharpLinkCallOptions options, bool includeClientDefault, @@ -41,15 +45,45 @@ private ResolvedCallControl ResolveCallControl( private async ValueTask GetReadyConnectionAsync( bool waitForReady, DateTimeOffset? deadline, - CancellationToken cancellationToken) + CancellationToken cancellationToken, + RpcMethodDescriptor? method = null, + AttemptOutcomeState? attemptOutcome = null) { while (true) { - if (!_shutdownCts.IsCancellationRequested && ReadyConnectionCount != 0) - return GetReadyConnection(); + attemptOutcome?.BeginAdmissionSelection(); + try + { + if (!_shutdownCts.IsCancellationRequested && ReadyConnectionCount != 0) + return method is { } descriptor + ? GetReadyConnection(descriptor, retrySelection: null, attemptOutcome) + : GetReadyConnection(); + + if (!waitForReady) + return method is { } descriptor + ? GetReadyConnection(descriptor, retrySelection: null, attemptOutcome) + : GetReadyConnection(); + } + catch (SharpLinkException exception) when ( + waitForReady && exception.Code == SharpLinkErrorCode.Unavailable) + { + if (attemptOutcome?.ShouldHonorAdmissionRetryAfter == true) + { + if (attemptOutcome.RetryAfter is not { } retryAfter) + throw; + var delay = retryAfter > TimeSpan.Zero ? retryAfter : TimeSpan.FromMilliseconds(1); + if (deadline is { } retryDeadline && WouldReachDeadline(retryDeadline, delay)) + throw CreateDeadlineExceededException(); + await DelayForRetryOrAdmissionAsync(delay, cancellationToken).ConfigureAwait(false); + continue; + } + + // A grant after the rejection supersedes that earlier delay, but a stale grant must + // not suppress a retry-after returned by a later rejected endpoint. + if (attemptOutcome?.HasAdmissionRejection == true && !attemptOutcome.HasAdmissionGrant) + throw; + } - if (!waitForReady) - return GetReadyConnection(); if (State == SharpLinkConnectionState.Stopped || _shutdownCts.IsCancellationRequested) throw CreateConnectionClosedException("Client has stopped."); @@ -74,6 +108,29 @@ private async ValueTask GetReadyConnectionAsync( } } + private async ValueTask DelayForRetryOrAdmissionAsync(TimeSpan delay, CancellationToken cancellationToken) + { + if (_shutdownCts.IsCancellationRequested) + throw CreateConnectionClosedException("Client has stopped."); + + using var linkedCancellation = + CancellationTokenSource.CreateLinkedTokenSource(cancellationToken, _shutdownCts.Token); + try + { + while (delay > MaximumRetryOrAdmissionDelay) + { + await Task.Delay(MaximumRetryOrAdmissionDelay, linkedCancellation.Token).ConfigureAwait(false); + delay -= MaximumRetryOrAdmissionDelay; + } + await Task.Delay(delay, linkedCancellation.Token).ConfigureAwait(false); + } + catch (OperationCanceledException) when ( + _shutdownCts.IsCancellationRequested && !cancellationToken.IsCancellationRequested) + { + throw CreateConnectionClosedException("Client has stopped."); + } + } + private static void AddDeadlineCandidate( ref DateTimeOffset? deadline, DateTimeOffset? candidate) @@ -82,6 +139,9 @@ private static void AddDeadlineCandidate( deadline = value; } + private static bool WouldReachDeadline(DateTimeOffset deadline, TimeSpan delay) + => delay >= deadline - DateTimeOffset.UtcNow; + private static DateTimeOffset AddTimeout(DateTimeOffset now, TimeSpan timeout) { try diff --git a/src/SharpLink.Client/SharpLinkClient.DynamicCluster.cs b/src/SharpLink.Client/SharpLinkClient.DynamicCluster.cs new file mode 100644 index 0000000..dd8cdaf --- /dev/null +++ b/src/SharpLink.Client/SharpLinkClient.DynamicCluster.cs @@ -0,0 +1,1381 @@ +namespace SharpLink.Client; + +internal sealed partial class SharpLinkClient +{ + /// + /// Owns a resolver-backed endpoint topology. It deliberately has no nested client: proxy, codec, + /// interceptor, pending-call, and session processing continue to belong to the enclosing client. + /// + private sealed class DynamicClusterRuntime : IEndpointClusterRuntime + { + private readonly SharpLinkClient _client; + private readonly ISharpLinkEndpointResolver _resolver; + private readonly SharpLinkEndpointTransportFactory _transportFactory; + private readonly SharpLinkClusterOptions _options; + private readonly SharpLinkLoadBalancingStrategy _strategy; + private readonly ISharpLinkEndpointSelector? _selector; + private readonly Lock _gate = new(); + private readonly Dictionary _currentById = new(StringComparer.Ordinal); + private readonly List _allStates = []; + private readonly HashSet _retiringConnections = []; + private readonly Dictionary>> _initialDialTasks = []; + private EndpointState[] _current = []; + private EndpointState[] _readyEndpoints = []; + private EndpointSelectionSnapshot _selectionSnapshot = EndpointSelectionSnapshot.Empty; + private Task? _connectTask; + private Task? _resolverTask; + private Task? _stopTask; + private long _lastAcceptedVersion = -1; + private long _nextGeneration; + private int _roundRobinCursor; + private int _leastPendingCursor; + private int _reconnectCursor; + private int _initialConnectCoordinatorCount; + private int _telemetryActiveEndpointCount; + private int _telemetryReadyEndpointCount; + private int _telemetryDrainingEndpointCount; + private int _stopping; + private int _resolverDisposed; + + public DynamicClusterRuntime( + SharpLinkClient client, + ISharpLinkEndpointResolver resolver, + SharpLinkEndpointTransportFactory transportFactory, + SharpLinkClusterOptions options, + SharpLinkLoadBalancingStrategy strategy, + ISharpLinkEndpointSelector? selector) + { + _client = client; + _resolver = resolver; + _transportFactory = transportFactory; + _options = options; + _strategy = strategy; + _selector = selector; + } + + public int ReadyConnectionCount + { + get + { + var endpoints = Volatile.Read(ref _readyEndpoints); + var count = 0; + for (var index = 0; index < endpoints.Length; index++) + count += endpoints[index].ReadyConnections.Length; + return count; + } + } + + public int PendingCallCount => CountConnections(static connection => connection.PendingCalls.Count); + + public int ActiveCallCount => CountConnections(static connection => connection.ActiveCallCount); + + public int ActiveStreamCount => CountConnections(static connection => + ((StreamManager)connection.Session.StreamManager).ActiveStreamCount); + + public ValueTask ConnectAsync(CancellationToken cancellationToken) + { + Task task; + lock (_gate) + { + if (Volatile.Read(ref _stopping) != 0 || _client._shutdownCts.IsCancellationRequested) + return ValueTask.FromException(CreateConnectionClosedException("Client has stopped.")); + if (ReadyConnectionCount != 0) + return ValueTask.CompletedTask; + _client.TransitionTo(SharpLinkConnectionState.Connecting); + if (_connectTask is null || + ((_connectTask.IsFaulted || _connectTask.IsCanceled) && _resolverTask is null)) + _connectTask = StartAsync(_client._shutdownCts.Token); + else if (_connectTask.IsFaulted || _connectTask.IsCanceled || + (_connectTask.IsCompletedSuccessfully && _current.Length != 0)) + _connectTask = WaitForRecoveryAsync(); + task = _connectTask; + } + return cancellationToken.CanBeCanceled ? new ValueTask(task.WaitAsync(cancellationToken)) : new ValueTask(task); + } + + public ClientConnection GetReadyConnection( + RpcMethodDescriptor? method, + EndpointRetrySelectionState? retrySelection, + AttemptOutcomeState? attemptOutcome) + { + var snapshot = Volatile.Read(ref _selectionSnapshot); + var endpoints = snapshot.Endpoints; + if (endpoints.Length == 0) + { + SharpLinkTelemetry.RecordSelectionFailure("no_ready_endpoint"); + throw new SharpLinkException(SharpLinkErrorCode.Unavailable, "No SharpLink endpoint is ready."); + } + + var excluded = retrySelection?.GetExcludedMask(snapshot, endpoints.Length) ?? 0UL; + for (var attempt = 0; attempt < endpoints.Length; attempt++) + { + var selectedIndex = SelectEndpoint(endpoints, snapshot.Candidates, excluded); + if ((uint)selectedIndex >= (uint)endpoints.Length || (excluded & (1UL << selectedIndex)) != 0) + { + throw new SharpLinkException( + SharpLinkErrorCode.FailedPrecondition, + "The endpoint selector returned an unavailable candidate index."); + } + var candidate = snapshot.Candidates[selectedIndex]; + if (method is not null && attemptOutcome is not null && !attemptOutcome.TryAcquire(candidate)) + { + retrySelection?.Exclude(snapshot, selectedIndex); + excluded |= 1UL << selectedIndex; + continue; + } + var connection = SelectConnection(endpoints[selectedIndex]); + retrySelection?.Exclude(snapshot, selectedIndex); + if (connection is not null) + { + attemptOutcome?.SetConnection(connection); + if (connection.ActiveCallCount != 0) + EnsureExpansion(endpoints[selectedIndex]); + return connection; + } + attemptOutcome?.CompleteWithoutPending( + PendingCallCompletionReason.ConnectionClosed, + CreateConnectionClosedException("The selected dynamic endpoint connection is no longer ready.")); + excluded |= 1UL << selectedIndex; + } + + SharpLinkTelemetry.RecordSelectionFailure("no_admitted_connection"); + throw new SharpLinkException(SharpLinkErrorCode.Unavailable, "No SharpLink endpoint connection is ready."); + } + + public void MarkConnectionDraining(ClientConnection connection) + { + ArgumentNullException.ThrowIfNull(connection); + EndpointState? endpoint; + var disposeNow = false; + lock (_gate) + { + endpoint = FindEndpointLocked(connection); + if (endpoint is null) + return; + connection.MarkDraining(); + if (connection.ActiveCallCount == 0) + { + endpoint.Connections.Remove(connection); + _retiringConnections.Remove(connection); + disposeNow = true; + } + else + { + _retiringConnections.Add(connection); + } + PublishReadySnapshotLocked(); + } + + if (disposeNow) + _client.TrackBackgroundTask(DisposeConnectionAsync(connection)); + if (endpoint.Retiring) + ScheduleRetiredStateRelease(endpoint); + else + EnsureReconnect(endpoint); + EnsureMinimumReadyEndpoints(); + UpdateClientReadiness(); + } + + public bool TryGetEndpointCandidate(ClientConnection connection, out SharpLinkEndpointCandidate candidate) + { + lock (_gate) + { + var endpoint = FindEndpointLocked(connection); + if (endpoint is null) + { + candidate = default; + return false; + } + + candidate = new SharpLinkEndpointCandidate( + endpoint.Configuration.Endpoint, + endpoint.ReadyConnectionCountProvider, + endpoint.ActiveCallCountProvider, + endpoint.Generation); + return true; + } + } + + public void RetireDrainingConnectionIfIdle(ClientConnection connection) + { + if (connection.State != ClientConnectionState.Draining || connection.ActiveCallCount != 0) + return; + EndpointState? endpoint; + lock (_gate) + { + endpoint = FindEndpointLocked(connection); + if (endpoint is null || !endpoint.Connections.Remove(connection)) + return; + _retiringConnections.Remove(connection); + PublishReadySnapshotLocked(); + } + _client.TrackBackgroundTask(DisposeConnectionAsync(connection)); + if (endpoint.Retiring) + ScheduleRetiredStateRelease(endpoint); + else + EnsureReconnect(endpoint); + EnsureMinimumReadyEndpoints(); + UpdateClientReadiness(); + } + + public ValueTask StopAsync() + { + lock (_gate) + { + _stopTask ??= StopCoreAsync(); + return new ValueTask(_stopTask); + } + } + + private async Task StartAsync(CancellationToken cancellationToken) + { + var resolverSucceeded = false; + try + { + var snapshot = await _resolver.ResolveAsync(cancellationToken).ConfigureAwait(false); + resolverSucceeded = true; + if (!await ApplySnapshotAsync(snapshot, deferInitialReconciliation: true).ConfigureAwait(false)) + { + throw new InvalidOperationException( + "The endpoint resolver returned an invalid initial topology."); + } + StartResolverWorker(resolveBeforeWatch: false); + await ConnectCurrentEndpointsAsync(cancellationToken).ConfigureAwait(false); + UpdateClientReadiness(); + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested || _client._shutdownCts.IsCancellationRequested) + { + throw; + } + catch (Exception exception) + { + if (!resolverSucceeded) + SharpLinkTelemetry.RecordClientResolverFailure(); + _client.TransitionTo(SharpLinkConnectionState.Reconnecting); + StartResolverWorker(resolveBeforeWatch: true); + throw new SharpLinkException( + SharpLinkErrorCode.Unavailable, + "The endpoint resolver could not provide an initial topology.", + exception); + } + } + + private async Task WaitForRecoveryAsync() + { + while (true) + { + if (Volatile.Read(ref _stopping) != 0 || _client._shutdownCts.IsCancellationRequested) + throw new OperationCanceledException(_client._shutdownCts.Token); + if (ReadyConnectionCount != 0 || HasAcceptedEmptyTopology()) + return; + + EnsureMinimumReadyEndpoints(); + var signal = Volatile.Read(ref _client._readySignal).Task; + if (ReadyConnectionCount != 0 || HasAcceptedEmptyTopology()) + return; + await signal.ConfigureAwait(false); + } + } + + private bool HasAcceptedEmptyTopology() + { + lock (_gate) + return _lastAcceptedVersion >= 0 && _current.Length == 0; + } + + private void StartResolverWorker(bool resolveBeforeWatch) + { + lock (_gate) + { + if (Volatile.Read(ref _stopping) != 0 || _resolverTask is { IsCompleted: false }) + return; + _resolverTask = RunResolverWorkerAsync(resolveBeforeWatch); + _client.TrackBackgroundTask(_resolverTask); + } + } + + private async Task RunResolverWorkerAsync(bool resolveBeforeWatch) + { + var delayMilliseconds = 100; + var mustResolve = resolveBeforeWatch; + while (Volatile.Read(ref _stopping) == 0 && !_client._shutdownCts.IsCancellationRequested) + { + if (mustResolve) + { + try + { + var snapshot = await _resolver.ResolveAsync(_client._shutdownCts.Token).ConfigureAwait(false); + if (await ApplySnapshotAsync(snapshot).ConfigureAwait(false)) + delayMilliseconds = 100; + mustResolve = false; + UpdateClientReadiness(); + } + catch (OperationCanceledException) when (_client._shutdownCts.IsCancellationRequested) + { + return; + } + catch (Exception exception) + { + SharpLinkTelemetry.RecordClientResolverFailure(); + LogClientBackgroundLoopUnhandledException(_client._logger, nameof(RunResolverWorkerAsync), exception); + await DelayResolverRetryAsync(delayMilliseconds).ConfigureAwait(false); + delayMilliseconds = Math.Min(delayMilliseconds * 2, 30_000); + continue; + } + } + + try + { + await foreach (var snapshot in _resolver.WatchAsync(_client._shutdownCts.Token) + .WithCancellation(_client._shutdownCts.Token) + .ConfigureAwait(false)) + { + if (Volatile.Read(ref _stopping) != 0) + return; + if (await ApplySnapshotAsync(snapshot).ConfigureAwait(false)) + delayMilliseconds = 100; + UpdateClientReadiness(); + } + mustResolve = true; + } + catch (OperationCanceledException) when (_client._shutdownCts.IsCancellationRequested) + { + return; + } + catch (Exception exception) + { + SharpLinkTelemetry.RecordClientResolverFailure(); + LogClientBackgroundLoopUnhandledException(_client._logger, nameof(RunResolverWorkerAsync), exception); + mustResolve = true; + } + + await DelayResolverRetryAsync(delayMilliseconds).ConfigureAwait(false); + delayMilliseconds = Math.Min(delayMilliseconds * 2, 30_000); + } + } + + private async Task DelayResolverRetryAsync(int delayMilliseconds) + { + var jitter = 0.8 + Random.Shared.NextDouble() * 0.4; + await Task.Delay(TimeSpan.FromMilliseconds(delayMilliseconds * jitter), _client._shutdownCts.Token) + .ConfigureAwait(false); + } + + private async Task ApplySnapshotAsync( + SharpLinkEndpointSnapshot snapshot, + bool deferInitialReconciliation = false) + { + ArgumentNullException.ThrowIfNull(snapshot); + SharpLinkEndpoint[] endpoints; + try + { + endpoints = SharpClientBuilder.CreateEndpointSnapshot(snapshot.Endpoints, allowEmpty: true); + if (endpoints.Length > _options.MaxEndpoints) + throw new ArgumentException("The endpoint resolver snapshot exceeds MaxEndpoints.", nameof(snapshot)); + } + catch (Exception exception) + { + SharpLinkTelemetry.RecordClientResolverFailure(); + LogClientBackgroundLoopUnhandledException(_client._logger, nameof(ApplySnapshotAsync), exception); + return false; + } + + Dictionary previous; + HashSet ownedFactories; + lock (_gate) + { + if (Volatile.Read(ref _stopping) != 0 || snapshot.Version <= _lastAcceptedVersion) + return false; + previous = new Dictionary(_currentById, StringComparer.Ordinal); + ownedFactories = GetOwnedFactoriesLocked(); + } + + var created = new Dictionary(StringComparer.Ordinal); + try + { + for (var index = 0; index < endpoints.Length; index++) + { + var endpoint = endpoints[index]; + if (previous.TryGetValue(endpoint.Id, out var existing) && SameGeneration(existing.Configuration.Endpoint, endpoint)) + continue; + var factory = SharpClientBuilder.CreateTransportFactory(endpoint, _transportFactory, _client._runtimeContext); + created.Add(endpoint.Id, new EndpointState( + new StaticEndpointConfiguration(endpoint, factory), + Interlocked.Increment(ref _nextGeneration))); + if (factory is AnonymousPipeClientTransportFactory) + { + throw new InvalidOperationException( + "Anonymous-pipe handle offers cannot be used by endpoint clusters."); + } + } + } + catch (Exception exception) + { + lock (_gate) + ownedFactories.UnionWith(GetOwnedFactoriesLocked()); + await DisposeCreatedFactoriesAsync(created.Values, ownedFactories).ConfigureAwait(false); + SharpLinkTelemetry.RecordClientResolverFailure(); + LogClientBackgroundLoopUnhandledException(_client._logger, nameof(ApplySnapshotAsync), exception); + return false; + } + + var abandoned = false; + var rejectedForFactoryOwnership = false; + var connectionsToDispose = new List(); + var statesToRelease = new List(); + EndpointState[] current; + lock (_gate) + { + if (Volatile.Read(ref _stopping) != 0 || snapshot.Version <= _lastAcceptedVersion) + { + abandoned = true; + ownedFactories.UnionWith(GetOwnedFactoriesLocked()); + current = []; + } + else if (!HasUniqueFactoryOwnershipLocked(created.Values)) + { + rejectedForFactoryOwnership = true; + ownedFactories.UnionWith(GetOwnedFactoriesLocked()); + current = []; + } + else + { + var nextById = new Dictionary(endpoints.Length, StringComparer.Ordinal); + current = new EndpointState[endpoints.Length]; + for (var index = 0; index < endpoints.Length; index++) + { + var endpoint = endpoints[index]; + EndpointState state; + if (previous.TryGetValue(endpoint.Id, out var existing) && SameGeneration(existing.Configuration.Endpoint, endpoint)) + { + existing.Configuration.ReplaceEndpoint(endpoint); + state = existing; + } + else + { + state = created[endpoint.Id]; + _allStates.Add(state); + } + nextById.Add(endpoint.Id, state); + current[index] = state; + } + + foreach (var old in _current) + { + if (!nextById.TryGetValue(old.Configuration.Endpoint.Id, out var replacement) || + !ReferenceEquals(replacement, old)) + { + RetireEndpointLocked(old, connectionsToDispose, statesToRelease); + } + } + + _currentById.Clear(); + foreach (var pair in nextById) + _currentById.Add(pair.Key, pair.Value); + _current = current; + _lastAcceptedVersion = snapshot.Version; + SharpLinkTelemetry.AddClientActiveEndpoints(current.Length - _telemetryActiveEndpointCount); + _telemetryActiveEndpointCount = current.Length; + PublishReadySnapshotLocked(force: true); + } + } + + if (abandoned || rejectedForFactoryOwnership) + { + await DisposeCreatedFactoriesAsync(created.Values, ownedFactories).ConfigureAwait(false); + if (rejectedForFactoryOwnership) + { + SharpLinkTelemetry.RecordClientResolverFailure(); + LogClientBackgroundLoopUnhandledException( + _client._logger, + nameof(ApplySnapshotAsync), + new InvalidOperationException( + "A resolver snapshot reused a transport factory owned by another endpoint generation.")); + } + return false; + } + + if (current.Length == 0) + Volatile.Read(ref _client._readySignal).TrySetResult(true); + for (var index = 0; index < connectionsToDispose.Count; index++) + _client.TrackBackgroundTask(DisposeConnectionAsync(connectionsToDispose[index])); + for (var index = 0; index < statesToRelease.Count; index++) + ScheduleRetiredStateRelease(statesToRelease[index]); + if (!deferInitialReconciliation) + EnsureMinimumReadyEndpoints(); + SharpLinkTelemetry.RecordClientResolverUpdate(); + return true; + } + + private void RetireEndpointLocked( + EndpointState endpoint, + List connectionsToDispose, + List statesToRelease) + { + if (endpoint.Retiring) + return; + endpoint.Retiring = true; + SharpLinkTelemetry.AddClientDrainingEndpoints(1); + _telemetryDrainingEndpointCount++; + var connections = endpoint.Connections.ToArray(); + for (var index = 0; index < connections.Length; index++) + { + var connection = connections[index]; + connection.MarkDraining(); + if (connection.ActiveCallCount == 0) + { + endpoint.Connections.Remove(connection); + _retiringConnections.Remove(connection); + connectionsToDispose.Add(connection); + } + else + { + _retiringConnections.Add(connection); + } + } + if (endpoint.Connections.Count == 0 && endpoint.ConnectingCount == 0) + statesToRelease.Add(endpoint); + } + + private async Task ConnectCurrentEndpointsAsync(CancellationToken cancellationToken) + { + EndpointState[] endpoints; + lock (_gate) + endpoints = [.. _current]; + if (endpoints.Length == 0) + return; + + Interlocked.Increment(ref _initialConnectCoordinatorCount); + try + { + Exception? lastFailure = null; + var parallelism = Math.Min(Math.Min(_options.MinReadyEndpoints, endpoints.Length), 4); + var nextEndpoint = 0; + var remaining = new List>(parallelism); + while (nextEndpoint < parallelism) + { + var startGate = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var endpoint = endpoints[nextEndpoint++]; + var attempt = TryConnectOneAfterInitialReservationAsync(endpoint, cancellationToken, startGate.Task); + TrackInitialDials([endpoint], [attempt]); + remaining.Add(attempt); + startGate.TrySetResult(); + + } + + while (remaining.Count != 0) + { + if (cancellationToken.IsCancellationRequested || _client._shutdownCts.IsCancellationRequested) + { + throw new OperationCanceledException( + cancellationToken.IsCancellationRequested ? cancellationToken : _client._shutdownCts.Token); + } + if (ReadyConnectionCount != 0 || HasAcceptedEmptyTopology()) + { + EnsureMinimumReadyEndpoints(); + return; + } + + var readySignal = Volatile.Read(ref _client._readySignal).Task; + var nextDial = Task.WhenAny(remaining); + var completed = await Task.WhenAny(nextDial, readySignal).ConfigureAwait(false); + if (ReferenceEquals(completed, readySignal)) + continue; + + var dial = await nextDial.ConfigureAwait(false); + remaining.Remove(dial); + lastFailure ??= await dial.ConfigureAwait(false); + if (ReadyConnectionCount != 0 || HasAcceptedEmptyTopology()) + { + EnsureMinimumReadyEndpoints(); + return; + } + + if (nextEndpoint >= endpoints.Length) + continue; + + var startGate = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var endpoint = endpoints[nextEndpoint++]; + var attempt = TryConnectOneAfterInitialReservationAsync(endpoint, cancellationToken, startGate.Task); + TrackInitialDials([endpoint], [attempt]); + remaining.Add(attempt); + startGate.TrySetResult(); + } + + EnsureMinimumReadyEndpoints(); + throw new SharpLinkException( + SharpLinkErrorCode.Unavailable, + "No dynamic SharpLink endpoint could connect.", + lastFailure); + } + finally + { + if (Interlocked.Decrement(ref _initialConnectCoordinatorCount) == 0 && + Volatile.Read(ref _stopping) == 0 && !_client._shutdownCts.IsCancellationRequested) + { + // A sibling can release its current-generation initial reservation while this + // coordinator is active. Reconcile once the coordinator hand-off is complete. + EnsureMinimumReadyEndpoints(); + } + } + } + + private void TrackInitialDials(EndpointState[] endpoints, Task[] attempts) + { + ArgumentOutOfRangeException.ThrowIfNotEqual(endpoints.Length, attempts.Length); + lock (_gate) + { + for (var index = 0; index < attempts.Length; index++) + { + var endpoint = endpoints[index]; + if (!_initialDialTasks.TryGetValue(endpoint, out var endpointAttempts)) + { + endpointAttempts = []; + _initialDialTasks.Add(endpoint, endpointAttempts); + } + endpointAttempts.Add(attempts[index]); + endpoint.InitialDialReservations++; + } + } + for (var index = 0; index < attempts.Length; index++) + _client.TrackBackgroundTask(ObserveInitialDialAsync(endpoints[index], attempts[index])); + } + + private async Task ObserveInitialDialAsync(EndpointState endpoint, Task attempt) + { + var shouldReconcile = false; + try + { + _ = await attempt.ConfigureAwait(false); + shouldReconcile = Volatile.Read(ref _stopping) == 0; + } + catch (OperationCanceledException) when (_client._shutdownCts.IsCancellationRequested) + { + } + finally + { + lock (_gate) + { + if (_initialDialTasks.TryGetValue(endpoint, out var endpointAttempts)) + { + endpointAttempts.Remove(attempt); + endpoint.InitialDialReservations--; + if (endpointAttempts.Count == 0) + _initialDialTasks.Remove(endpoint); + } + } + } + + if (shouldReconcile && Volatile.Read(ref _initialConnectCoordinatorCount) == 0) + EnsureMinimumReadyEndpoints(); + } + + private async Task TryConnectOneAfterInitialReservationAsync( + EndpointState endpoint, + CancellationToken cancellationToken, + Task startGate) + { + await startGate.ConfigureAwait(false); + return await TryConnectOneAsync(endpoint, cancellationToken).ConfigureAwait(false); + } + + private async Task TryConnectOneAsync(EndpointState endpoint, CancellationToken cancellationToken) + { + try + { + await ConnectOneAsync(endpoint, cancellationToken).ConfigureAwait(false); + return null; + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested || _client._shutdownCts.IsCancellationRequested) + { + throw; + } + catch (Exception exception) + { + return exception; + } + } + + private async Task ConnectOneAsync(EndpointState endpoint, CancellationToken cancellationToken) + { + lock (_gate) + { + if (Volatile.Read(ref _stopping) != 0 || _client._shutdownCts.IsCancellationRequested || + endpoint.Retiring || !IsCurrentLocked(endpoint) || + IsRetiringBudgetExceededLocked() || + TotalActiveConnectionsLocked() >= _options.MaxConnections || + endpoint.NonRetiringConnectionCount + endpoint.ConnectingCount >= _options.MaxConnectionsPerEndpoint) + { + return; + } + endpoint.ConnectingCount++; + } + + RpcSession? session = null; + ITransportConnection? transport = null; + ClientConnection? connection = null; + try + { + using var attemptCts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken, _client._shutdownCts.Token); + transport = await endpoint.Configuration.TransportFactory.ConnectAsync(attemptCts.Token).ConfigureAwait(false); + if (transport is ITransportSecurityInfo securityInfo) + LogTlsEstablished(_client._logger, securityInfo.Protocol, securityInfo.CipherSuite); + session = new RpcSession(transport, _client._rpcSessionFlushOptions); + transport = null; + session.SetTelemetrySide("client"); + session.BindRuntimeContext(_client._runtimeContext); + + using var handshakeTimeout = new CancellationTokenSource(_client._protocolOptions.HandshakeTimeout); + using var handshakeCts = CancellationTokenSource.CreateLinkedTokenSource(attemptCts.Token, handshakeTimeout.Token); + var handshakeException = await _client.ProcessHandshakeAsync(session, handshakeCts.Token).ConfigureAwait(false); + if (handshakeException is not null) + throw handshakeException; + + var sessionCts = CancellationTokenSource.CreateLinkedTokenSource(_client._shutdownCts.Token); + var createdConnection = new ClientConnection( + _client, + session, + sessionCts, + _client._protocolOptions.MaxPendingRequestsPerConnection, + _client._runtimeContext.Codecs, + endpoint.Configuration.Endpoint.Id, + endpoint.Generation); + connection = createdConnection; + createdConnection.Session.OnDisconnected += exception => HandleDisconnected( + endpoint, + createdConnection, + exception ?? CreateConnectionClosedException("Transport closed.")); + + lock (_gate) + { + if (Volatile.Read(ref _stopping) != 0 || endpoint.Retiring || !IsCurrentLocked(endpoint) || + IsRetiringBudgetExceededLocked()) + throw CreateConnectionClosedException("Endpoint generation retired while connecting."); + endpoint.Connections.Add(createdConnection); + PublishReadySnapshotLocked(); + } + session.NotifyConnected(); + _client.TrackBackgroundTask(_client.RunHeartbeatSendLoopAsync(createdConnection, sessionCts.Token)); + _client.TrackBackgroundTask(_client.RunProcessRequestLoopAsync(createdConnection, sessionCts.Token)); + session = null; + connection = null; + UpdateClientReadiness(); + EnsureMinimumReadyEndpoints(); + } + finally + { + var release = false; + lock (_gate) + { + endpoint.ConnectingCount--; + release = endpoint.Retiring && endpoint.Connections.Count == 0 && endpoint.ConnectingCount == 0; + } + if (release) + ScheduleRetiredStateRelease(endpoint); + if (transport is not null) + await transport.DisposeAsync().ConfigureAwait(false); + if (connection is not null) + await connection.DisposeAsync().ConfigureAwait(false); + else if (session is not null) + await session.DisposeAsync().ConfigureAwait(false); + } + } + + private void HandleDisconnected(EndpointState endpoint, ClientConnection connection, Exception exception) + { + var retired = false; + lock (_gate) + { + if (!endpoint.Connections.Remove(connection)) + return; + _retiringConnections.Remove(connection); + retired = endpoint.Retiring; + PublishReadySnapshotLocked(); + } + connection.Fail(exception); + _client.TrackBackgroundTask(DisposeConnectionAsync(connection)); + if (retired) + ScheduleRetiredStateRelease(endpoint); + else if (Volatile.Read(ref _stopping) == 0) + { + EnsureReconnect(endpoint); + } + if (Volatile.Read(ref _stopping) == 0) + EnsureMinimumReadyEndpoints(); + UpdateClientReadiness(); + } + + private void EnsureMinimumReadyEndpoints() + { + List? missing = null; + lock (_gate) + { + if (Volatile.Read(ref _stopping) != 0) + return; + var target = Math.Min(_options.MinReadyEndpoints, _current.Length); + var availableCapacity = _options.MaxConnections - TotalActiveConnectionsLocked(); + var activeReconnects = _current.Count(static endpoint => endpoint.ReconnectTask is { IsCompleted: false }); + var activeInitialDials = CountActiveCurrentInitialDialsLocked(); + var remaining = Math.Min(target - Volatile.Read(ref _readyEndpoints).Length - activeReconnects - activeInitialDials, availableCapacity); + var start = unchecked((uint)Interlocked.Increment(ref _reconnectCursor)); + for (var offset = 0; remaining > 0 && offset < _current.Length; offset++) + { + var index = (int)((start + (uint)offset) % (uint)_current.Length); + var endpoint = _current[index]; + if (endpoint.ReadyConnections.Length != 0 || + endpoint.NonRetiringConnectionCount + endpoint.ConnectingCount != 0 || + endpoint.ReconnectTask is { IsCompleted: false }) + continue; + (missing ??= []).Add(endpoint); + remaining--; + } + } + + if (missing is not null) + for (var index = 0; index < missing.Count; index++) + EnsureReconnect(missing[index]); + } + + private int CountActiveCurrentInitialDialsLocked() + { + var count = 0; + for (var index = 0; index < _current.Length; index++) + count += _current[index].InitialDialReservations; + return count; + } + + private void EnsureReconnect(EndpointState endpoint) + { + lock (_gate) + { + var target = Math.Min(_options.MinReadyEndpoints, _current.Length); + var activeReconnects = _current.Count(static candidate => candidate.ReconnectTask is { IsCompleted: false }); + if (endpoint.ReconnectTask is { IsCompleted: false } || !NeedsReconnectLocked(endpoint) || + activeReconnects >= target - Volatile.Read(ref _readyEndpoints).Length) + { + return; + } + endpoint.ReconnectTask = ReconnectAsync(endpoint); + _client.TrackBackgroundTask(endpoint.ReconnectTask); + } + } + + private void EnsureExpansion(EndpointState endpoint) + { + lock (_gate) + { + if (Volatile.Read(ref _stopping) != 0 || endpoint.Retiring || !IsCurrentLocked(endpoint) || + endpoint.ExpansionTask is { IsCompleted: false } || + IsRetiringBudgetExceededLocked() || + TotalActiveConnectionsLocked() >= _options.MaxConnections || + endpoint.NonRetiringConnectionCount + endpoint.ConnectingCount >= _options.MaxConnectionsPerEndpoint) + { + return; + } + endpoint.ExpansionTask = ExpandAsync(endpoint); + _client.TrackBackgroundTask(endpoint.ExpansionTask); + } + } + + private async Task ExpandAsync(EndpointState endpoint) + { + try + { + await ConnectOneAsync(endpoint, _client._shutdownCts.Token).ConfigureAwait(false); + } + catch (OperationCanceledException) when (_client._shutdownCts.IsCancellationRequested) + { + } + catch (Exception exception) + { + LogClientBackgroundLoopUnhandledException(_client._logger, nameof(ExpandAsync), exception); + if (endpoint.ReadyConnections.Length == 0) + EnsureReconnect(endpoint); + } + } + + private async Task ReconnectAsync(EndpointState endpoint) + { + int delayMilliseconds; + lock (_gate) + delayMilliseconds = endpoint.ReconnectDelayMilliseconds; + try + { + var jitterMilliseconds = Random.Shared.Next(delayMilliseconds / 4 + 1); + await Task.Delay(TimeSpan.FromMilliseconds(delayMilliseconds + jitterMilliseconds), _client._shutdownCts.Token).ConfigureAwait(false); + var shouldConnect = false; + lock (_gate) + shouldConnect = NeedsReconnectLocked(endpoint); + if (shouldConnect) + { + SharpLinkTelemetry.ReconnectAttempt(); + await ConnectOneAsync(endpoint, _client._shutdownCts.Token).ConfigureAwait(false); + lock (_gate) + endpoint.ReconnectDelayMilliseconds = endpoint.ReadyConnections.Length != 0 ? 100 : NextReconnectDelay(delayMilliseconds); + } + } + catch (OperationCanceledException) when (_client._shutdownCts.IsCancellationRequested) + { + return; + } + catch (Exception exception) + { + LogClientBackgroundLoopUnhandledException(_client._logger, nameof(ReconnectAsync), exception); + lock (_gate) + endpoint.ReconnectDelayMilliseconds = NextReconnectDelay(delayMilliseconds); + } + finally + { + lock (_gate) + endpoint.ReconnectTask = null; + } + if (Volatile.Read(ref _stopping) == 0 && !_client._shutdownCts.IsCancellationRequested) + EnsureMinimumReadyEndpoints(); + } + + private static int NextReconnectDelay(int delayMilliseconds) => Math.Min(delayMilliseconds * 2, 5000); + + private void UpdateClientReadiness() + { + if (ReadyConnectionCount != 0) + { + _client._readyTimestamp = Stopwatch.GetTimestamp(); + _client.TransitionTo(SharpLinkConnectionState.Ready); + Volatile.Read(ref _client._readySignal).TrySetResult(true); + return; + } + if (Volatile.Read(ref _stopping) == 0 && !_client._shutdownCts.IsCancellationRequested) + { + _client.ResetReadySignal(); + _client.TransitionTo(SharpLinkConnectionState.Reconnecting); + } + } + + private void PublishReadySnapshotLocked(bool force = false) + { + var ready = new List(_current.Length); + for (var index = 0; index < _current.Length; index++) + { + var endpoint = _current[index]; + endpoint.PublishReadyConnections(); + if (endpoint.ReadyConnections.Length != 0) + ready.Add(endpoint); + } + var endpoints = ready.ToArray(); + var existing = Volatile.Read(ref _readyEndpoints); + if (!force && HasSameMembership(existing, endpoints)) + { + if (endpoints.Length == 0) + _client.ResetReadySignal(); + return; + } + var candidates = new SharpLinkEndpointCandidate[endpoints.Length]; + for (var index = 0; index < endpoints.Length; index++) + { + var endpoint = endpoints[index]; + candidates[index] = new SharpLinkEndpointCandidate( + endpoint.Configuration.Endpoint, + endpoint.ReadyConnectionCountProvider, + endpoint.ActiveCallCountProvider, + endpoint.Generation); + } + Volatile.Write(ref _readyEndpoints, endpoints); + Volatile.Write(ref _selectionSnapshot, new EndpointSelectionSnapshot(endpoints, candidates)); + SharpLinkTelemetry.AddClientReadyEndpoints(endpoints.Length - _telemetryReadyEndpointCount); + _telemetryReadyEndpointCount = endpoints.Length; + if (endpoints.Length == 0) + _client.ResetReadySignal(); + } + + private int SelectEndpoint(EndpointState[] endpoints, SharpLinkEndpointCandidate[] candidates, ulong excluded) + { + var availableCount = 0; + for (var index = 0; index < endpoints.Length; index++) + availableCount += (excluded & (1UL << index)) == 0 ? 1 : 0; + if (availableCount == 0) + return -1; + if (availableCount == 1 && _selector is null) + { + for (var index = 0; index < endpoints.Length; index++) + if ((excluded & (1UL << index)) == 0) + return index; + } + if (_selector is not null) + { + try + { + return _selector.Select(new SharpLinkEndpointSelectionContext(candidates, excluded)); + } + catch (Exception exception) + { + _client._logger.LogError(exception, "SharpLink endpoint selector failed."); + throw new SharpLinkException(SharpLinkErrorCode.FailedPrecondition, "The endpoint selector failed.", exception); + } + } + return _strategy switch + { + SharpLinkLoadBalancingStrategy.Random => SelectRandom(endpoints.Length, excluded, availableCount), + SharpLinkLoadBalancingStrategy.RoundRobin => StaticEndpointSelection.SelectRoundRobinIndex(ref _roundRobinCursor, endpoints.Length, excluded), + SharpLinkLoadBalancingStrategy.LeastPending => SelectLeastPending(endpoints, excluded), + _ => SelectPowerOfTwo(endpoints, excluded, availableCount) + }; + } + + private int SelectPowerOfTwo(EndpointState[] endpoints, ulong excluded, int availableCount) + { + var first = SelectRandom(endpoints.Length, excluded, availableCount); + var second = SelectRandom(endpoints.Length, excluded | (1UL << first), availableCount - 1); + if (second < 0) + return first; + var firstState = endpoints[first]; + var secondState = endpoints[second]; + return StaticEndpointSelection.CompareNormalizedLoad( + firstState.ActiveCallCount, firstState.ReadyConnections.Length, + secondState.ActiveCallCount, secondState.ReadyConnections.Length) <= 0 ? first : second; + } + + private static int SelectRandom(int length, ulong excluded, int availableCount) + => availableCount <= 0 ? -1 : StaticEndpointSelection.SelectRandomIndex( + length, excluded, availableCount, Random.Shared.Next(availableCount)); + + private int SelectLeastPending(EndpointState[] endpoints, ulong excluded) + { + var start = unchecked((uint)Interlocked.Increment(ref _leastPendingCursor)); + var selected = -1; + for (var offset = 0; offset < endpoints.Length; offset++) + { + var index = (int)((start + (uint)offset) % (uint)endpoints.Length); + if ((excluded & (1UL << index)) != 0) + continue; + if (selected < 0 || endpoints[index].ActiveCallCount < endpoints[selected].ActiveCallCount) + selected = index; + } + return selected; + } + + private static ClientConnection? SelectConnection(EndpointState endpoint) + { + var connections = endpoint.ReadyConnections; + if (connections.Length == 0) + return null; + if (connections.Length == 1) + return connections[0].CanAcceptCalls ? connections[0] : null; + var first = Random.Shared.Next(connections.Length); + var second = Random.Shared.Next(connections.Length - 1); + if (second >= first) + second++; + var selected = SelectLeastLoaded(connections, first, second); + if (selected.CanAcceptCalls) + return selected; + for (var index = 0; index < connections.Length; index++) + if (connections[index].CanAcceptCalls) + return connections[index]; + return null; + } + + private EndpointState? FindEndpointLocked(ClientConnection connection) + { + for (var index = 0; index < _allStates.Count; index++) + if (_allStates[index].Connections.Contains(connection)) + return _allStates[index]; + return null; + } + + private bool IsCurrentLocked(EndpointState endpoint) + => _currentById.TryGetValue(endpoint.Configuration.Endpoint.Id, out var current) && ReferenceEquals(current, endpoint); + + private bool NeedsReconnectLocked(EndpointState endpoint) + => Volatile.Read(ref _stopping) == 0 && !_client._shutdownCts.IsCancellationRequested && + !endpoint.Retiring && IsCurrentLocked(endpoint) && + !IsRetiringBudgetExceededLocked() && + Volatile.Read(ref _readyEndpoints).Length < Math.Min(_options.MinReadyEndpoints, _current.Length) && + TotalActiveConnectionsLocked() < _options.MaxConnections && + endpoint.NonRetiringConnectionCount + endpoint.ConnectingCount == 0; + + private bool IsRetiringBudgetExceededLocked() + => _retiringConnections.Count > _options.MaxRetiringConnections; + + private int TotalActiveConnectionsLocked() + { + var count = 0; + for (var index = 0; index < _allStates.Count; index++) + count += _allStates[index].NonRetiringConnectionCount + _allStates[index].ConnectingCount; + return count; + } + + private int CountConnections(Func count) + { + lock (_gate) + { + var result = 0; + for (var index = 0; index < _allStates.Count; index++) + foreach (var connection in _allStates[index].Connections) + result += count(connection); + return result; + } + } + + private void ScheduleRetiredStateRelease(EndpointState endpoint) + => _client.TrackBackgroundTask(ReleaseRetiredStateAsync(endpoint)); + + private async Task ReleaseRetiredStateAsync(EndpointState endpoint) + { + lock (_gate) + { + if (!endpoint.Retiring || endpoint.FactoryReleased || endpoint.Connections.Count != 0 || endpoint.ConnectingCount != 0) + return; + endpoint.FactoryReleased = true; + _allStates.Remove(endpoint); + SharpLinkTelemetry.AddClientDrainingEndpoints(-1); + _telemetryDrainingEndpointCount--; + } + if (_client._endpointAdmissionPolicy is ISharpLinkEndpointAdmissionLifecycle lifecycle) + { + var candidate = new SharpLinkEndpointCandidate( + endpoint.Configuration.Endpoint, + endpoint.ReadyConnectionCountProvider, + endpoint.ActiveCallCountProvider, + endpoint.Generation); + lifecycle.Retire(candidate); + } + await DisposeFactoryQuietlyAsync(endpoint.Configuration.TransportFactory).ConfigureAwait(false); + } + + private async Task StopCoreAsync() + { + Interlocked.Exchange(ref _stopping, 1); + var cleanupFailures = new List(); + ClientConnection[] connections; + Task[] workers; + Task? initialConnectTask; + IClientTransportFactory[] factories; + lock (_gate) + { + initialConnectTask = _connectTask; + connections = [.. _allStates.SelectMany(static state => state.Connections)]; + workers = [.. _allStates + .SelectMany(static state => new[] { state.ReconnectTask, state.ExpansionTask }) + .Concat(_initialDialTasks.Values.SelectMany(static attempts => attempts)) + .Append(initialConnectTask) + .Append(_resolverTask) + .Where(static task => task is not null)!]; + factories = [.. _allStates + .Where(static state => !state.FactoryReleased) + .Select(static state => + { + state.FactoryReleased = true; + return state.Configuration.TransportFactory; + })]; + for (var index = 0; index < _allStates.Count; index++) + _allStates[index].Connections.Clear(); + _allStates.Clear(); + _currentById.Clear(); + _current = []; + _retiringConnections.Clear(); + Volatile.Write(ref _readyEndpoints, []); + Volatile.Write(ref _selectionSnapshot, EndpointSelectionSnapshot.Empty); + SharpLinkTelemetry.AddClientActiveEndpoints(-_telemetryActiveEndpointCount); + SharpLinkTelemetry.AddClientReadyEndpoints(-_telemetryReadyEndpointCount); + SharpLinkTelemetry.AddClientDrainingEndpoints(-_telemetryDrainingEndpointCount); + _telemetryActiveEndpointCount = 0; + _telemetryReadyEndpointCount = 0; + _telemetryDrainingEndpointCount = 0; + } + + if (Interlocked.Exchange(ref _resolverDisposed, 1) == 0) + { + try { await _resolver.DisposeAsync().ConfigureAwait(false); } + catch (Exception exception) { cleanupFailures.Add(exception); } + } + + var stopping = CreateConnectionClosedException("Client is stopping."); + for (var index = 0; index < connections.Length; index++) + { + connections[index].Fail(stopping); + try { await DisposeConnectionAsync(connections[index]).ConfigureAwait(false); } + catch (Exception exception) { cleanupFailures.Add(exception); } + } + for (var index = 0; index < workers.Length; index++) + { + var worker = workers[index]; + try { await worker.ConfigureAwait(false); } + catch (OperationCanceledException) when (_client._shutdownCts.IsCancellationRequested) { } + catch (Exception) when (ReferenceEquals(worker, initialConnectTask)) { } + catch (Exception exception) { cleanupFailures.Add(exception); } + } + await WaitForInitialDialsAsync(cleanupFailures).ConfigureAwait(false); + for (var index = 0; index < factories.Length; index++) + { + try { await DisposeFactoryQuietlyAsync(factories[index]).ConfigureAwait(false); } + catch (Exception exception) { cleanupFailures.Add(exception); } + } + ThrowCleanupFailures(cleanupFailures); + } + + private async Task WaitForInitialDialsAsync(List cleanupFailures) + { + while (true) + { + Task[] pending; + lock (_gate) + pending = [.. _initialDialTasks.Values + .SelectMany(static attempts => attempts) + .Where(static task => !task.IsCompleted)]; + if (pending.Length == 0) + return; + for (var index = 0; index < pending.Length; index++) + { + try { await pending[index].ConfigureAwait(false); } + catch (OperationCanceledException) when (_client._shutdownCts.IsCancellationRequested) { } + catch (Exception exception) { cleanupFailures.Add(exception); } + } + } + } + + private static void ThrowCleanupFailures(List failures) + { + if (failures.Count == 0) + return; + if (failures.Count == 1) + System.Runtime.ExceptionServices.ExceptionDispatchInfo.Capture(failures[0]).Throw(); + throw new AggregateException(failures); + } + + private static bool SameGeneration(SharpLinkEndpoint left, SharpLinkEndpoint right) + => Equals(left.Address, right.Address) && StringComparer.Ordinal.Equals(left.Authority, right.Authority); + + private bool HasUniqueFactoryOwnershipLocked(IEnumerable created) + { + var factories = GetOwnedFactoriesLocked(); + foreach (var state in created) + { + if (!factories.Add(state.Configuration.TransportFactory)) + return false; + } + return true; + } + + private HashSet GetOwnedFactoriesLocked() + { + var factories = new HashSet(ReferenceEqualityComparer.Instance); + for (var index = 0; index < _allStates.Count; index++) + factories.Add(_allStates[index].Configuration.TransportFactory); + return factories; + } + + private static bool HasSameMembership(EndpointState[] left, EndpointState[] right) + { + if (left.Length != right.Length) + return false; + for (var index = 0; index < left.Length; index++) + if (!ReferenceEquals(left[index], right[index])) + return false; + return true; + } + + private static async Task DisposeConnectionAsync(ClientConnection connection) + { + try { await connection.DisposeAsync().ConfigureAwait(false); } + catch (Exception exception) when (exception is IOException or SocketException or ObjectDisposedException) { } + } + + private static async Task DisposeFactoryQuietlyAsync(IClientTransportFactory factory) + { + try { await factory.DisposeAsync().ConfigureAwait(false); } + catch (Exception exception) when (exception is IOException or SocketException or ObjectDisposedException) { } + } + + private async Task DisposeCreatedFactoriesAsync( + IEnumerable states, + ISet? preservedFactories = null) + { + var factories = new HashSet(ReferenceEqualityComparer.Instance); + foreach (var state in states) + { + var factory = state.Configuration.TransportFactory; + if (factories.Add(factory) && (preservedFactories is null || !preservedFactories.Contains(factory))) + { + try { await DisposeFactoryQuietlyAsync(factory).ConfigureAwait(false); } + catch (Exception exception) + { + LogClientBackgroundLoopUnhandledException( + _client._logger, + nameof(DisposeCreatedFactoriesAsync), + exception); + } + } + } + } + + private sealed class EndpointState + { + private readonly Func _readyConnectionCountProvider; + private readonly Func _activeCallCountProvider; + private ClientConnection[] _readyConnections = []; + + public EndpointState(StaticEndpointConfiguration configuration, long generation) + { + Configuration = configuration; + Generation = generation; + _readyConnectionCountProvider = GetReadyConnectionCount; + _activeCallCountProvider = GetActiveCallCount; + } + + public StaticEndpointConfiguration Configuration { get; } + public long Generation { get; } + public HashSet Connections { get; } = []; + public ClientConnection[] ReadyConnections => Volatile.Read(ref _readyConnections); + public Func ReadyConnectionCountProvider => _readyConnectionCountProvider; + public Func ActiveCallCountProvider => _activeCallCountProvider; + public int ConnectingCount { get; set; } + public int InitialDialReservations { get; set; } + public int ReconnectDelayMilliseconds { get; set; } = 100; + public bool Retiring { get; set; } + public bool FactoryReleased { get; set; } + public Task? ReconnectTask { get; set; } + public Task? ExpansionTask { get; set; } + public int NonRetiringConnectionCount + { + get + { + var count = 0; + foreach (var connection in Connections) + if (connection.State == ClientConnectionState.Ready) + count++; + return count; + } + } + + public int ActiveCallCount => GetActiveCallCount(); + + private int GetReadyConnectionCount() => ReadyConnections.Length; + + private int GetActiveCallCount() + { + var connections = ReadyConnections; + var count = 0; + for (var index = 0; index < connections.Length; index++) + count += connections[index].ActiveCallCount; + return count; + } + + public void PublishReadyConnections() + { + var ready = new List(Connections.Count); + foreach (var connection in Connections) + if (connection.CanAcceptCalls) + ready.Add(connection); + Volatile.Write(ref _readyConnections, ready.ToArray()); + } + } + + private sealed class EndpointSelectionSnapshot( + EndpointState[] endpoints, + SharpLinkEndpointCandidate[] candidates) + { + public static readonly EndpointSelectionSnapshot Empty = new([], []); + public EndpointState[] Endpoints { get; } = endpoints; + public SharpLinkEndpointCandidate[] Candidates { get; } = candidates; + } + } +} diff --git a/src/SharpLink.Client/SharpLinkClient.EndpointCluster.cs b/src/SharpLink.Client/SharpLinkClient.EndpointCluster.cs new file mode 100644 index 0000000..0376204 --- /dev/null +++ b/src/SharpLink.Client/SharpLinkClient.EndpointCluster.cs @@ -0,0 +1,52 @@ +namespace SharpLink.Client; + +internal sealed partial class SharpLinkClient +{ + /// Provides the cluster-specific portion of client connection routing. + private interface IEndpointClusterRuntime + { + int ReadyConnectionCount { get; } + int PendingCallCount { get; } + int ActiveCallCount { get; } + int ActiveStreamCount { get; } + ValueTask ConnectAsync(CancellationToken cancellationToken); + ClientConnection GetReadyConnection( + RpcMethodDescriptor? method, + EndpointRetrySelectionState? retrySelection, + AttemptOutcomeState? attemptOutcome); + bool TryGetEndpointCandidate(ClientConnection connection, out SharpLinkEndpointCandidate candidate); + void MarkConnectionDraining(ClientConnection connection); + void RetireDrainingConnectionIfIdle(ClientConnection connection); + ValueTask StopAsync(); + } + + /// Keeps the zero-allocation per-logical-call endpoint exclusion mask for retry attempts. + private sealed class EndpointRetrySelectionState + { + private object? _snapshot; + private ulong _excludedMask; + + public ulong GetExcludedMask(object snapshot, int count) + { + if (!ReferenceEquals(_snapshot, snapshot)) + { + _snapshot = snapshot; + _excludedMask = 0; + } + var availableMask = count == 64 ? ulong.MaxValue : (1UL << count) - 1; + if ((_excludedMask & availableMask) == availableMask) + _excludedMask = 0; + return _excludedMask; + } + + public void Exclude(object snapshot, int index) + { + if (!ReferenceEquals(_snapshot, snapshot)) + { + _snapshot = snapshot; + _excludedMask = 0; + } + _excludedMask |= 1UL << index; + } + } +} diff --git a/src/SharpLink.Client/SharpLinkClient.Interceptors.cs b/src/SharpLink.Client/SharpLinkClient.Interceptors.cs index 5c17bab..8936a4b 100644 --- a/src/SharpLink.Client/SharpLinkClient.Interceptors.cs +++ b/src/SharpLink.Client/SharpLinkClient.Interceptors.cs @@ -193,9 +193,8 @@ protected override async ValueTask InvokeTermin { var control = Client.ResolveCallControl( context.Options, true, _method.HasMethodTimeout, _method.MethodTimeout); - var response = await Client.InvokeUnaryCoreAsync( - _method.ContractId, _method.MethodId, _method.HasResponsePayload, - _request, _requestCodec, _responseCodec, control, context.CancellationToken).ConfigureAwait(false); + var response = await Client.InvokeUnaryWithOptionalRetryAsync( + _method, _request, _requestCodec, _responseCodec, control, context.CancellationToken).ConfigureAwait(false); return new SharpLinkClientInvocationResult(response); } } @@ -233,7 +232,7 @@ protected override async ValueTask InvokeTermin var control = Client.ResolveCallControl( context.Options, false, _method.HasMethodTimeout, _method.MethodTimeout); await Client.InvokeOneWayCoreAsync( - _method.ContractId, _method.MethodId, _method.HasClientStreams, + _method, _request, _requestCodec, _streams, control, context.CancellationToken).ConfigureAwait(false); return default; } @@ -275,7 +274,7 @@ protected override async ValueTask InvokeTermin var control = Client.ResolveCallControl( context.Options, false, _method.HasMethodTimeout, _method.MethodTimeout); var response = await Client.InvokeClientStreamingCoreAsync( - _method.ContractId, _method.MethodId, _method.HasResponsePayload, + _method, _request, _requestCodec, _responseCodec, _streams, control, context.CancellationToken).ConfigureAwait(false); return new SharpLinkClientInvocationResult(response); diff --git a/src/SharpLink.Client/SharpLinkClient.Invokers.cs b/src/SharpLink.Client/SharpLinkClient.Invokers.cs index 50d9dd6..b83ce33 100644 --- a/src/SharpLink.Client/SharpLinkClient.Invokers.cs +++ b/src/SharpLink.Client/SharpLinkClient.Invokers.cs @@ -28,15 +28,8 @@ public ValueTask InvokeUnaryAsync( includeClientDefault: true, method.HasMethodTimeout, method.MethodTimeout); - return InvokeUnaryCoreAsync( - method.ContractId, - method.MethodId, - method.HasResponsePayload, - request, - requestCodec, - responseCodec, - control, - cancellationToken); + return InvokeUnaryWithOptionalRetryAsync( + method, request, requestCodec, responseCodec, control, cancellationToken); } public ValueTask InvokeOneWayAsync( @@ -66,9 +59,7 @@ public ValueTask InvokeOneWayAsync( method.HasMethodTimeout, method.MethodTimeout); return InvokeOneWayCoreAsync( - method.ContractId, - method.MethodId, - method.HasClientStreams, + method, request, requestCodec, streams, @@ -105,9 +96,7 @@ public ValueTask InvokeClientStreamingAsync InvokeServerStreamingCore.Rent(cancellationToken, responseCodec); TrackBackgroundTask(StartServerStreamingInvokerAsync( dispatcher, - method.ContractId, - method.MethodId, + method, request, requestCodec, control, @@ -208,8 +196,7 @@ private IAsyncEnumerable InvokeDuplexStreamingCore.Rent(cancellationToken, responseCodec); TrackBackgroundTask(StartDuplexStreamingInvokerAsync( dispatcher, - method.ContractId, - method.MethodId, + method, request, requestCodec, streams, @@ -219,9 +206,7 @@ private IAsyncEnumerable InvokeDuplexStreamingCore InvokeUnaryCoreAsync( - long contractId, - long methodId, - bool hasResponsePayload, + RpcMethodDescriptor method, TRequest request, IRpcCodec requestCodec, IRpcCodec responseCodec, @@ -231,9 +216,7 @@ private ValueTask InvokeUnaryCoreAsync( if (control.WaitForReady) { return InvokeUnaryWaitForReadyAsync( - contractId, - methodId, - hasResponsePayload, + method, request, requestCodec, responseCodec, @@ -241,21 +224,25 @@ private ValueTask InvokeUnaryCoreAsync( cancellationToken); } + var outcome = _endpointAdmissionPolicy is null ? null : new AttemptOutcomeState(this, method); + if (outcome is null) + SharpLinkTelemetry.RecordClientAttempt(); try { - var connection = GetReadyConnection(); + var connection = GetReadyConnection(method, retrySelection: null, outcome); var operation = connection.PendingCalls.Rent( responseCodec, PendingCallKind.Unary, control.DeadlineTimestamp, cancellationToken, - out var requestId); + out var requestId, + outcome); return StartUnaryCall( connection, - contractId, - methodId, + method.ContractId, + method.MethodId, requestId, - hasResponsePayload, + method.HasResponsePayload, request, requestCodec, operation, @@ -264,42 +251,55 @@ private ValueTask InvokeUnaryCoreAsync( } catch (Exception exception) { + outcome?.CompleteLocalFailure(exception); return ValueTask.FromException(exception); } } private async ValueTask InvokeUnaryWaitForReadyAsync( - long contractId, - long methodId, - bool hasResponsePayload, + RpcMethodDescriptor method, TRequest request, IRpcCodec requestCodec, IRpcCodec responseCodec, ResolvedCallControl control, CancellationToken cancellationToken) { - var connection = await GetReadyConnectionAsync( - waitForReady: true, - control.Deadline, - cancellationToken).ConfigureAwait(false); - var lease = await connection.PendingCalls.RentAsync( - responseCodec, - PendingCallKind.Unary, - control.DeadlineTimestamp, - waitForSlot: true, - control.Deadline, - cancellationToken).ConfigureAwait(false); - return await StartUnaryCall( - connection, - contractId, - methodId, - lease.Id, - hasResponsePayload, - request, - requestCodec, - lease.Operation, - control, - cancellationToken).ConfigureAwait(false); + var outcome = _endpointAdmissionPolicy is null ? null : new AttemptOutcomeState(this, method); + if (outcome is null) + SharpLinkTelemetry.RecordClientAttempt(); + try + { + var connection = await GetReadyConnectionAsync( + waitForReady: true, + control.Deadline, + cancellationToken, + method, + outcome).ConfigureAwait(false); + var lease = await connection.PendingCalls.RentAsync( + responseCodec, + PendingCallKind.Unary, + control.DeadlineTimestamp, + waitForSlot: true, + control.Deadline, + cancellationToken, + outcome).ConfigureAwait(false); + return await StartUnaryCall( + connection, + method.ContractId, + method.MethodId, + lease.Id, + method.HasResponsePayload, + request, + requestCodec, + lease.Operation, + control, + cancellationToken).ConfigureAwait(false); + } + catch (Exception exception) + { + outcome?.CompleteLocalFailure(exception); + throw; + } } private ValueTask StartUnaryCall( @@ -348,9 +348,7 @@ private ValueTask StartUnaryCall( } private async ValueTask InvokeOneWayCoreAsync( - long contractId, - long methodId, - bool hasClientStreams, + RpcMethodDescriptor method, TRequest request, IRpcCodec requestCodec, TStreams streams, @@ -358,49 +356,76 @@ private async ValueTask InvokeOneWayCoreAsync( CancellationToken cancellationToken) where TStreams : struct, IRpcClientStreamWriter { + var outcome = _endpointAdmissionPolicy is null ? null : new AttemptOutcomeState(this, method); + if (outcome is null) + SharpLinkTelemetry.RecordClientAttempt(); var connection = control.WaitForReady ? await GetReadyConnectionAsync( waitForReady: true, control.Deadline, - cancellationToken).ConfigureAwait(false) - : GetReadyConnection(); + cancellationToken, + method, + outcome).ConfigureAwait(false) + : GetReadyConnection(method, retrySelection: null, outcome); var flags = ProtocolV2FrameFlags.OneWay; - if (hasClientStreams && (cancellationToken.CanBeCanceled || control.Deadline is not null)) + if (method.HasClientStreams && (cancellationToken.CanBeCanceled || control.Deadline is not null)) flags |= ProtocolV2FrameFlags.Cancellable; - var oneWayStreamLease = hasClientStreams - ? connection.PendingCalls.RegisterOneWayClientStream( - control.DeadlineTimestamp, - cancellationToken) - : default; - var requestId = hasClientStreams - ? oneWayStreamLease.Id - : connection.PendingCalls.AllocateRequestId(); - var streamCancellationToken = hasClientStreams + PendingRequestLease oneWayStreamLease = default; + long requestId; + try + { + if (method.HasClientStreams) + { + oneWayStreamLease = connection.PendingCalls.RegisterOneWayClientStream( + control.DeadlineTimestamp, + cancellationToken, + outcome); + requestId = oneWayStreamLease.Id; + } + else + { + requestId = connection.PendingCalls.AllocateRequestId(); + } + } + catch (Exception exception) + { + outcome?.CompleteLocalFailure(exception); + throw; + } + var streamCancellationToken = method.HasClientStreams ? connection.PendingCalls.GetProducerCancellationToken(requestId) : CancellationToken.None; - if (hasClientStreams && !connection.PendingCalls.Contains(requestId)) + if (method.HasClientStreams && !connection.PendingCalls.Contains(requestId)) { cancellationToken.ThrowIfCancellationRequested(); - throw CreateDeadlineExceededException(); + var exception = CreateDeadlineExceededException(); + outcome?.CompleteLocalFailure(exception); + throw exception; + } + if (!method.HasClientStreams && !connection.TryBeginUntrackedCall()) + { + var exception = new SharpLinkException( + SharpLinkErrorCode.Unavailable, + "The selected connection is draining."); + outcome?.CompleteWithoutPending(PendingCallCompletionReason.ConnectionClosed, exception); + throw exception; } - if (!hasClientStreams && !connection.TryBeginUntrackedCall()) - throw new SharpLinkException(SharpLinkErrorCode.Unavailable, "The selected connection is draining."); try { try { SendRpcCall( connection.Session, - contractId, - methodId, + method.ContractId, + method.MethodId, requestId, flags, request, requestCodec, control.Deadline, control.Metadata); - if (hasClientStreams) + if (method.HasClientStreams) { await streams.WriteAsync(connection, requestId, streamCancellationToken).ConfigureAwait(false); connection.PendingCalls.TryComplete( @@ -408,10 +433,14 @@ private async ValueTask InvokeOneWayCoreAsync( PendingCallCompletionReason.LocalStreamComplete); _ = await oneWayStreamLease.Operation.AsValueTask().ConfigureAwait(false); } + else + { + outcome?.CompleteWithoutPending(PendingCallCompletionReason.LocalStreamComplete); + } } catch (Exception exception) { - if (hasClientStreams) + if (method.HasClientStreams) { connection.PendingCalls.TryComplete( requestId, @@ -419,20 +448,22 @@ private async ValueTask InvokeOneWayCoreAsync( exception); _ = await oneWayStreamLease.Operation.AsValueTask().ConfigureAwait(false); } + else + { + outcome?.CompleteWithoutPending(PendingCallCompletionReason.SendFailure, exception); + } throw; } } finally { - if (!hasClientStreams) + if (!method.HasClientStreams) connection.EndUntrackedCall(); } } private async ValueTask InvokeClientStreamingCoreAsync( - long contractId, - long methodId, - bool hasResponsePayload, + RpcMethodDescriptor method, TRequest request, IRpcCodec requestCodec, IRpcCodec responseCodec, @@ -442,36 +473,51 @@ private async ValueTask InvokeClientStreamingCoreAsync operation; - if (!control.WaitForReady) + try { - connection = GetReadyConnection(); - operation = connection.PendingCalls.Rent( - responseCodec, - PendingCallKind.ClientStreaming, - control.DeadlineTimestamp, - cancellationToken, - out requestId); + if (!control.WaitForReady) + { + connection = GetReadyConnection(method, retrySelection: null, outcome); + operation = connection.PendingCalls.Rent( + responseCodec, + PendingCallKind.ClientStreaming, + control.DeadlineTimestamp, + cancellationToken, + out requestId, + outcome); + } + else + { + connection = await GetReadyConnectionAsync( + waitForReady: true, + control.Deadline, + cancellationToken, + method, + outcome).ConfigureAwait(false); + var lease = await connection.PendingCalls.RentAsync( + responseCodec, + PendingCallKind.ClientStreaming, + control.DeadlineTimestamp, + waitForSlot: true, + control.Deadline, + cancellationToken, + outcome).ConfigureAwait(false); + requestId = lease.Id; + operation = lease.Operation; + } } - else + catch (Exception exception) { - connection = await GetReadyConnectionAsync( - waitForReady: true, - control.Deadline, - cancellationToken).ConfigureAwait(false); - var lease = await connection.PendingCalls.RentAsync( - responseCodec, - PendingCallKind.ClientStreaming, - control.DeadlineTimestamp, - waitForSlot: true, - control.Deadline, - cancellationToken).ConfigureAwait(false); - requestId = lease.Id; - operation = lease.Operation; + outcome?.CompleteLocalFailure(exception); + throw; } - var flags = hasResponsePayload + var flags = method.HasResponsePayload ? ProtocolV2FrameFlags.HasReturn | ProtocolV2FrameFlags.Cancellable : ProtocolV2FrameFlags.Cancellable; var streamCancellationToken = connection.PendingCalls.GetProducerCancellationToken(requestId); @@ -483,8 +529,8 @@ private async ValueTask InvokeClientStreamingCoreAsync( private async Task StartServerStreamingInvokerAsync( PooledAsyncStreamDispatcher dispatcher, - long contractId, - long methodId, + RpcMethodDescriptor method, TRequest request, IRpcCodec requestCodec, ResolvedCallControl control, @@ -558,14 +603,15 @@ private async Task StartServerStreamingInvokerAsync( var registration = await PrepareGeneratedServerStreamAsync( dispatcher, PendingCallKind.ServerStreaming, + method, control, cancellationToken).ConfigureAwait(false); connection = registration.Connection; requestId = registration.RequestId; SendRpcCall( connection.Session, - contractId, - methodId, + method.ContractId, + method.MethodId, requestId, cancellationToken.CanBeCanceled || control.Deadline is not null ? ProtocolV2FrameFlags.Cancellable @@ -587,8 +633,7 @@ private async Task StartServerStreamingInvokerAsync( private async Task StartDuplexStreamingInvokerAsync( PooledAsyncStreamDispatcher dispatcher, - long contractId, - long methodId, + RpcMethodDescriptor method, TRequest request, IRpcCodec requestCodec, TStreams streams, @@ -606,6 +651,7 @@ private async Task StartDuplexStreamingInvokerAsync PrepareGeneratedServerStreamAsync( PooledAsyncStreamDispatcher dispatcher, PendingCallKind kind, + RpcMethodDescriptor method, ResolvedCallControl control, CancellationToken cancellationToken) { - var connection = await GetReadyConnectionAsync( - control.WaitForReady, - control.Deadline, - cancellationToken).ConfigureAwait(false); - var requestId = connection.PendingCalls.RegisterStream( - kind, - dispatcher, - control.DeadlineTimestamp, - cancellationToken); - if (!connection.PendingCalls.Contains(requestId)) + var outcome = _endpointAdmissionPolicy is null ? null : new AttemptOutcomeState(this, method); + if (outcome is null) + SharpLinkTelemetry.RecordClientAttempt(); + ClientConnection? connection = null; + var requestId = 0L; + try { - cancellationToken.ThrowIfCancellationRequested(); - throw CreateDeadlineExceededException(); + connection = await GetReadyConnectionAsync( + control.WaitForReady, + control.Deadline, + cancellationToken, + method, + outcome).ConfigureAwait(false); + requestId = connection.PendingCalls.RegisterStream( + kind, + dispatcher, + control.DeadlineTimestamp, + cancellationToken, + outcome); + if (!connection.PendingCalls.Contains(requestId)) + { + cancellationToken.ThrowIfCancellationRequested(); + throw CreateDeadlineExceededException(); + } + dispatcher.SetConsumerAbandonedCallback(connection.ConsumerAbandonedCallback, requestId); + connection.Session.StreamManager.Register(requestId, 0, dispatcher); + return new StreamCallRegistration(connection, requestId); + } + catch (Exception exception) + { + if (connection is not null && requestId != 0) + { + connection.PendingCalls.TryComplete( + requestId, + PendingCallCompletionReason.SendFailure, + exception); + } + else + { + outcome?.CompleteLocalFailure(exception); + } + throw; } - dispatcher.SetConsumerAbandonedCallback(connection.ConsumerAbandonedCallback, requestId); - connection.Session.StreamManager.Register(requestId, 0, dispatcher); - return new StreamCallRegistration(connection, requestId); } private void CompleteFailedGeneratedStream( diff --git a/src/SharpLink.Client/SharpLinkClient.Lifecycle.cs b/src/SharpLink.Client/SharpLinkClient.Lifecycle.cs index 14da016..7fedd1b 100644 --- a/src/SharpLink.Client/SharpLinkClient.Lifecycle.cs +++ b/src/SharpLink.Client/SharpLinkClient.Lifecycle.cs @@ -4,6 +4,9 @@ internal sealed partial class SharpLinkClient { public ValueTask ConnectAsync(CancellationToken cancellationToken = default) { + if (_cluster is not null) + return _cluster.ConnectAsync(cancellationToken); + Task connectTask; lock (_stateGate) { diff --git a/src/SharpLink.Client/SharpLinkClient.Retry.cs b/src/SharpLink.Client/SharpLinkClient.Retry.cs new file mode 100644 index 0000000..59badbf --- /dev/null +++ b/src/SharpLink.Client/SharpLinkClient.Retry.cs @@ -0,0 +1,293 @@ +namespace SharpLink.Client; + +internal sealed partial class SharpLinkClient +{ + private ValueTask InvokeUnaryWithOptionalRetryAsync( + RpcMethodDescriptor method, + TRequest request, + IRpcCodec requestCodec, + IRpcCodec responseCodec, + ResolvedCallControl control, + CancellationToken cancellationToken) + { + var options = _retryOptions; + if (options is null || method.Kind != RpcMethodKind.Unary || !method.IsIdempotent) + { + return InvokeUnaryCoreAsync( + method, + request, requestCodec, responseCodec, control, cancellationToken); + } + + return InvokeUnaryWithRetryAsync( + method, request, requestCodec, responseCodec, control, options, cancellationToken); + } + + private async ValueTask InvokeUnaryWithRetryAsync( + RpcMethodDescriptor method, + TRequest request, + IRpcCodec requestCodec, + IRpcCodec responseCodec, + ResolvedCallControl control, + SharpLinkRetryOptions options, + CancellationToken cancellationToken) + { + Exception? lastFailure = null; + var selection = new EndpointRetrySelectionState(); + for (var attempt = 1; attempt <= options.MaxAttempts; attempt++) + { + cancellationToken.ThrowIfCancellationRequested(); + var outcome = new AttemptOutcomeState(this, method); + var attemptScope = SharpLinkTelemetry.StartClientAttempt(method, attempt); + try + { + var response = await InvokeUnaryRetryAttemptAsync( + method, + method.ContractId, method.MethodId, method.HasResponsePayload, + request, requestCodec, responseCodec, control, selection, outcome, cancellationToken).ConfigureAwait(false); + attemptScope.Complete(); + return response; + } + catch (OperationCanceledException exception) + { + attemptScope.Complete(exception); + throw; + } + catch (Exception exception) when (exception is not OperationCanceledException) + { + attemptScope.Complete(exception); + lastFailure = exception; + if (attempt == options.MaxAttempts) + throw; + + var attemptOutcome = outcome.CreateRetryOutcome(exception); + var context = new SharpLinkRetryContext( + method, + attempt, + attemptOutcome.ErrorCode, + attemptOutcome.ResponseObserved, + attemptOutcome.Elapsed); + var decision = EvaluateRetryDecision(context, options); + if (!decision.ShouldRetry) + throw; + SharpLinkTelemetry.RecordClientRetry(); + if (decision.Delay < TimeSpan.Zero) + { + throw new SharpLinkException( + SharpLinkErrorCode.FailedPrecondition, + "The retry policy returned a negative delay."); + } + var delay = decision.Delay; + if (outcome.ShouldHonorAdmissionRetryAfter && + outcome.RetryAfter is { } admissionDelay && admissionDelay > delay) + delay = admissionDelay; + if (delay == TimeSpan.Zero) + { + if (control.Deadline is { } zeroDelayDeadline && DateTimeOffset.UtcNow >= zeroDelayDeadline) + throw CreateDeadlineExceededException(); + continue; + } + + if (control.Deadline is { } deadline && WouldReachDeadline(deadline, delay)) + throw CreateDeadlineExceededException(); + await DelayForRetryOrAdmissionAsync(delay, cancellationToken).ConfigureAwait(false); + } + } + + throw lastFailure ?? new SharpLinkException(SharpLinkErrorCode.Internal, "Retry exhausted without an attempt outcome."); + } + + private SharpLinkRetryDecision EvaluateRetryDecision( + in SharpLinkRetryContext context, + SharpLinkRetryOptions options) + { + if (_retryPolicy is not null) + { + try + { + return _retryPolicy.Evaluate(context); + } + catch (Exception exception) + { + throw new SharpLinkException( + SharpLinkErrorCode.FailedPrecondition, + "The retry policy failed.", + exception); + } + } + + var retryable = context.ErrorCode is SharpLinkErrorCode.Unavailable or SharpLinkErrorCode.ConnectionClosed; + return retryable + ? new SharpLinkRetryDecision(true, GetRetryDelay(context.Attempt, options)) + : default; + } + + private static TimeSpan GetRetryDelay(int completedAttempt, SharpLinkRetryOptions options) + { + var ticks = options.InitialBackoff.Ticks; + for (var index = 1; index < completedAttempt && ticks < options.MaxBackoff.Ticks; index++) + ticks = Math.Min(ticks > long.MaxValue / 2 ? long.MaxValue : ticks * 2, options.MaxBackoff.Ticks); + if (ticks == 0 || options.JitterRatio == 0) + return TimeSpan.FromTicks(ticks); + + var multiplier = 1 - options.JitterRatio + Random.Shared.NextDouble() * options.JitterRatio * 2; + var jitteredTicks = ticks * multiplier; + var clampedTicks = jitteredTicks >= options.MaxBackoff.Ticks + ? options.MaxBackoff.Ticks + : (long)jitteredTicks; + return TimeSpan.FromTicks(clampedTicks); + } + + private ValueTask InvokeUnaryRetryAttemptAsync( + RpcMethodDescriptor method, + long contractId, + long methodId, + bool hasResponsePayload, + TRequest request, + IRpcCodec requestCodec, + IRpcCodec responseCodec, + ResolvedCallControl control, + EndpointRetrySelectionState selection, + AttemptOutcomeState outcome, + CancellationToken cancellationToken) + { + if (control.WaitForReady) + { + return InvokeUnaryRetryWaitForReadyAsync( + contractId, methodId, hasResponsePayload, request, requestCodec, responseCodec, + method, control, selection, outcome, cancellationToken); + } + + try + { + var connection = GetReadyConnection(method, selection, outcome); + outcome.SetConnection(connection); + var operation = connection.PendingCalls.Rent( + responseCodec, + PendingCallKind.Unary, + control.DeadlineTimestamp, + cancellationToken, + out var requestId, + outcome); + return StartUnaryCall( + connection, + contractId, + methodId, + requestId, + hasResponsePayload, + request, + requestCodec, + operation, + control, + cancellationToken); + } + catch (Exception exception) + { + outcome.CompleteLocalFailure(exception); + return ValueTask.FromException(exception); + } + } + + private async ValueTask InvokeUnaryRetryWaitForReadyAsync( + long contractId, + long methodId, + bool hasResponsePayload, + TRequest request, + IRpcCodec requestCodec, + IRpcCodec responseCodec, + RpcMethodDescriptor method, + ResolvedCallControl control, + EndpointRetrySelectionState selection, + AttemptOutcomeState outcome, + CancellationToken cancellationToken) + { + try + { + var connection = await GetReadyConnectionForRetryAsync( + method, selection, outcome, control.Deadline, cancellationToken).ConfigureAwait(false); + outcome.SetConnection(connection); + var lease = await connection.PendingCalls.RentAsync( + responseCodec, + PendingCallKind.Unary, + control.DeadlineTimestamp, + waitForSlot: true, + control.Deadline, + cancellationToken, + outcome).ConfigureAwait(false); + return await StartUnaryCall( + connection, + contractId, + methodId, + lease.Id, + hasResponsePayload, + request, + requestCodec, + lease.Operation, + control, + cancellationToken).ConfigureAwait(false); + } + catch (Exception exception) + { + if (!outcome.HasCompletion) + outcome.CompleteLocalFailure(exception); + throw; + } + } + + private async ValueTask GetReadyConnectionForRetryAsync( + RpcMethodDescriptor method, + EndpointRetrySelectionState selection, + AttemptOutcomeState outcome, + DateTimeOffset? deadline, + CancellationToken cancellationToken) + { + while (true) + { + outcome.BeginAdmissionSelection(); + try + { + return GetReadyConnection(method, selection, outcome); + } + catch (SharpLinkException exception) when (exception.Code == SharpLinkErrorCode.Unavailable) + { + if (State == SharpLinkConnectionState.Stopped || _shutdownCts.IsCancellationRequested) + throw CreateConnectionClosedException("Client has stopped."); + + if (outcome.ShouldHonorAdmissionRetryAfter) + { + if (outcome.RetryAfter is not { } retryAfter) + throw; + var delay = retryAfter > TimeSpan.Zero + ? retryAfter + : TimeSpan.FromMilliseconds(1); + if (deadline is { } retryDeadline && WouldReachDeadline(retryDeadline, delay)) + throw CreateDeadlineExceededException(); + await DelayForRetryOrAdmissionAsync(delay, cancellationToken).ConfigureAwait(false); + continue; + } + + if (outcome.HasAdmissionRejection && !outcome.HasAdmissionGrant) + throw; + + var signal = Volatile.Read(ref _readySignal).Task; + if (deadline is not { } absoluteDeadline) + { + await signal.WaitAsync(cancellationToken).ConfigureAwait(false); + continue; + } + + var remaining = absoluteDeadline - DateTimeOffset.UtcNow; + if (remaining <= TimeSpan.Zero) + throw CreateDeadlineExceededException(); + try + { + await signal.WaitAsync(remaining, cancellationToken).ConfigureAwait(false); + } + catch (TimeoutException) + { + throw CreateDeadlineExceededException(); + } + } + } + } + +} diff --git a/src/SharpLink.Client/SharpLinkClient.RpcChannel.cs b/src/SharpLink.Client/SharpLinkClient.RpcChannel.cs index 362c314..b469d04 100644 --- a/src/SharpLink.Client/SharpLinkClient.RpcChannel.cs +++ b/src/SharpLink.Client/SharpLinkClient.RpcChannel.cs @@ -234,6 +234,9 @@ private async Task ReconnectLoopAsync() private ClientConnection GetReadyConnection() { + if (_cluster is not null) + return _cluster.GetReadyConnection(method: null, retrySelection: null, attemptOutcome: null); + var connections = Volatile.Read(ref _readyConnections); if (!_shutdownCts.IsCancellationRequested && connections.Length != 0) { @@ -263,6 +266,40 @@ private ClientConnection GetReadyConnection() throw new SharpLinkException(SharpLinkErrorCode.Unavailable, "No SharpLink connection is ready."); } + private ClientConnection GetReadyConnection( + RpcMethodDescriptor method, + EndpointRetrySelectionState? retrySelection, + AttemptOutcomeState? attemptOutcome) + { + if (_cluster is not null) + return _cluster.GetReadyConnection(method, retrySelection, attemptOutcome); + + if (attemptOutcome is null || _fixedEndpoint is null) + return GetReadyConnection(); + + if (ReadyConnectionCount == 0) + return GetReadyConnection(); + + var candidate = new SharpLinkEndpointCandidate( + _fixedEndpoint, + ReadyConnectionCount, + ActiveClientCallCount, + generation: 0); + if (!attemptOutcome.TryAcquire(candidate)) + throw new SharpLinkException(SharpLinkErrorCode.Unavailable, "The configured endpoint admission policy rejected the endpoint."); + try + { + var connection = GetReadyConnection(); + attemptOutcome.SetConnection(connection); + return connection; + } + catch (Exception exception) + { + attemptOutcome.CompleteLocalFailure(exception); + throw; + } + } + internal static ClientConnection SelectLeastLoaded( ClientConnection[] connections, int first, @@ -293,7 +330,14 @@ private bool RemoveReadyConnection(ClientConnection connection) private void MarkConnectionDraining(ClientConnection connection) { - connection.MarkDraining(); + if (!connection.MarkDraining()) + return; + ReportGoAwayToCircuitBreaker(connection); + if (_cluster is not null) + { + _cluster.MarkConnectionDraining(connection); + return; + } lock (_poolGate) PublishReadySnapshotLocked(); @@ -309,8 +353,35 @@ private void MarkConnectionDraining(ClientConnection connection) EnsureReconnectLoop(); } + private void ReportGoAwayToCircuitBreaker(ClientConnection connection) + { + if (_endpointAdmissionPolicy is not SharpLinkCircuitBreaker breaker) + return; + + if (_cluster?.TryGetEndpointCandidate(connection, out var clusterEndpoint) == true) + { + breaker.ReportInfrastructureFailure(clusterEndpoint); + return; + } + + if (_fixedEndpoint is { } fixedEndpoint) + { + var endpoint = new SharpLinkEndpointCandidate( + fixedEndpoint, + ReadyConnectionCount, + ActiveClientCallCount, + generation: 0); + breaker.ReportInfrastructureFailure(endpoint); + } + } + internal void RetireDrainingConnectionIfIdle(ClientConnection connection) { + if (_cluster is not null) + { + _cluster.RetireDrainingConnectionIfIdle(connection); + return; + } if (connection.State != ClientConnectionState.Draining || connection.ActiveCallCount != 0 || !RemoveReadyConnection(connection)) diff --git a/src/SharpLink.Client/SharpLinkClient.StaticCluster.cs b/src/SharpLink.Client/SharpLinkClient.StaticCluster.cs new file mode 100644 index 0000000..37f995f --- /dev/null +++ b/src/SharpLink.Client/SharpLinkClient.StaticCluster.cs @@ -0,0 +1,945 @@ +namespace SharpLink.Client; + +internal sealed partial class SharpLinkClient +{ + /// + /// Owns static multi-endpoint transport state without introducing nested SharpLinkClient instances. + /// The enclosing client continues to own the proxy, interceptor, codec, pending-call and session pipeline. + /// + private sealed class StaticClusterRuntime : IEndpointClusterRuntime + { + private readonly SharpLinkClient _client; + private readonly SharpLinkClusterOptions _options; + private readonly SharpLinkLoadBalancingStrategy _strategy; + private readonly ISharpLinkEndpointSelector? _selector; + private readonly EndpointState[] _endpoints; + private readonly Lock _gate = new(); + private readonly HashSet _retiringConnections = []; + private readonly HashSet _initialDialTasks = []; + private EndpointState[] _readyEndpoints = []; + private EndpointSelectionSnapshot _selectionSnapshot = EndpointSelectionSnapshot.Empty; + private Task? _connectTask; + private Task? _stopTask; + private int _roundRobinCursor; + private int _leastPendingCursor; + private int _reconnectCursor; + private int _initialDialReservations; + private int _initialConnectCoordinatorCount; + private int _telemetryReadyEndpointCount; + private int _stopping; + + private int TargetReadyEndpointCount => Math.Min(_options.MinReadyEndpoints, _endpoints.Length); + + public StaticClusterRuntime( + SharpLinkClient client, + StaticEndpointConfiguration[] configurations, + SharpLinkClusterOptions options, + SharpLinkLoadBalancingStrategy strategy, + ISharpLinkEndpointSelector? selector) + { + _client = client; + _options = options; + _strategy = strategy; + _selector = selector; + _endpoints = new EndpointState[configurations.Length]; + for (var index = 0; index < configurations.Length; index++) + _endpoints[index] = new EndpointState(configurations[index], index); + SharpLinkTelemetry.AddClientActiveEndpoints(_endpoints.Length); + } + + public int ReadyConnectionCount + { + get + { + var endpoints = Volatile.Read(ref _readyEndpoints); + var count = 0; + for (var index = 0; index < endpoints.Length; index++) + count += endpoints[index].ReadyConnections.Length; + return count; + } + } + + public int PendingCallCount => CountConnections(static connection => connection.PendingCalls.Count); + + public int ActiveCallCount => CountConnections(static connection => connection.ActiveCallCount); + + public int ActiveStreamCount => CountConnections(static connection => + ((StreamManager)connection.Session.StreamManager).ActiveStreamCount); + + public ValueTask ConnectAsync(CancellationToken cancellationToken) + { + Task task; + lock (_gate) + { + if (Volatile.Read(ref _stopping) != 0 || _client._shutdownCts.IsCancellationRequested) + return ValueTask.FromException(CreateConnectionClosedException("Client has stopped.")); + if (ReadyConnectionCount != 0) + return ValueTask.CompletedTask; + _client.TransitionTo(SharpLinkConnectionState.Connecting); + // A cluster initialization attempt belongs to the client, not to the first caller. + // Individual callers still observe their own cancellation through WaitAsync below. + if (_connectTask is null || _connectTask.IsFaulted || _connectTask.IsCanceled) + _connectTask = ConnectInitialAsync(_client._shutdownCts.Token); + else if (_connectTask.IsCompleted) + _connectTask = WaitForRecoveryAsync(); + task = _connectTask; + } + return cancellationToken.CanBeCanceled ? new ValueTask(task.WaitAsync(cancellationToken)) : new ValueTask(task); + } + + public ClientConnection GetReadyConnection( + RpcMethodDescriptor? method, + EndpointRetrySelectionState? retrySelection, + AttemptOutcomeState? attemptOutcome) + { + var snapshot = Volatile.Read(ref _selectionSnapshot); + var endpoints = snapshot.Endpoints; + if (endpoints.Length == 0) + { + SharpLinkTelemetry.RecordSelectionFailure("no_ready_endpoint"); + throw new SharpLinkException(SharpLinkErrorCode.Unavailable, "No SharpLink endpoint is ready."); + } + + var excluded = retrySelection?.GetExcludedMask(snapshot, endpoints.Length) ?? 0UL; + for (var attempt = 0; attempt < endpoints.Length; attempt++) + { + var selectedIndex = SelectEndpoint(endpoints, snapshot.Candidates, excluded); + if ((uint)selectedIndex >= (uint)endpoints.Length || (excluded & (1UL << selectedIndex)) != 0) + { + throw new SharpLinkException( + SharpLinkErrorCode.FailedPrecondition, + "The endpoint selector returned an unavailable candidate index."); + } + var candidate = snapshot.Candidates[selectedIndex]; + if (method is not null && attemptOutcome is not null && !attemptOutcome.TryAcquire(candidate)) + { + retrySelection?.Exclude(snapshot, selectedIndex); + excluded |= 1UL << selectedIndex; + continue; + } + var connection = SelectConnection(endpoints[selectedIndex]); + retrySelection?.Exclude(snapshot, selectedIndex); + if (connection is not null) + { + attemptOutcome?.SetConnection(connection); + if (connection.ActiveCallCount != 0) + EnsureExpansion(endpoints[selectedIndex]); + return connection; + } + attemptOutcome?.CompleteWithoutPending( + PendingCallCompletionReason.ConnectionClosed, + CreateConnectionClosedException("The selected static endpoint connection is no longer ready.")); + excluded |= 1UL << selectedIndex; + } + + SharpLinkTelemetry.RecordSelectionFailure("no_admitted_connection"); + throw new SharpLinkException(SharpLinkErrorCode.Unavailable, "No SharpLink endpoint connection is ready."); + } + + public void MarkConnectionDraining(ClientConnection connection) + { + ArgumentNullException.ThrowIfNull(connection); + EndpointState? endpoint; + var retireImmediately = false; + var forceClose = false; + lock (_gate) + { + endpoint = FindEndpointLocked(connection); + if (endpoint is null) + return; + connection.MarkDraining(); + if (connection.ActiveCallCount == 0) + { + endpoint.Connections.Remove(connection); + PublishReadySnapshotLocked(); + retireImmediately = true; + } + else if (_retiringConnections.Add(connection) && + _retiringConnections.Count > _options.MaxRetiringConnections) + { + _retiringConnections.Remove(connection); + endpoint.Connections.Remove(connection); + PublishReadySnapshotLocked(); + forceClose = true; + } + else + { + PublishReadySnapshotLocked(); + } + } + + if (forceClose) + { + connection.Fail(CreateConnectionClosedException( + "The static cluster retiring-connection budget was exhausted.")); + _client.TrackBackgroundTask(DisposeConnectionAsync(connection)); + } + else if (retireImmediately) + { + _client.TrackBackgroundTask(DisposeConnectionAsync(connection)); + } + + EnsureReconnect(endpoint); + if (ReadyConnectionCount == 0) + _client.TransitionTo(SharpLinkConnectionState.Reconnecting); + } + + public bool TryGetEndpointCandidate(ClientConnection connection, out SharpLinkEndpointCandidate candidate) + { + lock (_gate) + { + var endpoint = FindEndpointLocked(connection); + if (endpoint is null) + { + candidate = default; + return false; + } + + candidate = new SharpLinkEndpointCandidate( + endpoint.Configuration.Endpoint, + endpoint.ReadyConnectionCountProvider, + endpoint.ActiveCallCountProvider, + generation: 1); + return true; + } + } + + public void RetireDrainingConnectionIfIdle(ClientConnection connection) + { + if (connection.State != ClientConnectionState.Draining || connection.ActiveCallCount != 0) + return; + EndpointState? endpoint; + lock (_gate) + { + endpoint = FindEndpointLocked(connection); + if (endpoint is null) + return; + if (!endpoint.Connections.Remove(connection)) + return; + _retiringConnections.Remove(connection); + PublishReadySnapshotLocked(); + } + _client.TrackBackgroundTask(DisposeConnectionAsync(connection)); + EnsureReconnect(endpoint); + } + + public ValueTask StopAsync() + { + lock (_gate) + { + _stopTask ??= StopCoreAsync(); + return new ValueTask(_stopTask); + } + } + + private async Task ConnectInitialAsync(CancellationToken cancellationToken) + { + Interlocked.Increment(ref _initialConnectCoordinatorCount); + try + { + Exception? lastFailure = null; + var parallelism = Math.Min(Math.Min(TargetReadyEndpointCount, _endpoints.Length), 4); + var nextEndpoint = 0; + var remaining = new List>(parallelism); + while (nextEndpoint < parallelism) + { + var endpoint = _endpoints[nextEndpoint++]; + var startGate = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var attempt = TryConnectOneAfterInitialReservationAsync(endpoint, cancellationToken, startGate.Task); + TrackInitialDials([attempt]); + remaining.Add(attempt); + startGate.TrySetResult(); + } + + while (remaining.Count != 0) + { + var completed = await Task.WhenAny(remaining).ConfigureAwait(false); + remaining.Remove(completed); + lastFailure ??= await completed.ConfigureAwait(false); + if (ReadyConnectionCount != 0) + { + EnsureMinimumReadyEndpoints(); + return; + } + + if (nextEndpoint >= _endpoints.Length) + continue; + + var endpoint = _endpoints[nextEndpoint++]; + var startGate = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var attempt = TryConnectOneAfterInitialReservationAsync(endpoint, cancellationToken, startGate.Task); + TrackInitialDials([attempt]); + remaining.Add(attempt); + startGate.TrySetResult(); + } + + PublishClientReadiness(); + EnsureMinimumReadyEndpoints(); + if (ReadyConnectionCount == 0) + { + _client.TransitionTo(SharpLinkConnectionState.Faulted); + throw new SharpLinkException( + SharpLinkErrorCode.Unavailable, + "No static SharpLink endpoint could connect.", + lastFailure); + } + } + finally + { + if (Interlocked.Decrement(ref _initialConnectCoordinatorCount) == 0 && + Volatile.Read(ref _stopping) == 0 && !_client._shutdownCts.IsCancellationRequested) + { + // A sibling can release its initial reservation while this coordinator is still + // active. Its observer intentionally defers reconciliation, so perform the + // hand-off reconciliation once the final coordinator exits. + EnsureMinimumReadyEndpoints(); + } + } + } + + private async Task WaitForRecoveryAsync() + { + while (true) + { + if (Volatile.Read(ref _stopping) != 0 || _client._shutdownCts.IsCancellationRequested) + throw new OperationCanceledException(_client._shutdownCts.Token); + if (ReadyConnectionCount != 0) + return; + + EnsureMinimumReadyEndpoints(); + var signal = Volatile.Read(ref _client._readySignal).Task; + if (ReadyConnectionCount != 0) + return; + await signal.ConfigureAwait(false); + } + } + + private void TrackInitialDials(IEnumerable> attempts) + { + var tracked = attempts.ToArray(); + lock (_gate) + { + foreach (var attempt in tracked) + _initialDialTasks.Add(attempt); + _initialDialReservations += tracked.Length; + } + foreach (var attempt in tracked) + _client.TrackBackgroundTask(ObserveInitialDialAsync(attempt)); + } + + private async Task ObserveInitialDialAsync(Task attempt) + { + var shouldReconcile = false; + try + { + _ = await attempt.ConfigureAwait(false); + shouldReconcile = Volatile.Read(ref _stopping) == 0; + } + catch (OperationCanceledException) when (_client._shutdownCts.IsCancellationRequested) + { + } + finally + { + lock (_gate) + { + _initialDialTasks.Remove(attempt); + _initialDialReservations--; + } + } + + if (shouldReconcile && Volatile.Read(ref _initialConnectCoordinatorCount) == 0) + EnsureMinimumReadyEndpoints(); + } + + private async Task TryConnectOneAfterInitialReservationAsync( + EndpointState endpoint, + CancellationToken cancellationToken, + Task startGate) + { + await startGate.ConfigureAwait(false); + return await TryConnectOneAsync(endpoint, cancellationToken).ConfigureAwait(false); + } + + private async Task TryConnectOneAsync( + EndpointState endpoint, + CancellationToken cancellationToken) + { + try + { + await ConnectOneAsync(endpoint, cancellationToken).ConfigureAwait(false); + return null; + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested || _client._shutdownCts.IsCancellationRequested) + { + throw; + } + catch (Exception exception) + { + return exception; + } + } + + private async Task ConnectOneAsync(EndpointState endpoint, CancellationToken cancellationToken) + { + lock (_gate) + { + if (Volatile.Read(ref _stopping) != 0 || _client._shutdownCts.IsCancellationRequested) + throw CreateConnectionClosedException("Client has stopped."); + if (TotalConnectionsLocked() >= _options.MaxConnections || + endpoint.NonRetiringConnectionCount + endpoint.ConnectingCount >= _options.MaxConnectionsPerEndpoint) + { + return; + } + endpoint.ConnectingCount++; + } + + RpcSession? session = null; + ITransportConnection? transport = null; + ClientConnection? connection = null; + try + { + using var attemptCts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken, _client._shutdownCts.Token); + transport = await endpoint.Configuration.TransportFactory.ConnectAsync(attemptCts.Token).ConfigureAwait(false); + if (transport is ITransportSecurityInfo securityInfo) + LogTlsEstablished(_client._logger, securityInfo.Protocol, securityInfo.CipherSuite); + session = new RpcSession(transport, _client._rpcSessionFlushOptions); + transport = null; + session.SetTelemetrySide("client"); + session.BindRuntimeContext(_client._runtimeContext); + + using var handshakeTimeout = new CancellationTokenSource(_client._protocolOptions.HandshakeTimeout); + using var handshakeCts = CancellationTokenSource.CreateLinkedTokenSource(attemptCts.Token, handshakeTimeout.Token); + var handshakeException = await _client.ProcessHandshakeAsync(session, handshakeCts.Token).ConfigureAwait(false); + if (handshakeException is not null) + throw handshakeException; + + var sessionCts = CancellationTokenSource.CreateLinkedTokenSource(_client._shutdownCts.Token); + var createdConnection = new ClientConnection( + _client, + session, + sessionCts, + _client._protocolOptions.MaxPendingRequestsPerConnection, + _client._runtimeContext.Codecs, + endpoint.Configuration.Endpoint.Id); + connection = createdConnection; + createdConnection.Session.OnDisconnected += exception => HandleDisconnected( + endpoint, + createdConnection, + exception ?? CreateConnectionClosedException("Transport closed.")); + + lock (_gate) + { + if (Volatile.Read(ref _stopping) != 0 || _client._shutdownCts.IsCancellationRequested) + throw CreateConnectionClosedException("Client stopped while connecting."); + endpoint.Connections.Add(createdConnection); + PublishReadySnapshotLocked(); + } + session.NotifyConnected(); + _client.TrackBackgroundTask(_client.RunHeartbeatSendLoopAsync(createdConnection, sessionCts.Token)); + _client.TrackBackgroundTask(_client.RunProcessRequestLoopAsync(createdConnection, sessionCts.Token)); + session = null; + connection = null; + PublishClientReadiness(); + EnsureMinimumReadyEndpoints(); + } + finally + { + lock (_gate) + endpoint.ConnectingCount--; + if (transport is not null) + await transport.DisposeAsync().ConfigureAwait(false); + if (connection is not null) + await connection.DisposeAsync().ConfigureAwait(false); + else if (session is not null) + await session.DisposeAsync().ConfigureAwait(false); + } + } + + private void HandleDisconnected(EndpointState endpoint, ClientConnection connection, Exception exception) + { + lock (_gate) + { + if (!endpoint.Connections.Remove(connection)) + return; + _retiringConnections.Remove(connection); + PublishReadySnapshotLocked(); + } + connection.Fail(exception); + _client.TrackBackgroundTask(DisposeConnectionAsync(connection)); + if (Volatile.Read(ref _stopping) == 0) + { + _client.TransitionTo(ReadyConnectionCount == 0 + ? SharpLinkConnectionState.Reconnecting + : SharpLinkConnectionState.Ready); + EnsureReconnect(endpoint); + EnsureMinimumReadyEndpoints(); + } + } + + private void EnsureMinimumReadyEndpoints() + { + List? missing = null; + lock (_gate) + { + if (Volatile.Read(ref _stopping) != 0) + return; + var readyCount = Volatile.Read(ref _readyEndpoints).Length; + var availableCapacity = _options.MaxConnections - TotalConnectionsLocked(); + var activeReconnects = _endpoints.Count(static endpoint => endpoint.ReconnectTask is { IsCompleted: false }); + var activeInitialDials = _initialDialReservations; + var remaining = Math.Min(TargetReadyEndpointCount - readyCount - activeReconnects - activeInitialDials, availableCapacity); + var start = unchecked((uint)Interlocked.Increment(ref _reconnectCursor)); + for (var offset = 0; remaining > 0 && offset < _endpoints.Length; offset++) + { + var index = (int)((start + (uint)offset) % (uint)_endpoints.Length); + var endpoint = _endpoints[index]; + if (endpoint.ReadyConnections.Length != 0 || + endpoint.NonRetiringConnectionCount + endpoint.ConnectingCount != 0 || + endpoint.ReconnectTask is { IsCompleted: false }) + { + continue; + } + (missing ??= []).Add(endpoint); + remaining--; + } + } + + if (missing is null) + return; + for (var index = 0; index < missing.Count; index++) + EnsureReconnect(missing[index]); + } + + private void EnsureReconnect(EndpointState endpoint) + { + lock (_gate) + { + var activeReconnects = _endpoints.Count(static candidate => candidate.ReconnectTask is { IsCompleted: false }); + if (endpoint.ReconnectTask is { IsCompleted: false } || !NeedsReconnectLocked(endpoint) || + activeReconnects >= TargetReadyEndpointCount - Volatile.Read(ref _readyEndpoints).Length) + { + return; + } + endpoint.ReconnectTask = ReconnectAsync(endpoint); + _client.TrackBackgroundTask(endpoint.ReconnectTask); + } + } + + private void EnsureExpansion(EndpointState endpoint) + { + lock (_gate) + { + if (Volatile.Read(ref _stopping) != 0 || + endpoint.ExpansionTask is { IsCompleted: false } || + TotalConnectionsLocked() >= _options.MaxConnections || + endpoint.NonRetiringConnectionCount + endpoint.ConnectingCount >= _options.MaxConnectionsPerEndpoint) + { + return; + } + + endpoint.ExpansionTask = ExpandAsync(endpoint); + _client.TrackBackgroundTask(endpoint.ExpansionTask); + } + } + + private async Task ExpandAsync(EndpointState endpoint) + { + try + { + await ConnectOneAsync(endpoint, _client._shutdownCts.Token).ConfigureAwait(false); + } + catch (OperationCanceledException) when (_client._shutdownCts.IsCancellationRequested) + { + } + catch (Exception exception) + { + LogClientBackgroundLoopUnhandledException(_client._logger, nameof(ExpandAsync), exception); + if (endpoint.ReadyConnections.Length == 0) + EnsureReconnect(endpoint); + } + } + + private async Task ReconnectAsync(EndpointState endpoint) + { + int delayMilliseconds; + lock (_gate) + delayMilliseconds = endpoint.ReconnectDelayMilliseconds; + try + { + var jitterMilliseconds = Random.Shared.Next(delayMilliseconds / 4 + 1); + await Task.Delay(TimeSpan.FromMilliseconds(delayMilliseconds + jitterMilliseconds), _client._shutdownCts.Token).ConfigureAwait(false); + var shouldConnect = false; + lock (_gate) + shouldConnect = NeedsReconnectLocked(endpoint); + if (shouldConnect) + { + SharpLinkTelemetry.ReconnectAttempt(); + await ConnectOneAsync(endpoint, _client._shutdownCts.Token).ConfigureAwait(false); + lock (_gate) + endpoint.ReconnectDelayMilliseconds = endpoint.ReadyConnections.Length != 0 ? 100 : NextReconnectDelay(delayMilliseconds); + } + } + catch (OperationCanceledException) when (_client._shutdownCts.IsCancellationRequested) + { + return; + } + catch (Exception exception) + { + LogClientBackgroundLoopUnhandledException(_client._logger, nameof(ReconnectAsync), exception); + lock (_gate) + endpoint.ReconnectDelayMilliseconds = NextReconnectDelay(delayMilliseconds); + } + finally + { + lock (_gate) + endpoint.ReconnectTask = null; + } + if (Volatile.Read(ref _stopping) == 0 && !_client._shutdownCts.IsCancellationRequested) + EnsureMinimumReadyEndpoints(); + } + + private static int NextReconnectDelay(int delayMilliseconds) => Math.Min(delayMilliseconds * 2, 5000); + + private void PublishClientReadiness() + { + if (ReadyConnectionCount == 0) + return; + _client._readyTimestamp = Stopwatch.GetTimestamp(); + _client.TransitionTo(SharpLinkConnectionState.Ready); + Volatile.Read(ref _client._readySignal).TrySetResult(true); + } + + private void PublishReadySnapshotLocked() + { + var ready = new List(_endpoints.Length); + for (var index = 0; index < _endpoints.Length; index++) + { + var endpoint = _endpoints[index]; + endpoint.PublishReadyConnections(); + if (endpoint.ReadyConnections.Length != 0) + ready.Add(endpoint); + } + var endpoints = ready.ToArray(); + var existing = Volatile.Read(ref _readyEndpoints); + if (HasSameMembership(existing, endpoints)) + { + if (endpoints.Length == 0) + _client.ResetReadySignal(); + return; + } + var candidates = new SharpLinkEndpointCandidate[endpoints.Length]; + for (var index = 0; index < endpoints.Length; index++) + { + var endpoint = endpoints[index]; + candidates[index] = new SharpLinkEndpointCandidate( + endpoint.Configuration.Endpoint, + endpoint.ReadyConnectionCountProvider, + endpoint.ActiveCallCountProvider, + generation: 1); + } + Volatile.Write(ref _readyEndpoints, endpoints); + Volatile.Write(ref _selectionSnapshot, new EndpointSelectionSnapshot(endpoints, candidates)); + SharpLinkTelemetry.AddClientReadyEndpoints(endpoints.Length - _telemetryReadyEndpointCount); + _telemetryReadyEndpointCount = endpoints.Length; + if (endpoints.Length == 0) + _client.ResetReadySignal(); + } + + private static bool HasSameMembership(EndpointState[] left, EndpointState[] right) + { + if (left.Length != right.Length) + return false; + for (var index = 0; index < left.Length; index++) + if (!ReferenceEquals(left[index], right[index])) + return false; + return true; + } + + private int SelectEndpoint( + EndpointState[] endpoints, + SharpLinkEndpointCandidate[] candidates, + ulong excluded) + { + var availableCount = 0; + for (var index = 0; index < endpoints.Length; index++) + availableCount += (excluded & (1UL << index)) == 0 ? 1 : 0; + if (availableCount == 0) + return -1; + if (availableCount == 1 && _selector is null) + { + for (var index = 0; index < endpoints.Length; index++) + if ((excluded & (1UL << index)) == 0) + return index; + } + if (_selector is not null) + { + try + { + return _selector.Select(new SharpLinkEndpointSelectionContext(candidates, excluded)); + } + catch (Exception exception) + { + _client._logger.LogError(exception, "SharpLink endpoint selector failed."); + throw new SharpLinkException(SharpLinkErrorCode.FailedPrecondition, "The endpoint selector failed.", exception); + } + } + return _strategy switch + { + SharpLinkLoadBalancingStrategy.Random => SelectRandom(endpoints.Length, excluded, availableCount), + SharpLinkLoadBalancingStrategy.RoundRobin => SelectRoundRobin(endpoints.Length, excluded), + SharpLinkLoadBalancingStrategy.LeastPending => SelectLeastPending(endpoints, excluded), + _ => SelectPowerOfTwo(endpoints, excluded, availableCount) + }; + } + + private int SelectPowerOfTwo(EndpointState[] endpoints, ulong excluded, int availableCount) + { + var first = SelectRandom(endpoints.Length, excluded, availableCount); + var second = SelectRandom(endpoints.Length, excluded | (1UL << first), availableCount - 1); + if (second < 0) + return first; + var firstState = endpoints[first]; + var secondState = endpoints[second]; + return StaticEndpointSelection.CompareNormalizedLoad( + firstState.ActiveCallCount, + firstState.ReadyConnections.Length, + secondState.ActiveCallCount, + secondState.ReadyConnections.Length) <= 0 + ? first + : second; + } + + private static int SelectRandom(int length, ulong excluded, int availableCount) + { + if (availableCount <= 0) + return -1; + return StaticEndpointSelection.SelectRandomIndex( + length, + excluded, + availableCount, + Random.Shared.Next(availableCount)); + } + + private int SelectRoundRobin(int length, ulong excluded) + { + return StaticEndpointSelection.SelectRoundRobinIndex(ref _roundRobinCursor, length, excluded); + } + + private int SelectLeastPending(EndpointState[] endpoints, ulong excluded) + { + var start = unchecked((uint)Interlocked.Increment(ref _leastPendingCursor)); + var selected = -1; + for (var offset = 0; offset < endpoints.Length; offset++) + { + var index = (int)((start + (uint)offset) % (uint)endpoints.Length); + if ((excluded & (1UL << index)) != 0) + continue; + if (selected < 0 || endpoints[index].ActiveCallCount < endpoints[selected].ActiveCallCount) + selected = index; + } + return selected; + } + + private static ClientConnection? SelectConnection(EndpointState endpoint) + { + var connections = endpoint.ReadyConnections; + if (connections.Length == 0) + return null; + if (connections.Length == 1) + return connections[0].CanAcceptCalls ? connections[0] : null; + var first = Random.Shared.Next(connections.Length); + var second = Random.Shared.Next(connections.Length - 1); + if (second >= first) + second++; + var selected = SelectLeastLoaded(connections, first, second); + if (selected.CanAcceptCalls) + return selected; + for (var index = 0; index < connections.Length; index++) + if (connections[index].CanAcceptCalls) + return connections[index]; + return null; + } + + private EndpointState? FindEndpointLocked(ClientConnection connection) + { + for (var index = 0; index < _endpoints.Length; index++) + if (_endpoints[index].Connections.Contains(connection)) + return _endpoints[index]; + return null; + } + + private int TotalConnectionsLocked() + { + var count = 0; + for (var index = 0; index < _endpoints.Length; index++) + count += _endpoints[index].NonRetiringConnectionCount + _endpoints[index].ConnectingCount; + return count; + } + + private bool NeedsReconnectLocked(EndpointState endpoint) + => Volatile.Read(ref _stopping) == 0 && !_client._shutdownCts.IsCancellationRequested && + Volatile.Read(ref _readyEndpoints).Length < TargetReadyEndpointCount && + TotalConnectionsLocked() < _options.MaxConnections && + endpoint.NonRetiringConnectionCount + endpoint.ConnectingCount == 0; + + private int CountConnections(Func count) + { + lock (_gate) + { + var result = 0; + for (var index = 0; index < _endpoints.Length; index++) + foreach (var connection in _endpoints[index].Connections) + result += count(connection); + return result; + } + } + + private async Task StopCoreAsync() + { + Interlocked.Exchange(ref _stopping, 1); + var cleanupFailures = new List(); + ClientConnection[] connections; + lock (_gate) + { + connections = [.. _endpoints.SelectMany(static endpoint => endpoint.Connections)]; + for (var index = 0; index < _endpoints.Length; index++) + _endpoints[index].Connections.Clear(); + _retiringConnections.Clear(); + Volatile.Write(ref _readyEndpoints, []); + Volatile.Write(ref _selectionSnapshot, EndpointSelectionSnapshot.Empty); + SharpLinkTelemetry.AddClientReadyEndpoints(-_telemetryReadyEndpointCount); + _telemetryReadyEndpointCount = 0; + } + SharpLinkTelemetry.AddClientActiveEndpoints(-_endpoints.Length); + var stopping = CreateConnectionClosedException("Client is stopping."); + for (var index = 0; index < connections.Length; index++) + { + connections[index].Fail(stopping); + try { await DisposeConnectionAsync(connections[index]).ConfigureAwait(false); } + catch (Exception exception) { cleanupFailures.Add(exception); } + } + await WaitForWorkersAsync(cleanupFailures).ConfigureAwait(false); + for (var index = 0; index < _endpoints.Length; index++) + { + try { await _endpoints[index].Configuration.TransportFactory.DisposeAsync().ConfigureAwait(false); } + catch (Exception exception) { cleanupFailures.Add(exception); } + } + ThrowCleanupFailures(cleanupFailures); + } + + private async Task WaitForWorkersAsync(List cleanupFailures) + { + while (true) + { + Task[] workers; + lock (_gate) + { + var pending = new HashSet(); + if (_connectTask is { IsCompleted: false }) + pending.Add(_connectTask); + foreach (var endpoint in _endpoints) + { + if (endpoint.ReconnectTask is { IsCompleted: false }) + pending.Add(endpoint.ReconnectTask); + if (endpoint.ExpansionTask is { IsCompleted: false }) + pending.Add(endpoint.ExpansionTask); + } + foreach (var attempt in _initialDialTasks) + if (!attempt.IsCompleted) + pending.Add(attempt); + workers = [.. pending]; + } + + if (workers.Length == 0) + return; + + try { await Task.WhenAll(workers).ConfigureAwait(false); } + catch (OperationCanceledException) when (_client._shutdownCts.IsCancellationRequested) { } + catch (Exception exception) { cleanupFailures.Add(exception); } + } + } + + private static void ThrowCleanupFailures(List failures) + { + if (failures.Count == 0) + return; + if (failures.Count == 1) + System.Runtime.ExceptionServices.ExceptionDispatchInfo.Capture(failures[0]).Throw(); + throw new AggregateException(failures); + } + + private static async Task DisposeConnectionAsync(ClientConnection connection) + { + try { await connection.DisposeAsync().ConfigureAwait(false); } + catch (Exception exception) when (exception is IOException or SocketException or ObjectDisposedException) { } + } + + private sealed class EndpointState + { + private readonly Func _readyConnectionCountProvider; + private readonly Func _activeCallCountProvider; + private ClientConnection[] _readyConnections = []; + + public EndpointState(StaticEndpointConfiguration configuration, int index) + { + Configuration = configuration; + Index = index; + _readyConnectionCountProvider = GetReadyConnectionCount; + _activeCallCountProvider = GetActiveCallCount; + } + + public StaticEndpointConfiguration Configuration { get; } + public int Index { get; } + public HashSet Connections { get; } = []; + public ClientConnection[] ReadyConnections => Volatile.Read(ref _readyConnections); + public Func ReadyConnectionCountProvider => _readyConnectionCountProvider; + public Func ActiveCallCountProvider => _activeCallCountProvider; + public int ConnectingCount { get; set; } + public int ReconnectDelayMilliseconds { get; set; } = 100; + public Task? ReconnectTask { get; set; } + public Task? ExpansionTask { get; set; } + public int NonRetiringConnectionCount + { + get + { + var count = 0; + foreach (var connection in Connections) + if (connection.State == ClientConnectionState.Ready) + count++; + return count; + } + } + + public int ActiveCallCount => GetActiveCallCount(); + + private int GetReadyConnectionCount() => ReadyConnections.Length; + + private int GetActiveCallCount() + { + var connections = ReadyConnections; + var count = 0; + for (var index = 0; index < connections.Length; index++) + count += connections[index].ActiveCallCount; + return count; + } + + public void PublishReadyConnections() + { + var ready = new List(Connections.Count); + foreach (var connection in Connections) + if (connection.CanAcceptCalls) + ready.Add(connection); + Volatile.Write(ref _readyConnections, ready.ToArray()); + } + } + + private sealed class EndpointSelectionSnapshot( + EndpointState[] endpoints, + SharpLinkEndpointCandidate[] candidates) + { + public static readonly EndpointSelectionSnapshot Empty = new([], []); + public EndpointState[] Endpoints { get; } = endpoints; + public SharpLinkEndpointCandidate[] Candidates { get; } = candidates; + } + } +} diff --git a/src/SharpLink.Client/SharpLinkClient.Telemetry.cs b/src/SharpLink.Client/SharpLinkClient.Telemetry.cs index 920d904..6d1d0b6 100644 --- a/src/SharpLink.Client/SharpLinkClient.Telemetry.cs +++ b/src/SharpLink.Client/SharpLinkClient.Telemetry.cs @@ -23,9 +23,8 @@ private ValueTask InvokeUnaryWithTelemetryAsync( { var control = ResolveCallControl( options, true, method.HasMethodTimeout, method.MethodTimeout); - invocation = InvokeUnaryCoreAsync( - method.ContractId, method.MethodId, method.HasResponsePayload, - request, requestCodec, responseCodec, control, cancellationToken); + invocation = InvokeUnaryWithOptionalRetryAsync( + method, request, requestCodec, responseCodec, control, cancellationToken); } return ObserveCallAsync(invocation, scope); } @@ -59,7 +58,7 @@ private ValueTask InvokeOneWayWithTelemetryAsync( var control = ResolveCallControl( options, false, method.HasMethodTimeout, method.MethodTimeout); invocation = InvokeOneWayCoreAsync( - method.ContractId, method.MethodId, method.HasClientStreams, + method, request, requestCodec, streams, control, cancellationToken); } return ObserveCallAsync(invocation, scope); @@ -95,7 +94,7 @@ private ValueTask InvokeClientStreamingWithTelemetryAsync _staticManifests = SharpLinkGeneratedAssemblyCatalog.CreateSnapshot(); @@ -42,6 +46,50 @@ internal sealed partial class SharpLinkClient(IClientTransportFactory transportF private readonly RpcSessionFlushOptions? _rpcSessionFlushOptions; private readonly SharpLinkConnectionPoolOptions _connectionPoolOptions = new(); private readonly ISharpLinkClientInterceptor[] _clientInterceptors = []; + private readonly SharpLinkRetryOptions? _retryOptions; + private readonly ISharpLinkRetryPolicy? _retryPolicy; + private readonly ISharpLinkEndpointAdmissionPolicy? _endpointAdmissionPolicy; + + private SharpLinkClient( + IClientTransportFactory transportFactory, + StaticEndpointConfiguration[]? staticEndpoints = null, + SharpLinkClusterOptions? clusterOptions = null, + SharpLinkLoadBalancingStrategy loadBalancingStrategy = SharpLinkLoadBalancingStrategy.PowerOfTwoChoices, + ISharpLinkEndpointSelector? endpointSelector = null, + SharpLinkEndpoint? fixedEndpoint = null, + ISharpLinkEndpointResolver? dynamicResolver = null, + SharpLinkEndpointTransportFactory? dynamicTransportFactory = null, + SharpLinkRetryOptions? retryOptions = null, + ISharpLinkRetryPolicy? retryPolicy = null, + ISharpLinkEndpointAdmissionPolicy? endpointAdmissionPolicy = null) + { + this.transportFactory = transportFactory ?? throw new ArgumentNullException(nameof(transportFactory)); + _fixedEndpoint = fixedEndpoint; + _retryOptions = retryOptions; + _retryPolicy = retryPolicy; + _endpointAdmissionPolicy = endpointAdmissionPolicy; + if (staticEndpoints is not null && dynamicResolver is not null) + throw new ArgumentException("Static endpoints and an endpoint resolver cannot both be configured."); + if (staticEndpoints is not null) + { + _cluster = new StaticClusterRuntime( + this, + staticEndpoints, + clusterOptions ?? throw new ArgumentNullException(nameof(clusterOptions)), + loadBalancingStrategy, + endpointSelector); + } + else if (dynamicResolver is not null) + { + _cluster = new DynamicClusterRuntime( + this, + dynamicResolver, + dynamicTransportFactory ?? throw new ArgumentNullException(nameof(dynamicTransportFactory)), + clusterOptions ?? throw new ArgumentNullException(nameof(clusterOptions)), + loadBalancingStrategy, + endpointSelector); + } + } public SharpLinkClient( IClientTransportFactory transportFactory, @@ -53,8 +101,19 @@ public SharpLinkClient( SharpLinkRuntimeContext? runtimeContext = null, RpcSessionFlushOptions? rpcSessionFlushOptions = null, SharpLinkConnectionPoolOptions? connectionPoolOptions = null, - ISharpLinkClientInterceptor[]? clientInterceptors = null) - : this(transportFactory) + ISharpLinkClientInterceptor[]? clientInterceptors = null, + StaticEndpointConfiguration[]? staticEndpoints = null, + SharpLinkClusterOptions? clusterOptions = null, + SharpLinkLoadBalancingStrategy loadBalancingStrategy = SharpLinkLoadBalancingStrategy.PowerOfTwoChoices, + ISharpLinkEndpointSelector? endpointSelector = null, + SharpLinkEndpoint? fixedEndpoint = null, + ISharpLinkEndpointResolver? dynamicResolver = null, + SharpLinkEndpointTransportFactory? dynamicTransportFactory = null, + SharpLinkRetryOptions? retryOptions = null, + ISharpLinkRetryPolicy? retryPolicy = null, + ISharpLinkEndpointAdmissionPolicy? endpointAdmissionPolicy = null) + : this(transportFactory, staticEndpoints, clusterOptions, loadBalancingStrategy, endpointSelector, fixedEndpoint, + dynamicResolver, dynamicTransportFactory, retryOptions, retryPolicy, endpointAdmissionPolicy) { ArgumentOutOfRangeException.ThrowIfLessThanOrEqual(heartbeatInterval, TimeSpan.Zero); ArgumentOutOfRangeException.ThrowIfLessThanOrEqual(heartbeatTimeout, TimeSpan.Zero); @@ -89,9 +148,21 @@ public SharpLinkClient( SharpLinkRuntimeContext? runtimeContext = null, RpcSessionFlushOptions? rpcSessionFlushOptions = null, SharpLinkConnectionPoolOptions? connectionPoolOptions = null, - ISharpLinkClientInterceptor[]? clientInterceptors = null) + ISharpLinkClientInterceptor[]? clientInterceptors = null, + StaticEndpointConfiguration[]? staticEndpoints = null, + SharpLinkClusterOptions? clusterOptions = null, + SharpLinkLoadBalancingStrategy loadBalancingStrategy = SharpLinkLoadBalancingStrategy.PowerOfTwoChoices, + ISharpLinkEndpointSelector? endpointSelector = null, + SharpLinkEndpoint? fixedEndpoint = null, + ISharpLinkEndpointResolver? dynamicResolver = null, + SharpLinkEndpointTransportFactory? dynamicTransportFactory = null, + SharpLinkRetryOptions? retryOptions = null, + ISharpLinkRetryPolicy? retryPolicy = null, + ISharpLinkEndpointAdmissionPolicy? endpointAdmissionPolicy = null) : this(transportFactory, heartbeatInterval, heartbeatTimeout, requestTimeout, authenticator, protocolOptions, - runtimeContext, rpcSessionFlushOptions, connectionPoolOptions, clientInterceptors) + runtimeContext, rpcSessionFlushOptions, connectionPoolOptions, clientInterceptors, staticEndpoints, + clusterOptions, loadBalancingStrategy, endpointSelector, fixedEndpoint, dynamicResolver, dynamicTransportFactory, + retryOptions, retryPolicy, endpointAdmissionPolicy) { ArgumentNullException.ThrowIfNull(loggerFactory); _logger = loggerFactory.CreateLogger(); @@ -120,6 +191,12 @@ public ValueTask StopAsync(CancellationToken cancellationToken = default) private async Task StopCoreAsync() { + if (_cluster is not null) + { + await StopStaticClusterCoreAsync().ConfigureAwait(false); + return; + } + lock (_registryGate) TransitionTo(SharpLinkConnectionState.Draining); _shutdownCts.Cancel(); @@ -167,6 +244,44 @@ private async Task StopCoreAsync() TransitionTo(SharpLinkConnectionState.Stopped); } + private async Task StopStaticClusterCoreAsync() + { + var cleanupFailures = new List(); + lock (_registryGate) + TransitionTo(SharpLinkConnectionState.Draining); + _shutdownCts.Cancel(); + Volatile.Read(ref _readySignal).TrySetResult(true); + try { await _cluster!.StopAsync().ConfigureAwait(false); } + catch (Exception exception) { cleanupFailures.Add(exception); } + try { await WaitForBackgroundTasksAsync().ConfigureAwait(false); } + catch (Exception exception) { cleanupFailures.Add(exception); } + + Assembly[] dynamicAssemblies; + lock (_registryGate) + dynamicAssemblies = [.. _dynamicModules.Keys]; + for (var index = 0; index < dynamicAssemblies.Length; index++) + { + try { await UnregisterAssemblyAsync(dynamicAssemblies[index], TimeSpan.Zero).ConfigureAwait(false); } + catch (Exception exception) { cleanupFailures.Add(exception); } + } + + try { _reconnectSignal.Dispose(); } + catch (Exception exception) { cleanupFailures.Add(exception); } + try { _shutdownCts.Dispose(); } + catch (Exception exception) { cleanupFailures.Add(exception); } + TransitionTo(SharpLinkConnectionState.Stopped); + ThrowStopCleanupFailures(cleanupFailures); + } + + private static void ThrowStopCleanupFailures(List failures) + { + if (failures.Count == 0) + return; + if (failures.Count == 1) + System.Runtime.ExceptionServices.ExceptionDispatchInfo.Capture(failures[0]).Throw(); + throw new AggregateException(failures); + } + private async Task IgnoreExpectedStopExceptionAsync(Task? task) { if (task is null) @@ -246,12 +361,14 @@ private void ResetReadySignal() } } - internal int ReadyConnectionCount => Volatile.Read(ref _readyConnections).Length; + internal int ReadyConnectionCount => _cluster?.ReadyConnectionCount ?? Volatile.Read(ref _readyConnections).Length; internal int PendingCallCount { get { + if (_cluster is not null) + return _cluster.PendingCallCount; var connections = Volatile.Read(ref _readyConnections); var count = 0; for (var index = 0; index < connections.Length; index++) @@ -264,6 +381,8 @@ internal int ActiveClientCallCount { get { + if (_cluster is not null) + return _cluster.ActiveCallCount; var connections = Volatile.Read(ref _readyConnections); var count = 0; for (var index = 0; index < connections.Length; index++) @@ -276,6 +395,8 @@ internal int ActiveClientStreamCount { get { + if (_cluster is not null) + return _cluster.ActiveStreamCount; var connections = Volatile.Read(ref _readyConnections); var count = 0; for (var index = 0; index < connections.Length; index++) diff --git a/src/SharpLink.Client/SharpLinkClusterOptions.cs b/src/SharpLink.Client/SharpLinkClusterOptions.cs new file mode 100644 index 0000000..50aaf51 --- /dev/null +++ b/src/SharpLink.Client/SharpLinkClusterOptions.cs @@ -0,0 +1,93 @@ +namespace SharpLink.Client; + +/// Configures the bounded resources owned by a static multi-endpoint client. +public sealed class SharpLinkClusterOptions +{ + /// The maximum number of endpoints accepted by one static topology. + public const int MaximumEndpoints = 64; + + /// Gets or sets the maximum number of endpoints. Values must be from one through 64. + public int MaxEndpoints { get; set; } = MaximumEndpoints; + + /// Gets or sets the target number of endpoints with at least one Ready connection. + public int MinReadyEndpoints { get; set; } = 2; + + /// Gets or sets the global Ready and Connecting connection budget. + public int MaxConnections { get; set; } = 4; + + /// Gets or sets the maximum number of Ready and Connecting connections per endpoint. + public int MaxConnectionsPerEndpoint { get; set; } = 2; + + /// Gets or sets the separate maximum number of retiring connections. + public int MaxRetiringConnections { get; set; } = 4; + + internal SharpLinkClusterOptions CloneValidated(int endpointCount) + { + if (endpointCount is < 2 or > MaximumEndpoints) + throw new ArgumentOutOfRangeException(nameof(endpointCount)); + if (MaxEndpoints is < 1 or > MaximumEndpoints) + throw new ArgumentOutOfRangeException(nameof(MaxEndpoints)); + if (endpointCount > MaxEndpoints) + throw new ArgumentException("The configured endpoint collection exceeds MaxEndpoints.", nameof(endpointCount)); + if (MinReadyEndpoints is < 1 or > MaximumEndpoints) + throw new ArgumentOutOfRangeException(nameof(MinReadyEndpoints)); + if (MaxConnections is < 1 or > SharpLinkConnectionPoolOptions.MaximumConnections) + throw new ArgumentOutOfRangeException(nameof(MaxConnections)); + if (MaxConnectionsPerEndpoint < 1 || MaxConnectionsPerEndpoint > MaxConnections) + throw new ArgumentOutOfRangeException(nameof(MaxConnectionsPerEndpoint)); + if (Math.Min(MinReadyEndpoints, endpointCount) > MaxConnections) + throw new ArgumentException("MinReadyEndpoints cannot exceed MaxConnections.", nameof(MinReadyEndpoints)); + if (MaxRetiringConnections is < 0 or > SharpLinkConnectionPoolOptions.MaximumConnections) + throw new ArgumentOutOfRangeException(nameof(MaxRetiringConnections)); + + return new SharpLinkClusterOptions + { + MaxEndpoints = MaxEndpoints, + MinReadyEndpoints = MinReadyEndpoints, + MaxConnections = MaxConnections, + MaxConnectionsPerEndpoint = MaxConnectionsPerEndpoint, + MaxRetiringConnections = MaxRetiringConnections + }; + } + + internal SharpLinkClusterOptions CloneValidatedForDynamicResolver() + { + if (MaxEndpoints is < 1 or > MaximumEndpoints) + throw new ArgumentOutOfRangeException(nameof(MaxEndpoints)); + if (MinReadyEndpoints is < 1 or > MaximumEndpoints) + throw new ArgumentOutOfRangeException(nameof(MinReadyEndpoints)); + if (MaxConnections is < 1 or > SharpLinkConnectionPoolOptions.MaximumConnections) + throw new ArgumentOutOfRangeException(nameof(MaxConnections)); + if (MaxConnectionsPerEndpoint < 1 || MaxConnectionsPerEndpoint > MaxConnections) + throw new ArgumentOutOfRangeException(nameof(MaxConnectionsPerEndpoint)); + if (Math.Min(MinReadyEndpoints, MaxEndpoints) > MaxConnections) + throw new ArgumentException("MinReadyEndpoints cannot exceed MaxConnections.", nameof(MinReadyEndpoints)); + if (MaxRetiringConnections is < 0 or > SharpLinkConnectionPoolOptions.MaximumConnections) + throw new ArgumentOutOfRangeException(nameof(MaxRetiringConnections)); + + return new SharpLinkClusterOptions + { + MaxEndpoints = MaxEndpoints, + MinReadyEndpoints = MinReadyEndpoints, + MaxConnections = MaxConnections, + MaxConnectionsPerEndpoint = MaxConnectionsPerEndpoint, + MaxRetiringConnections = MaxRetiringConnections + }; + } +} + +/// Chooses the built-in strategy used for static endpoint selection. +public enum SharpLinkLoadBalancingStrategy +{ + /// Compares two random Ready endpoints by active calls per Ready connection. + PowerOfTwoChoices, + + /// Chooses one random Ready endpoint. + Random, + + /// Cycles through Ready endpoints with an instance-scoped atomic cursor. + RoundRobin, + + /// Scans Ready endpoints for the least pending work with rotating tie breaking. + LeastPending +} diff --git a/src/SharpLink.Client/SharpLinkEndpointResolvers.cs b/src/SharpLink.Client/SharpLinkEndpointResolvers.cs new file mode 100644 index 0000000..37fdc76 --- /dev/null +++ b/src/SharpLink.Client/SharpLinkEndpointResolvers.cs @@ -0,0 +1,332 @@ +using System.Net; +using System.Net.Sockets; +using System.Runtime.CompilerServices; + +namespace SharpLink.Client; + +/// Adapts application-supplied resolve and watch delegates to an endpoint resolver. +/// +/// When no watch delegate is supplied, the resolver polls the resolve delegate with one bounded delay. +/// This allows applications to adapt an existing registry client without SharpLink taking a dependency on it. +/// +public sealed class DelegateSharpLinkEndpointResolver : ISharpLinkEndpointResolver +{ + private static readonly TimeSpan DefaultPollingInterval = TimeSpan.FromSeconds(30); + private readonly Func> _resolve; + private readonly Func>? _watch; + private readonly TimeSpan _pollingInterval; + private readonly CancellationTokenSource _disposeCts = new(); + private int _disposed; + + /// Initializes a polling delegate resolver. + /// Returns the latest complete endpoint snapshot. + /// The positive delay between resolve calls after the initial resolution. + public DelegateSharpLinkEndpointResolver( + Func> resolve, + TimeSpan? pollingInterval = null) + : this(resolve, watch: null, pollingInterval) + { + } + + /// Initializes a delegate resolver with an optional continuous watch. + /// Returns the latest complete endpoint snapshot. + /// Optionally returns subsequent snapshots. When null, is polled. + /// The positive polling delay used when is null. + public DelegateSharpLinkEndpointResolver( + Func> resolve, + Func>? watch, + TimeSpan? pollingInterval = null) + { + _resolve = resolve ?? throw new ArgumentNullException(nameof(resolve)); + _watch = watch; + _pollingInterval = pollingInterval ?? DefaultPollingInterval; + ArgumentOutOfRangeException.ThrowIfLessThanOrEqual(_pollingInterval, TimeSpan.Zero); + } + + /// + public async ValueTask ResolveAsync(CancellationToken cancellationToken) + { + ThrowIfDisposed(); + using var linked = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken, _disposeCts.Token); + return await _resolve(linked.Token).ConfigureAwait(false); + } + + /// + public async IAsyncEnumerable WatchAsync( + [EnumeratorCancellation] CancellationToken cancellationToken) + { + ThrowIfDisposed(); + using var linked = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken, _disposeCts.Token); + if (_watch is not null) + { + await foreach (var snapshot in _watch(linked.Token).WithCancellation(linked.Token).ConfigureAwait(false)) + yield return snapshot; + yield break; + } + + while (true) + { + await Task.Delay(_pollingInterval, linked.Token).ConfigureAwait(false); + yield return await _resolve(linked.Token).ConfigureAwait(false); + } + } + + /// + public ValueTask DisposeAsync() + { + if (Interlocked.Exchange(ref _disposed, 1) == 0) + _disposeCts.Cancel(); + return ValueTask.CompletedTask; + } + + private void ThrowIfDisposed() + { + if (Volatile.Read(ref _disposed) != 0) + throw new ObjectDisposedException(nameof(DelegateSharpLinkEndpointResolver)); + } +} + +/// Configures DNS endpoint discovery refresh behavior. +public sealed class SharpLinkDnsResolverOptions +{ + /// Gets or sets the default refresh interval when DNS TTL is not available from the platform. + public TimeSpan RefreshInterval { get; set; } = TimeSpan.FromSeconds(30); + + /// Gets or sets the smallest permitted refresh interval. + public TimeSpan MinimumRefreshInterval { get; set; } = TimeSpan.FromSeconds(1); + + /// Gets or sets the largest permitted refresh interval. + public TimeSpan MaximumRefreshInterval { get; set; } = TimeSpan.FromMinutes(5); + + /// Gets or sets the symmetric random jitter ratio from zero through one. + public double JitterRatio { get; set; } = 0.2; + + /// Gets or sets an optional IP address-family filter. Null accepts A and AAAA records. + public AddressFamily? AddressFamily { get; set; } + + internal SharpLinkDnsResolverOptions CloneValidated() + { + ArgumentOutOfRangeException.ThrowIfLessThanOrEqual(RefreshInterval, TimeSpan.Zero); + ArgumentOutOfRangeException.ThrowIfLessThanOrEqual(MinimumRefreshInterval, TimeSpan.Zero); + ArgumentOutOfRangeException.ThrowIfLessThanOrEqual(MaximumRefreshInterval, TimeSpan.Zero); + if (MinimumRefreshInterval > MaximumRefreshInterval) + throw new ArgumentException("MinimumRefreshInterval cannot exceed MaximumRefreshInterval."); + if (JitterRatio is < 0 or > 1 || double.IsNaN(JitterRatio)) + throw new ArgumentOutOfRangeException(nameof(JitterRatio)); + if (AddressFamily is { } addressFamily && + addressFamily is not (System.Net.Sockets.AddressFamily.InterNetwork or System.Net.Sockets.AddressFamily.InterNetworkV6)) + { + throw new ArgumentOutOfRangeException(nameof(AddressFamily)); + } + + return new SharpLinkDnsResolverOptions + { + RefreshInterval = RefreshInterval, + MinimumRefreshInterval = MinimumRefreshInterval, + MaximumRefreshInterval = MaximumRefreshInterval, + JitterRatio = JitterRatio, + AddressFamily = AddressFamily + }; + } +} + +internal interface ISharpLinkDnsQuery +{ + ValueTask QueryAsync(string host, CancellationToken cancellationToken); +} + +internal sealed class BclSharpLinkDnsQuery : ISharpLinkDnsQuery +{ + public static readonly BclSharpLinkDnsQuery Instance = new(); + + private BclSharpLinkDnsQuery() + { + } + + public async ValueTask QueryAsync(string host, CancellationToken cancellationToken) + => await Dns.GetHostAddressesAsync(host, cancellationToken).ConfigureAwait(false); +} + +/// Resolves A and AAAA records into a dynamic TCP endpoint topology. +/// +/// DNS record order is ignored. Stable endpoint IDs are derived from the original host, port, address +/// family, and normalized IP address, while the original host remains the default TLS authority. +/// +public sealed class SharpLinkDnsEndpointResolver : ISharpLinkEndpointResolver +{ + private readonly string _host; + private readonly int _port; + private readonly SharpLinkDnsResolverOptions _options; + private readonly ISharpLinkDnsQuery _query; + private readonly CancellationTokenSource _disposeCts = new(); + private readonly Lock _gate = new(); + private SharpLinkEndpointSnapshot? _lastSnapshot; + private string[] _lastEndpointKeys = []; + private long _version; + private int _disposed; + + /// Initializes a DNS endpoint resolver. + /// The non-empty DNS host name retained as endpoint authority. + /// The TCP port from 1 through 65535. + /// Optional options copied and frozen by the resolver. + public SharpLinkDnsEndpointResolver( + string host, + int port, + SharpLinkDnsResolverOptions? options = null) + : this(host, port, (options ?? new SharpLinkDnsResolverOptions()).CloneValidated(), BclSharpLinkDnsQuery.Instance) + { + } + + internal SharpLinkDnsEndpointResolver( + string host, + int port, + SharpLinkDnsResolverOptions options, + ISharpLinkDnsQuery query) + { + ArgumentException.ThrowIfNullOrWhiteSpace(host); + if (port is < 1 or > 65535) + throw new ArgumentOutOfRangeException(nameof(port)); + _host = host; + _port = port; + _options = (options ?? throw new ArgumentNullException(nameof(options))).CloneValidated(); + _query = query ?? throw new ArgumentNullException(nameof(query)); + } + + /// + public async ValueTask ResolveAsync(CancellationToken cancellationToken) + { + ThrowIfDisposed(); + using var linked = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken, _disposeCts.Token); + try + { + var result = await QueryAndCreateSnapshotAsync(linked.Token).ConfigureAwait(false); + return result.Snapshot; + } + catch (OperationCanceledException) when (linked.IsCancellationRequested) + { + throw; + } + catch when (TryGetLastSnapshot(out var lastSnapshot)) + { + return lastSnapshot; + } + } + + /// + public async IAsyncEnumerable WatchAsync( + [EnumeratorCancellation] CancellationToken cancellationToken) + { + ThrowIfDisposed(); + using var linked = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken, _disposeCts.Token); + while (true) + { + await Task.Delay(GetRefreshDelay(), linked.Token).ConfigureAwait(false); + SharpLinkEndpointSnapshot? published = null; + try + { + var result = await QueryAndCreateSnapshotAsync(linked.Token).ConfigureAwait(false); + if (result.Changed) + published = result.Snapshot; + } + catch (OperationCanceledException) when (linked.IsCancellationRequested) + { + yield break; + } + catch + { + // DNS failure intentionally retains the last successful topology until the next refresh. + } + if (published is not null) + yield return published; + } + } + + /// + public ValueTask DisposeAsync() + { + if (Interlocked.Exchange(ref _disposed, 1) == 0) + _disposeCts.Cancel(); + return ValueTask.CompletedTask; + } + + private async ValueTask<(SharpLinkEndpointSnapshot Snapshot, bool Changed)> QueryAndCreateSnapshotAsync( + CancellationToken cancellationToken) + { + var addresses = await _query.QueryAsync(_host, cancellationToken).ConfigureAwait(false); + var values = new List<(string Key, IPAddress Address)>(addresses.Length); + var seen = new HashSet(StringComparer.Ordinal); + for (var index = 0; index < addresses.Length; index++) + { + var address = addresses[index]; + var normalized = address.IsIPv4MappedToIPv6 ? address.MapToIPv4() : address; + if (_options.AddressFamily is { } family && normalized.AddressFamily != family) + continue; + if (normalized.AddressFamily is not (System.Net.Sockets.AddressFamily.InterNetwork or System.Net.Sockets.AddressFamily.InterNetworkV6)) + continue; + var key = $"{normalized.AddressFamily}:{normalized}"; + if (seen.Add(key)) + values.Add((key, normalized)); + } + values.Sort(static (left, right) => StringComparer.Ordinal.Compare(left.Key, right.Key)); + var keys = new string[values.Count]; + for (var index = 0; index < values.Count; index++) + keys[index] = values[index].Key; + + lock (_gate) + { + if (_lastSnapshot is not null && keys.AsSpan().SequenceEqual(_lastEndpointKeys, StringComparer.Ordinal)) + return (_lastSnapshot, false); + + var endpoints = new SharpLinkEndpoint[values.Count]; + for (var index = 0; index < values.Count; index++) + { + var value = values[index]; + endpoints[index] = new SharpLinkEndpoint + { + Id = $"dns:{GetHostHash(_host)}:{_port}:{value.Key}", + Address = new SharpLinkTcpAddress(value.Address.ToString(), _port), + Authority = _host + }; + } + var snapshot = new SharpLinkEndpointSnapshot(++_version, endpoints); + _lastEndpointKeys = keys; + _lastSnapshot = snapshot; + return (snapshot, true); + } + } + + private static string GetHostHash(string host) + => Convert.ToHexString( + System.Security.Cryptography.SHA256.HashData(System.Text.Encoding.UTF8.GetBytes(host)).AsSpan(0, 16)); + + private TimeSpan GetRefreshDelay() + { + var baseInterval = _options.RefreshInterval; + if (baseInterval < _options.MinimumRefreshInterval) + baseInterval = _options.MinimumRefreshInterval; + if (baseInterval > _options.MaximumRefreshInterval) + baseInterval = _options.MaximumRefreshInterval; + if (_options.JitterRatio == 0) + return baseInterval; + + var multiplier = 1 - _options.JitterRatio + Random.Shared.NextDouble() * _options.JitterRatio * 2; + var jittered = TimeSpan.FromTicks((long)(baseInterval.Ticks * multiplier)); + return jittered < _options.MinimumRefreshInterval ? _options.MinimumRefreshInterval : + jittered > _options.MaximumRefreshInterval ? _options.MaximumRefreshInterval : jittered; + } + + private bool TryGetLastSnapshot(out SharpLinkEndpointSnapshot snapshot) + { + lock (_gate) + { + snapshot = _lastSnapshot!; + return snapshot is not null; + } + } + + private void ThrowIfDisposed() + { + if (Volatile.Read(ref _disposed) != 0) + throw new ObjectDisposedException(nameof(SharpLinkDnsEndpointResolver)); + } +} diff --git a/src/SharpLink.Client/SharpLinkRetryOptions.cs b/src/SharpLink.Client/SharpLinkRetryOptions.cs new file mode 100644 index 0000000..f324162 --- /dev/null +++ b/src/SharpLink.Client/SharpLinkRetryOptions.cs @@ -0,0 +1,41 @@ +namespace SharpLink.Client; + +/// Configures the built-in retry policy for idempotent unary calls. +/// +/// is the total number of attempts, including the first attempt. Retry is +/// disabled until or an overload is called. +/// +public sealed class SharpLinkRetryOptions +{ + /// Gets or sets the total attempt limit from one through ten. The default is three. + public int MaxAttempts { get; set; } = 3; + + /// Gets or sets the initial non-negative retry delay. The default is 50 ms. + public TimeSpan InitialBackoff { get; set; } = TimeSpan.FromMilliseconds(50); + + /// Gets or sets the non-negative upper bound for exponential retry delays. The default is 200 ms. + public TimeSpan MaxBackoff { get; set; } = TimeSpan.FromMilliseconds(200); + + /// Gets or sets symmetric delay jitter from zero through one. The default is 0.2. + public double JitterRatio { get; set; } = 0.2; + + internal SharpLinkRetryOptions CloneValidated() + { + if (MaxAttempts is < 1 or > 10) + throw new ArgumentOutOfRangeException(nameof(MaxAttempts)); + if (InitialBackoff < TimeSpan.Zero) + throw new ArgumentOutOfRangeException(nameof(InitialBackoff)); + if (MaxBackoff < TimeSpan.Zero || MaxBackoff < InitialBackoff) + throw new ArgumentOutOfRangeException(nameof(MaxBackoff)); + if (JitterRatio is < 0 or > 1 || double.IsNaN(JitterRatio)) + throw new ArgumentOutOfRangeException(nameof(JitterRatio)); + + return new SharpLinkRetryOptions + { + MaxAttempts = MaxAttempts, + InitialBackoff = InitialBackoff, + MaxBackoff = MaxBackoff, + JitterRatio = JitterRatio + }; + } +} diff --git a/src/SharpLink.Client/SharpLinkTransportFactories.cs b/src/SharpLink.Client/SharpLinkTransportFactories.cs new file mode 100644 index 0000000..2799685 --- /dev/null +++ b/src/SharpLink.Client/SharpLinkTransportFactories.cs @@ -0,0 +1,70 @@ +namespace SharpLink.Client; + +/// Creates built-in endpoint transport factories for static SharpLink topologies. +public static class SharpLinkTransportFactories +{ + /// Creates a factory for TCP and Unix-domain socket endpoint addresses. + /// Optional socket settings copied by each created transport factory. + /// An endpoint factory that accepts and . + public static SharpLinkEndpointTransportFactory Sockets(SocketTransportOptions? options = null) + => endpoint => endpoint.Address switch + { + SharpLinkTcpAddress tcp => new SocketClientTransportFactory(CreateTcpEndPoint(tcp), options), + SharpLinkUnixDomainSocketAddress uds => new SocketClientTransportFactory(new UnixDomainSocketEndPoint(uds.Path), options), + _ => throw new ArgumentException("Sockets require a TCP or Unix-domain socket endpoint address.", nameof(endpoint)) + }; + + /// Creates a TLS socket factory for TCP and Unix-domain socket endpoint addresses. + /// TLS settings copied for every endpoint factory. + /// Optional socket settings copied by each created transport factory. + /// An optional positive TLS handshake timeout. + /// + /// When the supplied TLS options omit , + /// the endpoint Authority is used; TCP endpoints then fall back to their Host. + /// + public static SharpLinkEndpointTransportFactory Sockets( + SslClientAuthenticationOptions tlsOptions, + SocketTransportOptions? options = null, + TimeSpan? tlsHandshakeTimeout = null) + { + ArgumentNullException.ThrowIfNull(tlsOptions); + return endpoint => endpoint.Address switch + { + SharpLinkTcpAddress tcp => new SocketClientTransportFactory( + CreateTcpEndPoint(tcp), options, CreateTlsOptions(tlsOptions, endpoint.Authority ?? tcp.Host), tlsHandshakeTimeout), + SharpLinkUnixDomainSocketAddress uds => new SocketClientTransportFactory( + new UnixDomainSocketEndPoint(uds.Path), options, CreateTlsOptions(tlsOptions, endpoint.Authority), tlsHandshakeTimeout), + _ => throw new ArgumentException("Sockets require a TCP or Unix-domain socket endpoint address.", nameof(endpoint)) + }; + } + + /// Creates a factory for named-pipe endpoint addresses. + /// An endpoint factory that accepts . + public static SharpLinkEndpointTransportFactory NamedPipes() + => endpoint => endpoint.Address is SharpLinkNamedPipeAddress pipe + ? new NamedPipeClientTransportFactory(pipe.PipeName, pipe.ServerName) + : throw new ArgumentException("Named pipes require a named-pipe endpoint address.", nameof(endpoint)); + + /// Creates a factory for shared-memory endpoint addresses. + /// Optionally configures options copied by each created factory. + /// An endpoint factory that accepts . + public static SharpLinkEndpointTransportFactory SharedMemory(Action? configure = null) + { + var options = new SharedMemoryTransportOptions(); + configure?.Invoke(options); + options.Validate(); + return endpoint => endpoint.Address is SharpLinkSharedMemoryAddress memory + ? new SharedMemoryClientTransportFactory(memory.Name, options) + : throw new ArgumentException("Shared memory requires a shared-memory endpoint address.", nameof(endpoint)); + } + + private static EndPoint CreateTcpEndPoint(SharpLinkTcpAddress address) + => IPAddress.TryParse(address.Host, out var ipAddress) + ? new IPEndPoint(ipAddress, address.Port) + : new DnsEndPoint(address.Host, address.Port); + + private static SslClientAuthenticationOptions CreateTlsOptions( + SslClientAuthenticationOptions source, + string? defaultTargetHost) + => TlsAuthenticationOptionsSnapshot.Clone(source, defaultTargetHost)!; +} diff --git a/src/SharpLink.Client/StaticEndpointConfiguration.cs b/src/SharpLink.Client/StaticEndpointConfiguration.cs new file mode 100644 index 0000000..8ad5343 --- /dev/null +++ b/src/SharpLink.Client/StaticEndpointConfiguration.cs @@ -0,0 +1,28 @@ +namespace SharpLink.Client; + +internal sealed class StaticEndpointConfiguration( + SharpLinkEndpoint endpoint, + IClientTransportFactory transportFactory) +{ + public SharpLinkEndpoint Endpoint { get; private set; } = endpoint; + public IClientTransportFactory TransportFactory { get; } = transportFactory; + + public void ReplaceEndpoint(SharpLinkEndpoint endpoint) + => Endpoint = endpoint ?? throw new ArgumentNullException(nameof(endpoint)); +} + +/// Guards the fixed-transport field for resolver-backed clients, which never use it. +internal sealed class DynamicClusterTransportPlaceholder : IClientTransportFactory +{ + public static readonly DynamicClusterTransportPlaceholder Instance = new(); + + private DynamicClusterTransportPlaceholder() + { + } + + public ValueTask ConnectAsync(CancellationToken cancellationToken = default) + => ValueTask.FromException(new InvalidOperationException( + "Resolver-backed clients create endpoint-specific transport factories.")); + + public ValueTask DisposeAsync() => ValueTask.CompletedTask; +} diff --git a/src/SharpLink.Client/StaticEndpointSelection.cs b/src/SharpLink.Client/StaticEndpointSelection.cs new file mode 100644 index 0000000..33e7c39 --- /dev/null +++ b/src/SharpLink.Client/StaticEndpointSelection.cs @@ -0,0 +1,45 @@ +namespace SharpLink.Client; + +internal static class StaticEndpointSelection +{ + public static int CompareNormalizedLoad( + int firstActiveCalls, + int firstReadyConnections, + int secondActiveCalls, + int secondReadyConnections) + { + ArgumentOutOfRangeException.ThrowIfNegative(firstActiveCalls); + ArgumentOutOfRangeException.ThrowIfNegative(secondActiveCalls); + ArgumentOutOfRangeException.ThrowIfNegativeOrZero(firstReadyConnections); + ArgumentOutOfRangeException.ThrowIfNegativeOrZero(secondReadyConnections); + var first = (long)firstActiveCalls * secondReadyConnections; + var second = (long)secondActiveCalls * firstReadyConnections; + return first.CompareTo(second); + } + + public static int SelectRandomIndex(int length, ulong excluded, int availableCount, int target) + { + if (availableCount <= 0 || target < 0 || target >= availableCount) + return -1; + for (var index = 0; index < length; index++) + { + if ((excluded & (1UL << index)) != 0) + continue; + if (target-- == 0) + return index; + } + return -1; + } + + public static int SelectRoundRobinIndex(ref int cursor, int length, ulong excluded) + { + var start = unchecked((uint)Interlocked.Increment(ref cursor)); + for (var offset = 0; offset < length; offset++) + { + var index = (int)((start + (uint)offset) % (uint)length); + if ((excluded & (1UL << index)) == 0) + return index; + } + return -1; + } +} diff --git a/src/SharpLink.Runtime/Transport/SocketTransportV2.cs b/src/SharpLink.Runtime/Transport/SocketTransportV2.cs index 1151c8f..eb1b5f5 100644 --- a/src/SharpLink.Runtime/Transport/SocketTransportV2.cs +++ b/src/SharpLink.Runtime/Transport/SocketTransportV2.cs @@ -237,6 +237,27 @@ internal static class SocketTransportSocketFactory { public static Socket Create(EndPoint endPoint) { + if (endPoint is DnsEndPoint) + { + if (Socket.OSSupportsIPv6) + { + Socket? dualMode = null; + try + { + dualMode = new Socket(AddressFamily.InterNetworkV6, SocketType.Stream, ProtocolType.Tcp) + { + DualMode = true + }; + return dualMode; + } + catch (Exception exception) when (exception is SocketException or NotSupportedException) + { + dualMode?.Dispose(); + } + } + + return new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp); + } var addressFamily = endPoint.AddressFamily == AddressFamily.Unspecified ? AddressFamily.InterNetwork : endPoint.AddressFamily; diff --git a/src/SharpLink.Runtime/Transport/TlsTransport.cs b/src/SharpLink.Runtime/Transport/TlsTransport.cs index d87beae..f0c1334 100644 --- a/src/SharpLink.Runtime/Transport/TlsTransport.cs +++ b/src/SharpLink.Runtime/Transport/TlsTransport.cs @@ -111,17 +111,21 @@ public static TimeSpan ValidateTimeout(TimeSpan? timeout) return value; } - public static SslClientAuthenticationOptions? Clone(SslClientAuthenticationOptions? source) + public static SslClientAuthenticationOptions? Clone( + SslClientAuthenticationOptions? source, + string? defaultTargetHost = null) { if (source is null) return null; - ArgumentException.ThrowIfNullOrWhiteSpace(source.TargetHost); - return new SslClientAuthenticationOptions + var targetHost = string.IsNullOrWhiteSpace(source.TargetHost) ? defaultTargetHost : source.TargetHost; + ArgumentException.ThrowIfNullOrWhiteSpace(targetHost); + var clone = new SslClientAuthenticationOptions { - TargetHost = source.TargetHost, + TargetHost = targetHost, ClientCertificates = source.ClientCertificates is null ? null : new X509CertificateCollection(source.ClientCertificates), + ClientCertificateContext = source.ClientCertificateContext, EnabledSslProtocols = source.EnabledSslProtocols, CertificateRevocationCheckMode = source.CertificateRevocationCheckMode, EncryptionPolicy = source.EncryptionPolicy, @@ -135,6 +139,12 @@ public static TimeSpan ValidateTimeout(TimeSpan? timeout) CipherSuitesPolicy = source.CipherSuitesPolicy, CertificateChainPolicy = source.CertificateChainPolicy }; + if (OperatingSystem.IsLinux() || OperatingSystem.IsWindows()) + { + clone.AllowRsaPkcs1Padding = source.AllowRsaPkcs1Padding; + clone.AllowRsaPssPadding = source.AllowRsaPssPadding; + } + return clone; } public static SslServerAuthenticationOptions? Clone(SslServerAuthenticationOptions? source) diff --git a/test/SharpLink.AotSmoke/Program.cs b/test/SharpLink.AotSmoke/Program.cs index 2e65f0a..bb6dab6 100644 --- a/test/SharpLink.AotSmoke/Program.cs +++ b/test/SharpLink.AotSmoke/Program.cs @@ -68,13 +68,31 @@ public static async Task Main(string[] args) } }, CancellationToken.None); - var clientBuilder = SharpClientBuilder.Create() - .UseRuntime(ConfigureCompression); + ISharpLinkClient client; if (useSharedMemory) - clientBuilder.UseSharedMemory(sharedMemoryName); + { + client = SharpClientBuilder.Create() + .UseRuntime(ConfigureCompression) + .UseSharedMemory(sharedMemoryName) + .Build(); + } else - clientBuilder.UseTcp(IPAddress.Loopback.ToString(), port); - var client = clientBuilder.Build(); + { + client = SharpClientBuilder.Create() + .UseRuntime(ConfigureCompression) + .UseEndpointResolver( + new DelegateSharpLinkEndpointResolver( + _ => ValueTask.FromResult(new SharpLinkEndpointSnapshot(1, + [ + new SharpLinkEndpoint + { + Id = "aot-local", + Address = new SharpLinkTcpAddress(IPAddress.Loopback.ToString(), port) + } + ]))), + SharpLinkTransportFactories.Sockets()) + .Build(); + } try { diff --git a/test/SharpLink.IntegrationTests/DynamicEndpointIntegrationTests.cs b/test/SharpLink.IntegrationTests/DynamicEndpointIntegrationTests.cs new file mode 100644 index 0000000..997fe3a --- /dev/null +++ b/test/SharpLink.IntegrationTests/DynamicEndpointIntegrationTests.cs @@ -0,0 +1,1066 @@ +using System.Threading.Channels; + +namespace SharpLink.IntegrationTests; + +public sealed class DynamicEndpointIntegrationTests +{ + [Test] + [NotInParallel] + public async Task DynamicResolverShouldAddRemoveReplaceAndUpdateAttributesWithoutReconnecting() + { + await using var first = await TcpServerScope.StartAsync("first"); + await using var second = await TcpServerScope.StartAsync("second"); + await using var replacement = await TcpServerScope.StartAsync("replacement"); + var resolver = new ControllableResolver(new SharpLinkEndpointSnapshot(1, [Endpoint("first", first.Port, "blue")])); + var selector = new ZoneSelector("blue"); + var factoryCreates = 0; + var sockets = SharpLinkTransportFactories.Sockets(); + var client = SharpClientBuilder.Create() + .UseSerializer(MemoryPackCodec.Resolver) + .UseHeartbeat(TimeSpan.FromMilliseconds(100), TimeSpan.FromMilliseconds(500)) + .UseEndpointResolver( + resolver, + endpoint => + { + Interlocked.Increment(ref factoryCreates); + return sockets(endpoint); + }) + .UseEndpointSelector(selector) + .UseCluster(options => + { + options.MinReadyEndpoints = 2; + options.MaxConnections = 2; + options.MaxConnectionsPerEndpoint = 1; + }) + .Build(); + + try + { + await client.ConnectAsync(); + var service = client.Get(); + Ensure(await service.GetEndpointIdAsync() == "first", "initial resolver endpoint"); + + resolver.Publish(new SharpLinkEndpointSnapshot(2, + [ + Endpoint("first", first.Port, "blue"), + Endpoint("second", second.Port, "green") + ])); + await WaitUntilAsync(() => ((SharpLinkClient)client).ReadyConnectionCount == 2, TimeSpan.FromSeconds(3)); + Ensure(((SharpLinkClient)client).ReadyConnectionCount == 2, "added endpoint should become ready"); + Ensure(factoryCreates == 2, "only the added endpoint should create a factory"); + + selector.Zone = "blue"; + Ensure(await service.GetEndpointIdAsync() == "first", "blue selector before attributes update"); + resolver.Publish(new SharpLinkEndpointSnapshot(3, + [ + Endpoint("first", first.Port, "red"), + Endpoint("second", second.Port, "blue") + ])); + await WaitUntilAsync(async () => await service.GetEndpointIdAsync() == "second", TimeSpan.FromSeconds(3)); + Ensure(factoryCreates == 2, "attributes-only update must not create a new endpoint factory"); + + resolver.Publish(new SharpLinkEndpointSnapshot(4, [Endpoint("second", second.Port, "blue")])); + await WaitUntilAsync(() => ((SharpLinkClient)client).ReadyConnectionCount == 1, TimeSpan.FromSeconds(3)); + Ensure(await service.GetEndpointIdAsync() == "second", "removed endpoint must leave the candidate set"); + + resolver.Publish(new SharpLinkEndpointSnapshot(5, [Endpoint("second", replacement.Port, "blue")])); + await WaitUntilAsync(async () => await service.GetEndpointIdAsync() == "replacement", TimeSpan.FromSeconds(4)); + Ensure(factoryCreates == 3, "address change must create exactly one new generation factory"); + } + finally + { + await client.DisposeAsync(); + } + + Ensure(resolver.DisposeCount == 1, "dynamic client should dispose its resolver exactly once"); + } + + [Test] + [NotInParallel] + public async Task EmptyDynamicTopologyShouldRecoverWhenTheResolverPublishesAnEndpoint() + { + await using var server = await TcpServerScope.StartAsync("recovered"); + var resolver = new ControllableResolver(new SharpLinkEndpointSnapshot(1, [])); + await using var client = SharpClientBuilder.Create() + .UseSerializer(MemoryPackCodec.Resolver) + .UseEndpointResolver(resolver, SharpLinkTransportFactories.Sockets()) + .Build(); + + await client.ConnectAsync(); + Ensure(((SharpLinkClient)client).ReadyConnectionCount == 0, "empty topology has no ready connection"); + var repeatedConnect = client.ConnectAsync(); + Ensure(repeatedConnect.IsCompletedSuccessfully, + "repeated ConnectAsync on an accepted empty topology must complete without waiting for recovery"); + await repeatedConnect; + resolver.Publish(new SharpLinkEndpointSnapshot(2, [Endpoint("recovered", server.Port, "blue")])); + await WaitUntilAsync(() => ((SharpLinkClient)client).ReadyConnectionCount == 1, TimeSpan.FromSeconds(3)); + Ensure(await client.Get().GetEndpointIdAsync() == "recovered", "topology recovery RPC"); + } + + [Test] + [NotInParallel] + public async Task DynamicEndpointRemovalShouldDrainAnAcceptedStreamAndRouteNewCallsElsewhere() + { + await using var first = await TcpServerScope.StartAsync("first"); + await using var second = await TcpServerScope.StartAsync("second"); + var resolver = new ControllableResolver(new SharpLinkEndpointSnapshot(1, + [ + Endpoint("first", first.Port, "blue"), + Endpoint("second", second.Port, "green") + ])); + await using var client = SharpClientBuilder.Create() + .UseSerializer(MemoryPackCodec.Resolver) + .UseHeartbeat(TimeSpan.FromMilliseconds(100), TimeSpan.FromMilliseconds(500)) + .UseEndpointResolver(resolver, SharpLinkTransportFactories.Sockets()) + .UseEndpointSelector(new IdSelector("first")) + .Build(); + + await client.ConnectAsync(); + var service = client.Get(); + await using var stream = service.SlowRangeAsync(3, 80, CancellationToken.None).GetAsyncEnumerator(); + Ensure(await stream.MoveNextAsync() && stream.Current == 0, "first stream item"); + + resolver.Publish(new SharpLinkEndpointSnapshot(2, [Endpoint("second", second.Port, "green")])); + await WaitUntilAsync(() => ((SharpLinkClient)client).ReadyConnectionCount == 1, TimeSpan.FromSeconds(3)); + Ensure(await service.GetEndpointIdAsync() == "second", "new call after endpoint removal"); + Ensure(await stream.MoveNextAsync() && stream.Current == 1, "draining stream second item"); + Ensure(await stream.MoveNextAsync() && stream.Current == 2, "draining stream third item"); + Ensure(!await stream.MoveNextAsync(), "draining stream completion"); + } + + [Test] + [NotInParallel] + public async Task CustomDynamicSelectorShouldRejectTheOnlyNonMatchingReadyEndpoint() + { + await using var east = await TcpServerScope.StartAsync("east"); + await using var west = await TcpServerScope.StartAsync("west"); + var resolver = new ControllableResolver(new SharpLinkEndpointSnapshot(1, + [ + Endpoint("east", east.Port, "east"), + Endpoint("west", west.Port, "west") + ])); + await using var client = SharpClientBuilder.Create() + .UseSerializer(MemoryPackCodec.Resolver) + .UseEndpointResolver(resolver, SharpLinkTransportFactories.Sockets()) + .UseEndpointSelector(new ZoneSelector("west")) + .UseCluster(options => + { + options.MinReadyEndpoints = 2; + options.MaxConnections = 2; + options.MaxConnectionsPerEndpoint = 1; + }) + .Build(); + + await client.ConnectAsync(); + await WaitUntilAsync(() => ((SharpLinkClient)client).ReadyConnectionCount == 2, TimeSpan.FromSeconds(3)); + resolver.Publish(new SharpLinkEndpointSnapshot(2, [Endpoint("east", east.Port, "east")])); + await WaitUntilAsync(() => ((SharpLinkClient)client).ReadyConnectionCount == 1, TimeSpan.FromSeconds(3)); + + var exception = await CaptureSharpLinkException( + client.Get().GetEndpointIdAsync().AsTask()); + Ensure(exception.Code == SharpLinkErrorCode.FailedPrecondition, + "a strict dynamic selector must not be bypassed for one candidate"); + } + + [Test] + [NotInParallel] + public async Task RejectedDynamicFactoryReuseMustKeepTheLastGoodFactoryAlive() + { + await using var first = await TcpServerScope.StartAsync("first"); + await using var replacement = await TcpServerScope.StartAsync("replacement"); + var resolver = new ControllableResolver(new SharpLinkEndpointSnapshot(1, [Endpoint("first", first.Port, "blue")])); + var sockets = SharpLinkTransportFactories.Sockets(); + TrackingTransportFactory? factory = null; + var factoryCreates = 0; + var client = SharpClientBuilder.Create() + .UseSerializer(MemoryPackCodec.Resolver) + .UseEndpointResolver( + resolver, + endpoint => + { + Interlocked.Increment(ref factoryCreates); + return factory ??= new TrackingTransportFactory(sockets(endpoint)); + }) + .Build(); + + try + { + await client.ConnectAsync(); + var service = client.Get(); + Ensure(await service.GetEndpointIdAsync() == "first", "initial dynamic endpoint"); + + resolver.Publish(new SharpLinkEndpointSnapshot(2, [Endpoint("replacement", replacement.Port, "green")])); + await WaitUntilAsync(() => Volatile.Read(ref factoryCreates) == 2, TimeSpan.FromSeconds(3)); + + Ensure(factory is not null && factory.DisposeCount == 0, + "rejected snapshot must not dispose the last-good factory reference"); + Ensure(await service.GetEndpointIdAsync() == "first", + "rejected snapshot must retain the last-good endpoint"); + } + finally + { + await client.DisposeAsync(); + } + + Ensure(factory is not null && factory.DisposeCount == 1, + "last-good factory must be released exactly once during client stop"); + } + + [Test] + [NotInParallel] + public async Task FailedInitialDynamicTopologyShouldAllowConnectToWaitForRecovery() + { + var resolver = new ControllableResolver(new SharpLinkEndpointSnapshot(1, [Endpoint("failed", 1, "red")])); + var factory = new FailingConnectFactory(); + var client = SharpClientBuilder.Create() + .UseEndpointResolver(resolver, _ => factory) + .Build(); + + try + { + var first = await CaptureSharpLinkException(client.ConnectAsync().AsTask()); + using var recoveryCancellation = new CancellationTokenSource(TimeSpan.FromMilliseconds(100)); + var recovery = client.ConnectAsync(recoveryCancellation.Token).AsTask(); + await Task.Delay(20); + + Ensure(first.Code == SharpLinkErrorCode.Unavailable, "initial failed dynamic topology error"); + Ensure(!recovery.IsCompleted, + "failed initial dynamic topology must wait for recovery instead of replaying the stale failure"); + await CaptureCancellation(recovery); + Ensure(factory.ConnectCount != 0, "resolved endpoint connection must have been attempted"); + } + finally + { + await client.DisposeAsync(); + } + } + + [Test] + [NotInParallel] + public async Task FailedInitialDynamicDialShouldProbeLaterEndpointsWithoutWaitingForASlowSibling() + { + await using var healthy = await TcpServerScope.StartAsync("healthy"); + var resolver = new ControllableResolver(new SharpLinkEndpointSnapshot( + 1, + [Endpoint("failed", 1, "red"), Endpoint("blocked", 2, "red"), Endpoint("healthy", healthy.Port, "green")])); + var blocking = new BlockingConnectFactory(); + var failing = new FailingConnectFactory(); + var sockets = SharpLinkTransportFactories.Sockets(); + var client = SharpClientBuilder.Create() + .UseSerializer(MemoryPackCodec.Resolver) + .UseEndpointResolver(resolver, endpoint => endpoint.Id switch + { + "failed" => failing, + "blocked" => blocking, + _ => sockets(endpoint) + }) + .UseCluster(options => + { + options.MinReadyEndpoints = 2; + options.MaxConnections = 4; + options.MaxConnectionsPerEndpoint = 1; + }) + .Build(); + + try + { + var connect = client.ConnectAsync().AsTask(); + await blocking.Entered.WaitAsync(TimeSpan.FromSeconds(2)); + + await connect.WaitAsync(TimeSpan.FromSeconds(2)); + Ensure(((SharpLinkClient)client).ReadyConnectionCount == 1, + "a failed resolver-backed initial dial must immediately probe a later healthy endpoint"); + } + finally + { + blocking.Release(); + await client.DisposeAsync(); + } + } + + [Test] + [NotInParallel] + public async Task DynamicRecoveryToAnEmptyTopologyShouldReleaseConnectWaiters() + { + var resolver = new FailingThenEmptyResolver(); + await using var client = SharpClientBuilder.Create() + .UseEndpointResolver(resolver, _ => new FailingConnectFactory()) + .Build(); + + var initial = await CaptureSharpLinkException(client.ConnectAsync().AsTask()); + Ensure(initial.Code == SharpLinkErrorCode.Unavailable, "initial resolver failure error code"); + await resolver.EmptyResolveStarted.WaitAsync(TimeSpan.FromSeconds(2)); + + var recovery = client.ConnectAsync().AsTask(); + await Task.Delay(20); + Ensure(!recovery.IsCompleted, "ConnectAsync must wait for the resolver recovery result"); + + resolver.ReleaseEmptyTopology(); + await recovery.WaitAsync(TimeSpan.FromSeconds(2)); + Ensure(((SharpLinkClient)client).ReadyConnectionCount == 0, + "an accepted empty topology completes recovery without fabricating a ready connection"); + } + + [Test] + [NotInParallel] + public async Task FailedInitialDynamicDialShouldReconnectWithoutANewerResolverVersion() + { + await using var server = await TcpServerScope.StartAsync("recovered"); + var resolver = new ControllableResolver(new SharpLinkEndpointSnapshot(1, [Endpoint("recovered", server.Port, "green")])); + var factory = new FailOnceConnectFactory(SharpLinkTransportFactories.Sockets()(Endpoint("recovered", server.Port, "green"))); + await using var client = SharpClientBuilder.Create() + .UseSerializer(MemoryPackCodec.Resolver) + .UseEndpointResolver(resolver, _ => factory) + .Build(); + + var initial = await CaptureSharpLinkException(client.ConnectAsync().AsTask()); + Ensure(initial.Code == SharpLinkErrorCode.Unavailable, "initial failed dial reports unavailable"); + + var recovery = client.ConnectAsync().AsTask(); + var completed = await Task.WhenAny(recovery, Task.Delay(20)); + Ensure(!ReferenceEquals(completed, recovery), + "repeated ConnectAsync must wait for recovery instead of replaying the initial dial failure"); + await recovery.WaitAsync(TimeSpan.FromSeconds(3)); + Ensure(factory.ConnectCount >= 2, "same accepted topology must reconnect after the first dial fails"); + Ensure(await client.Get().GetEndpointIdAsync() == "recovered", + "same-version dynamic topology recovers after its reconnect worker succeeds"); + } + + [Test] + [NotInParallel] + public async Task RejectedDynamicSnapshotCleanupShouldContinueAfterFactoryDisposalFailure() + { + await using var first = await TcpServerScope.StartAsync("first"); + var resolver = new ControllableResolver(new SharpLinkEndpointSnapshot(1, [Endpoint("first", first.Port, "blue")])); + var sockets = SharpLinkTransportFactories.Sockets(); + TrackingTransportFactory? goodFactory = null; + var throwingFactory = new ThrowingDisposeFactory(); + var remainingFactory = new FailingConnectFactory(); + var factoryCreates = 0; + var client = SharpClientBuilder.Create() + .UseSerializer(MemoryPackCodec.Resolver) + .UseEndpointResolver( + resolver, + endpoint => + { + Interlocked.Increment(ref factoryCreates); + return endpoint.Id switch + { + "first" => goodFactory ??= new TrackingTransportFactory(sockets(endpoint)), + "throwing" => throwingFactory, + "remaining" => remainingFactory, + _ => throw new InvalidOperationException("test factory construction failure") + }; + }) + .Build(); + + try + { + await client.ConnectAsync(); + var service = client.Get(); + Ensure(await service.GetEndpointIdAsync() == "first", "initial retained topology"); + + resolver.Publish(new SharpLinkEndpointSnapshot(2, + [ + Endpoint("throwing", first.Port, "red"), + Endpoint("remaining", first.Port, "green"), + Endpoint("factory-failure", first.Port, "yellow") + ])); + await WaitUntilAsync( + () => Volatile.Read(ref factoryCreates) == 4 && + throwingFactory.DisposeCount == 1 && + remainingFactory.DisposeCount == 1, + TimeSpan.FromSeconds(3)); + + Ensure(goodFactory is not null && goodFactory.DisposeCount == 0, + "failed cleanup must not affect the last-good factory"); + Ensure(await service.GetEndpointIdAsync() == "first", + "failed snapshot cleanup must retain the last-good topology"); + } + finally + { + await client.DisposeAsync(); + } + + Ensure(goodFactory is not null && goodFactory.DisposeCount == 1, + "last-good factory disposal after rejected snapshot cleanup"); + } + + [Test] + [NotInParallel] + public async Task DynamicReplacementShouldWaitForExcessRetiringConnectionsToDrain() + { + await using var first = await TcpServerScope.StartAsync("first"); + await using var second = await TcpServerScope.StartAsync("second"); + var resolver = new ControllableResolver(new SharpLinkEndpointSnapshot(1, [Endpoint("first", first.Port, "blue")])); + await using var client = SharpClientBuilder.Create() + .UseSerializer(MemoryPackCodec.Resolver) + .UseHeartbeat(TimeSpan.FromMilliseconds(100), TimeSpan.FromMilliseconds(500)) + .UseEndpointResolver(resolver, SharpLinkTransportFactories.Sockets()) + .UseCluster(options => + { + options.MinReadyEndpoints = 1; + options.MaxConnections = 2; + options.MaxConnectionsPerEndpoint = 1; + options.MaxRetiringConnections = 0; + }) + .Build(); + + await client.ConnectAsync(); + var service = client.Get(); + await using var stream = service.SlowRangeAsync(3, 100, CancellationToken.None).GetAsyncEnumerator(); + Ensure(await stream.MoveNextAsync() && stream.Current == 0, "initial active stream"); + + resolver.Publish(new SharpLinkEndpointSnapshot(2, [Endpoint("second", second.Port, "green")])); + await Task.Delay(150); + Ensure(((SharpLinkClient)client).ReadyConnectionCount == 0, + "retiring budget must defer replacement connection creation while an old stream is active"); + Ensure(await stream.MoveNextAsync() && stream.Current == 1, + "budget pressure must not abort the already accepted stream"); + Ensure(await stream.MoveNextAsync() && stream.Current == 2, + "retired stream must remain bound through its final item"); + Ensure(!await stream.MoveNextAsync(), "retired stream completion after replacement"); + await WaitUntilAsync(async () => await service.GetEndpointIdAsync() == "second", TimeSpan.FromSeconds(3)); + Ensure(((SharpLinkClient)client).ReadyConnectionCount == 1, + "replacement must resume once the excess retiring stream has drained"); + } + + [Test] + [NotInParallel] + public async Task DynamicReconnectShouldProbeHealthyEndpointsAfterAFailingEndpoint() + { + await using var first = await TcpServerScope.StartAsync("first"); + await using var second = await TcpServerScope.StartAsync("second"); + var resolver = new ControllableResolver(new SharpLinkEndpointSnapshot(1, + [ + Endpoint("bad", 1, "red"), + Endpoint("first", first.Port, "green"), + Endpoint("second", second.Port, "blue") + ])); + var failing = new FailingConnectFactory(); + var sockets = SharpLinkTransportFactories.Sockets(); + await using var client = SharpClientBuilder.Create() + .UseSerializer(MemoryPackCodec.Resolver) + .UseEndpointResolver(resolver, endpoint => endpoint.Id == "bad" ? failing : sockets(endpoint)) + .UseCluster(options => + { + options.MinReadyEndpoints = 2; + options.MaxConnections = 2; + options.MaxConnectionsPerEndpoint = 1; + }) + .Build(); + + await client.ConnectAsync(); + await WaitUntilAsync(() => ((SharpLinkClient)client).ReadyConnectionCount == 2, TimeSpan.FromSeconds(3)); + + Ensure(failing.ConnectCount != 0, "the failing endpoint should have been probed"); + Ensure(((SharpLinkClient)client).ReadyConnectionCount == 2, + "a persistently failing endpoint must not monopolize the remaining ready target"); + } + + [Test] + [NotInParallel] + public async Task DynamicStopShouldWaitForAnInitialConnectThatIgnoresCancellation() + { + var resolver = new ControllableResolver(new SharpLinkEndpointSnapshot(1, [Endpoint("blocked", 1, "red")])); + var blocking = new BlockingConnectFactory(); + var client = SharpClientBuilder.Create() + .UseEndpointResolver(resolver, _ => blocking) + .Build(); + + try + { + var connect = client.ConnectAsync().AsTask(); + await blocking.Entered.WaitAsync(TimeSpan.FromSeconds(2)); + + var stop = client.StopAsync().AsTask(); + await Task.Delay(100); + Ensure(!stop.IsCompleted, "dynamic stop must wait for the initial connect worker"); + + blocking.Release(); + await CaptureCancellation(connect); + await stop.WaitAsync(TimeSpan.FromSeconds(2)); + Ensure(((SharpLinkClient)client).State == SharpLinkConnectionState.Stopped, + "dynamic cluster must stop after the initial connect worker exits"); + } + finally + { + blocking.Release(); + await client.DisposeAsync(); + } + } + + [Test] + [NotInParallel] + public async Task InitialDynamicConnectShouldCompleteWhenAReplacementTopologyBecomesReady() + { + await using var replacement = await TcpServerScope.StartAsync("replacement"); + var resolver = new ControllableResolver(new SharpLinkEndpointSnapshot(1, [Endpoint("blocked", 1, "red")])); + var blocking = new BlockingConnectFactory(); + var sockets = SharpLinkTransportFactories.Sockets(); + var client = SharpClientBuilder.Create() + .UseSerializer(MemoryPackCodec.Resolver) + .UseEndpointResolver(resolver, endpoint => endpoint.Id == "blocked" ? blocking : sockets(endpoint)) + .UseCluster(options => + { + options.MinReadyEndpoints = 1; + options.MaxConnections = 2; + options.MaxConnectionsPerEndpoint = 1; + }) + .Build(); + + try + { + var initial = client.ConnectAsync().AsTask(); + await blocking.Entered.WaitAsync(TimeSpan.FromSeconds(2)); + resolver.Publish(new SharpLinkEndpointSnapshot(2, [Endpoint("replacement", replacement.Port, "green")])); + + await initial.WaitAsync(TimeSpan.FromSeconds(3)); + Ensure(await client.Get().GetEndpointIdAsync() == "replacement", + "initial ConnectAsync must observe a ready replacement instead of waiting for a retired dial"); + } + finally + { + blocking.Release(); + await client.DisposeAsync(); + } + } + + [Test] + [NotInParallel] + public async Task RetiredDynamicDialsShouldContinueToConsumeTheConnectionBudget() + { + await using var replacement = await TcpServerScope.StartAsync("replacement"); + var resolver = new ControllableResolver(new SharpLinkEndpointSnapshot(1, [Endpoint("blocked", 1, "red")])); + var blocking = new BlockingConnectFactory(); + var sockets = SharpLinkTransportFactories.Sockets(); + var replacementFactory = new CountingConnectFactory(sockets(Endpoint("replacement", replacement.Port, "green"))); + var client = SharpClientBuilder.Create() + .UseSerializer(MemoryPackCodec.Resolver) + .UseEndpointResolver(resolver, endpoint => endpoint.Id == "blocked" ? blocking : replacementFactory) + .UseCluster(options => + { + options.MinReadyEndpoints = 1; + options.MaxConnections = 1; + options.MaxConnectionsPerEndpoint = 1; + }) + .Build(); + + try + { + var initial = client.ConnectAsync().AsTask(); + await blocking.Entered.WaitAsync(TimeSpan.FromSeconds(2)); + resolver.Publish(new SharpLinkEndpointSnapshot(2, [Endpoint("replacement", replacement.Port, "green")])); + await Task.Delay(100); + + Ensure(replacementFactory.ConnectCount == 0, + "a retired in-flight dial must keep the sole connection budget reserved"); + + blocking.Release(); + var initialFailure = await CaptureSharpLinkException(initial); + Ensure(initialFailure.Code == SharpLinkErrorCode.Unavailable, "retired initial dial failure result"); + await WaitUntilAsync( + () => replacementFactory.ConnectCount == 1 && ((SharpLinkClient)client).ReadyConnectionCount == 1, + TimeSpan.FromSeconds(3)); + } + finally + { + blocking.Release(); + await client.DisposeAsync(); + } + } + + [Test] + [NotInParallel] + public async Task ConnectAfterDynamicClusterDisconnectShouldAwaitRecovery() + { + await using var first = await TcpServerScope.StartAsync("first"); + var resolver = new ControllableResolver(new SharpLinkEndpointSnapshot(1, + [ + Endpoint("first", first.Port, "blue"), + Endpoint("unavailable", 1, "red") + ])); + var sockets = SharpLinkTransportFactories.Sockets(); + var blocking = new BlockAfterFirstConnectFactory(sockets(Endpoint("first", first.Port, "blue"))); + var unavailable = new FailingConnectFactory(); + await using var client = SharpClientBuilder.Create() + .UseSerializer(MemoryPackCodec.Resolver) + .UseEndpointResolver(resolver, endpoint => endpoint.Id == "first" ? blocking : unavailable) + .UseCluster(options => + { + options.MinReadyEndpoints = 1; + options.MaxConnections = 2; + options.MaxConnectionsPerEndpoint = 1; + }) + .Build(); + + await client.ConnectAsync(); + await first.StopAsync(); + await WaitUntilAsync(() => ((SharpLinkClient)client).ReadyConnectionCount == 0, TimeSpan.FromSeconds(3)); + await blocking.Entered.WaitAsync(TimeSpan.FromSeconds(2)); + + var reconnect = client.ConnectAsync().AsTask(); + var completed = await Task.WhenAny(reconnect, Task.Delay(100)); + Ensure(!ReferenceEquals(completed, reconnect), + "dynamic ConnectAsync must wait for a new ready connection instead of returning startup success"); + } + + [Test] + [NotInParallel] + public async Task InitialDynamicDialReservationsShouldPreventSurplusTargetFill() + { + await using var first = await TcpServerScope.StartAsync("first"); + var resolver = new ControllableResolver(new SharpLinkEndpointSnapshot(1, + [ + Endpoint("first", first.Port, "blue"), + Endpoint("blocked", 1, "green"), + Endpoint("surplus", 2, "red") + ])); + var blocking = new BlockingConnectFactory(); + var surplus = new FailingConnectFactory(); + var sockets = SharpLinkTransportFactories.Sockets(); + var client = SharpClientBuilder.Create() + .UseSerializer(MemoryPackCodec.Resolver) + .UseEndpointResolver(resolver, endpoint => endpoint.Id switch + { + "first" => sockets(endpoint), + "blocked" => blocking, + _ => surplus + }) + .UseCluster(options => + { + options.MinReadyEndpoints = 2; + options.MaxConnections = 3; + options.MaxConnectionsPerEndpoint = 1; + }) + .Build(); + + try + { + var connect = client.ConnectAsync().AsTask(); + await blocking.Entered.WaitAsync(TimeSpan.FromSeconds(2)); + await connect.WaitAsync(TimeSpan.FromSeconds(2)); + await Task.Delay(200); + + Ensure(surplus.ConnectCount == 0, + "a pending current-generation initial dial must reserve the remaining ready target"); + } + finally + { + blocking.Release(); + await client.DisposeAsync(); + } + } + + [Test] + [NotInParallel] + public async Task ResolverWatchEndAndFailureShouldRetryAndRetainTheLastGoodTopology() + { + await using var first = await TcpServerScope.StartAsync("first"); + await using var recovered = await TcpServerScope.StartAsync("recovered"); + var resolver = new RestartingResolver( + new SharpLinkEndpointSnapshot(1, [Endpoint("first", first.Port, "blue")]), + new SharpLinkEndpointSnapshot(2, [Endpoint("recovered", recovered.Port, "green")])); + await using var client = SharpClientBuilder.Create() + .UseSerializer(MemoryPackCodec.Resolver) + .UseEndpointResolver(resolver, SharpLinkTransportFactories.Sockets()) + .Build(); + + await client.ConnectAsync(); + Ensure(await client.Get().GetEndpointIdAsync() == "first", "initial last-good topology"); + await WaitUntilAsync(async () => await client.Get().GetEndpointIdAsync() == "recovered", + TimeSpan.FromSeconds(4)); + Ensure(resolver.ResolveCount >= 3, "watch end should retry Resolve after one resolver failure"); + } + + [Test] + [NotInParallel] + public async Task DnsEndpointHelperShouldResolveLocalhostAndPreserveHostnameAuthority() + { + await using var server = await TcpServerScope.StartAsync("dns"); + await using var client = SharpClientBuilder.Create() + .UseSerializer(MemoryPackCodec.Resolver) + .UseDnsEndpoints( + "localhost", + server.Port, + SharpLinkTransportFactories.Sockets(), + options => + { + options.AddressFamily = AddressFamily.InterNetwork; + options.RefreshInterval = TimeSpan.FromMilliseconds(10); + options.MinimumRefreshInterval = TimeSpan.FromMilliseconds(1); + }) + .Build(); + + await client.ConnectAsync(); + Ensure(await client.Get().GetEndpointIdAsync() == "dns", "DNS discovery RPC"); + } + + private static SharpLinkEndpoint Endpoint(string id, int port, string zone) => new() + { + Id = id, + Address = new SharpLinkTcpAddress(IPAddress.Loopback.ToString(), port), + Attributes = new Dictionary { ["zone"] = zone } + }; + + private static async Task WaitUntilAsync(Func condition, TimeSpan timeout) + { + var deadline = Stopwatch.GetTimestamp() + (long)(timeout.TotalSeconds * Stopwatch.Frequency); + while (!condition() && Stopwatch.GetTimestamp() < deadline) + await Task.Delay(20); + if (!condition()) + throw new TimeoutException("Dynamic topology did not reach the expected state."); + } + + private static async Task WaitUntilAsync(Func> condition, TimeSpan timeout) + { + var deadline = Stopwatch.GetTimestamp() + (long)(timeout.TotalSeconds * Stopwatch.Frequency); + while (Stopwatch.GetTimestamp() < deadline) + { + try + { + if (await condition()) + return; + } + catch (SharpLinkException) + { + // A generation replacement deliberately has a short interval with no ready endpoint. + } + await Task.Delay(20); + } + throw new TimeoutException("Dynamic topology did not reach the expected state."); + } + + private static void Ensure(bool condition, string message) + { + if (!condition) + throw new Exception(message); + } + + private static async Task CaptureSharpLinkException(Task task) + { + try + { + await task; + throw new Exception("expected SharpLinkException"); + } + catch (SharpLinkException exception) + { + return exception; + } + } + + private static async Task CaptureCancellation(Task task) + { + try + { + await task; + throw new Exception("expected cancellation"); + } + catch (OperationCanceledException) + { + } + } + + private sealed class ControllableResolver(SharpLinkEndpointSnapshot initial) : ISharpLinkEndpointResolver + { + private readonly Channel _snapshots = Channel.CreateUnbounded(); + private int _disposeCount; + + public int DisposeCount => Volatile.Read(ref _disposeCount); + + public ValueTask ResolveAsync(CancellationToken cancellationToken) + => ValueTask.FromResult(initial); + + public async IAsyncEnumerable WatchAsync( + [EnumeratorCancellation] CancellationToken cancellationToken) + { + await foreach (var snapshot in _snapshots.Reader.ReadAllAsync(cancellationToken)) + yield return snapshot; + } + + public void Publish(SharpLinkEndpointSnapshot snapshot) + => _snapshots.Writer.TryWrite(snapshot); + + public ValueTask DisposeAsync() + { + if (Interlocked.Increment(ref _disposeCount) == 1) + _snapshots.Writer.TryComplete(); + return ValueTask.CompletedTask; + } + } + + private sealed class TrackingTransportFactory(IClientTransportFactory inner) : IClientTransportFactory + { + private int _disposeCount; + + public int DisposeCount => Volatile.Read(ref _disposeCount); + + public ValueTask ConnectAsync(CancellationToken cancellationToken = default) + => inner.ConnectAsync(cancellationToken); + + public async ValueTask DisposeAsync() + { + Interlocked.Increment(ref _disposeCount); + await inner.DisposeAsync().ConfigureAwait(false); + } + } + + private sealed class FailingConnectFactory : IClientTransportFactory + { + private int _connectCount; + private int _disposeCount; + + public int ConnectCount => Volatile.Read(ref _connectCount); + public int DisposeCount => Volatile.Read(ref _disposeCount); + + public ValueTask ConnectAsync(CancellationToken cancellationToken = default) + { + Interlocked.Increment(ref _connectCount); + return ValueTask.FromException(new InvalidOperationException("test transport failure")); + } + + public ValueTask DisposeAsync() + { + Interlocked.Increment(ref _disposeCount); + return ValueTask.CompletedTask; + } + } + + private sealed class FailOnceConnectFactory(IClientTransportFactory inner) : IClientTransportFactory + { + private int _connectCount; + + public int ConnectCount => Volatile.Read(ref _connectCount); + + public ValueTask ConnectAsync(CancellationToken cancellationToken = default) + => Interlocked.Increment(ref _connectCount) == 1 + ? ValueTask.FromException(new InvalidOperationException("test first dynamic dial failure")) + : inner.ConnectAsync(cancellationToken); + + public ValueTask DisposeAsync() => inner.DisposeAsync(); + } + + private sealed class BlockingConnectFactory : IClientTransportFactory + { + private readonly TaskCompletionSource _entered = new(TaskCreationOptions.RunContinuationsAsynchronously); + private readonly TaskCompletionSource _release = new(TaskCreationOptions.RunContinuationsAsynchronously); + + public Task Entered => _entered.Task; + + public ValueTask ConnectAsync(CancellationToken cancellationToken = default) + { + _entered.TrySetResult(); + return AwaitReleaseAsync(); + } + + public void Release() => _release.TrySetResult(); + + public ValueTask DisposeAsync() => ValueTask.CompletedTask; + + private async ValueTask AwaitReleaseAsync() + { + await _release.Task.ConfigureAwait(false); + throw new OperationCanceledException(); + } + } + + private sealed class BlockAfterFirstConnectFactory(IClientTransportFactory inner) : IClientTransportFactory + { + private readonly TaskCompletionSource _entered = new(TaskCreationOptions.RunContinuationsAsynchronously); + private int _connectCount; + + public Task Entered => _entered.Task; + + public async ValueTask ConnectAsync(CancellationToken cancellationToken = default) + { + if (Interlocked.Increment(ref _connectCount) == 1) + return await inner.ConnectAsync(cancellationToken).ConfigureAwait(false); + + _entered.TrySetResult(); + await Task.Delay(Timeout.InfiniteTimeSpan, cancellationToken).ConfigureAwait(false); + throw new UnreachableException(); + } + + public ValueTask DisposeAsync() => inner.DisposeAsync(); + } + + private sealed class CountingConnectFactory(IClientTransportFactory inner) : IClientTransportFactory + { + private int _connectCount; + + public int ConnectCount => Volatile.Read(ref _connectCount); + + public async ValueTask ConnectAsync(CancellationToken cancellationToken = default) + { + Interlocked.Increment(ref _connectCount); + return await inner.ConnectAsync(cancellationToken).ConfigureAwait(false); + } + + public ValueTask DisposeAsync() => inner.DisposeAsync(); + } + + private sealed class ThrowingDisposeFactory : IClientTransportFactory + { + private int _disposeCount; + + public int DisposeCount => Volatile.Read(ref _disposeCount); + + public ValueTask ConnectAsync(CancellationToken cancellationToken = default) + => ValueTask.FromException(new InvalidOperationException("test transport failure")); + + public ValueTask DisposeAsync() + { + Interlocked.Increment(ref _disposeCount); + return ValueTask.FromException(new InvalidOperationException("test disposal failure")); + } + } + + private sealed class ZoneSelector(string zone) : ISharpLinkEndpointSelector + { + private string _zone = zone; + + public string Zone + { + get => Volatile.Read(ref _zone); + set => Volatile.Write(ref _zone, value); + } + + public int Select(in SharpLinkEndpointSelectionContext context) + { + var zone = Zone; + for (var index = 0; index < context.Count; index++) + { + if ((context.ExcludedMask & (1UL << index)) == 0 && + context[index].Endpoint.Attributes.TryGetValue("zone", out var candidateZone) && + candidateZone == zone) + { + return index; + } + } + return -1; + } + } + + private sealed class IdSelector(string id) : ISharpLinkEndpointSelector + { + public int Select(in SharpLinkEndpointSelectionContext context) + { + for (var index = 0; index < context.Count; index++) + { + if ((context.ExcludedMask & (1UL << index)) == 0 && context[index].Endpoint.Id == id) + return index; + } + for (var index = 0; index < context.Count; index++) + if ((context.ExcludedMask & (1UL << index)) == 0) + return index; + return -1; + } + } + + private sealed class RestartingResolver( + SharpLinkEndpointSnapshot initial, + SharpLinkEndpointSnapshot recovered) : ISharpLinkEndpointResolver + { + private int _resolveCount; + + public int ResolveCount => Volatile.Read(ref _resolveCount); + + public ValueTask ResolveAsync(CancellationToken cancellationToken) + { + var count = Interlocked.Increment(ref _resolveCount); + return count == 2 + ? ValueTask.FromException(new InvalidOperationException("transient resolver failure")) + : ValueTask.FromResult(count >= 3 ? recovered : initial); + } + + public async IAsyncEnumerable WatchAsync( + [EnumeratorCancellation] CancellationToken cancellationToken) + { + await Task.Yield(); + yield break; + } + + public ValueTask DisposeAsync() => ValueTask.CompletedTask; + } + + private sealed class FailingThenEmptyResolver : ISharpLinkEndpointResolver + { + private readonly TaskCompletionSource _emptyResolveStarted = + new(TaskCreationOptions.RunContinuationsAsynchronously); + private readonly TaskCompletionSource _releaseEmptyTopology = + new(TaskCreationOptions.RunContinuationsAsynchronously); + private int _resolveCount; + + public Task EmptyResolveStarted => _emptyResolveStarted.Task; + + public async ValueTask ResolveAsync(CancellationToken cancellationToken) + { + if (Interlocked.Increment(ref _resolveCount) == 1) + throw new InvalidOperationException("initial resolver failure"); + + _emptyResolveStarted.TrySetResult(); + await _releaseEmptyTopology.Task.WaitAsync(cancellationToken).ConfigureAwait(false); + return new SharpLinkEndpointSnapshot(1, []); + } + + public async IAsyncEnumerable WatchAsync( + [EnumeratorCancellation] CancellationToken cancellationToken) + { + await Task.Delay(Timeout.InfiniteTimeSpan, cancellationToken).ConfigureAwait(false); + yield break; + } + + public void ReleaseEmptyTopology() => _releaseEmptyTopology.TrySetResult(); + + public ValueTask DisposeAsync() + { + _releaseEmptyTopology.TrySetResult(); + return ValueTask.CompletedTask; + } + } + + private sealed class TcpServerScope : IAsyncDisposable + { + private readonly ISharpLinkServer _server; + private readonly CancellationTokenSource _cancellation = new(); + private readonly Task _runTask; + private int _stopped; + + private TcpServerScope(ISharpLinkServer server, int port) + { + _server = server; + Port = port; + _runTask = Task.Run(() => _server.RunAsync(_cancellation.Token).AsTask(), CancellationToken.None); + } + + public int Port { get; } + + public async ValueTask StopAsync() + { + if (Interlocked.Exchange(ref _stopped, 1) != 0) + return; + await _server.StopAsync(TimeSpan.Zero); + await _cancellation.CancelAsync(); + await Task.WhenAny(_runTask, Task.Delay(1000)); + } + + public static Task StartAsync(string endpointId) + { + var builder = SharpLinkServerBuilder.Create() + .UseTcp(0, IPAddress.Loopback.ToString()) + .UseSerializer(MemoryPackCodec.Resolver) + .UseHeartbeat(TimeSpan.FromMilliseconds(100), TimeSpan.FromMilliseconds(500)); + builder.ReplaceService(new ConnectionBehaviorService { EndpointId = endpointId }); + var port = ((IPEndPoint)builder.Transport!.LocalEndPoint!).Port; + return Task.FromResult(new TcpServerScope(builder.Build(), port)); + } + + public async ValueTask DisposeAsync() + { + await StopAsync(); + await _server.DisposeAsync(); + _cancellation.Dispose(); + } + } +} diff --git a/test/SharpLink.IntegrationTests/StaticEndpointIntegrationTests.cs b/test/SharpLink.IntegrationTests/StaticEndpointIntegrationTests.cs new file mode 100644 index 0000000..3a73a1c --- /dev/null +++ b/test/SharpLink.IntegrationTests/StaticEndpointIntegrationTests.cs @@ -0,0 +1,1033 @@ +namespace SharpLink.IntegrationTests; + +public sealed class StaticEndpointIntegrationTests +{ + [Test] + public async Task StaticTcpEndpointsShouldConnectAndContinueWhenOneEndpointStops() + { + await using var first = await TcpServerScope.StartAsync(); + await using var second = await TcpServerScope.StartAsync(); + await using var client = SharpClientBuilder.Create() + .UseSerializer(MemoryPackCodec.Resolver) + .UseHeartbeat(TimeSpan.FromMilliseconds(100), TimeSpan.FromMilliseconds(500)) + .UseEndpoints( + [ + Endpoint("first", first.Port), + Endpoint("second", second.Port) + ], + SharpLinkTransportFactories.Sockets()) + .UseCluster(options => + { + options.MinReadyEndpoints = 2; + options.MaxConnections = 2; + options.MaxConnectionsPerEndpoint = 1; + }) + .Build(); + + await client.ConnectAsync(); + var service = client.Get(); + Ensure(await service.PingAsync(41) == 42, "initial static-cluster RPC"); + + await first.StopAsync(); + await WaitUntilAsync(() => ((SharpLinkClient)client).ReadyConnectionCount == 1, TimeSpan.FromSeconds(2)); + Ensure(await service.PingAsync(8) == 9, "remaining endpoint should serve RPC"); + } + + [Test] + public async Task InitialEndpointFailureShouldNotPreventAnotherEndpointFromConnecting() + { + using var unavailableListener = new TcpListener(IPAddress.Loopback, 0); + unavailableListener.Start(); + var unavailablePort = ((IPEndPoint)unavailableListener.LocalEndpoint).Port; + unavailableListener.Stop(); + + await using var available = await TcpServerScope.StartAsync("available"); + await using var client = SharpClientBuilder.Create() + .UseSerializer(MemoryPackCodec.Resolver) + .UseEndpoints( + [Endpoint("unavailable", unavailablePort), Endpoint("available", available.Port)], + SharpLinkTransportFactories.Sockets()) + .UseCluster(options => + { + options.MinReadyEndpoints = 2; + options.MaxConnections = 2; + options.MaxConnectionsPerEndpoint = 1; + }) + .Build(); + + await client.ConnectAsync(); + Ensure(await client.Get().GetEndpointIdAsync() == "available", + "a healthy endpoint should connect when another is unavailable"); + } + + [Test] + [NotInParallel] + public async Task FailedInitialStaticDialShouldProbeLaterEndpointsWithoutWaitingForASlowSibling() + { + await using var healthy = await TcpServerScope.StartAsync("healthy"); + var blocking = new BlockingConnectFactory(); + var failing = new FailingConnectFactory(); + var sockets = SharpLinkTransportFactories.Sockets(); + var client = SharpClientBuilder.Create() + .UseSerializer(MemoryPackCodec.Resolver) + .UseEndpoints( + [ + Endpoint("failed", 1), + Endpoint("blocked", 2), + Endpoint("healthy", healthy.Port) + ], + endpoint => endpoint.Id switch + { + "failed" => failing, + "blocked" => blocking, + _ => sockets(endpoint) + }) + .UseCluster(options => + { + options.MinReadyEndpoints = 2; + options.MaxConnections = 4; + options.MaxConnectionsPerEndpoint = 1; + }) + .Build(); + + try + { + var connect = client.ConnectAsync().AsTask(); + await blocking.Entered.WaitAsync(TimeSpan.FromSeconds(2)); + + await connect.WaitAsync(TimeSpan.FromSeconds(2)); + Ensure(((SharpLinkClient)client).ReadyConnectionCount == 1, + "a failed initial dial must immediately free a probe for a later healthy endpoint"); + } + finally + { + blocking.Release(); + await client.DisposeAsync(); + } + } + + [Test] + public async Task AllUnavailableEndpointsShouldReportUnavailable() + { + var firstPort = GetUnusedTcpPort(); + var secondPort = GetUnusedTcpPort(); + await using var client = SharpClientBuilder.Create() + .UseSerializer(MemoryPackCodec.Resolver) + .UseEndpoints( + [Endpoint("first", firstPort), Endpoint("second", secondPort)], + SharpLinkTransportFactories.Sockets()) + .Build(); + + var exception = await EnsureThrowsSharpLink(client.ConnectAsync().AsTask(), "all unavailable endpoints"); + Ensure(exception.Code == SharpLinkErrorCode.Unavailable, "all unavailable endpoints error code"); + } + + [Test] + [NotInParallel] + public async Task DisconnectedEndpointShouldReconnectWithoutInterruptingAnotherEndpoint() + { + await using var first = await TcpServerScope.StartAsync("first"); + await using var second = await TcpServerScope.StartAsync("second"); + await using var client = SharpClientBuilder.Create() + .UseSerializer(MemoryPackCodec.Resolver) + .UseEndpoints( + [Endpoint("first", first.Port), Endpoint("second", second.Port)], + SharpLinkTransportFactories.Sockets()) + .UseCluster(options => + { + options.MinReadyEndpoints = 2; + options.MaxConnections = 2; + options.MaxConnectionsPerEndpoint = 1; + }) + .UseEndpointSelector(new PreferEndpointSelector("first")) + .Build(); + + await client.ConnectAsync(); + var service = client.Get(); + Ensure(await service.GetEndpointIdAsync() == "first", "initial preferred endpoint"); + + var port = first.Port; + await first.StopAsync(); + await WaitUntilAsync(() => ((SharpLinkClient)client).ReadyConnectionCount == 1, TimeSpan.FromSeconds(2)); + Ensure(await service.GetEndpointIdAsync() == "second", "healthy endpoint remains available during reconnect"); + + await using var replacement = await TcpServerScope.StartAsync("first-reconnected", port); + await WaitUntilAsync(() => ((SharpLinkClient)client).ReadyConnectionCount == 2, TimeSpan.FromSeconds(3)); + Ensure(((SharpLinkClient)client).ReadyConnectionCount == 2, "disconnected endpoint should reconnect independently"); + Ensure(await service.GetEndpointIdAsync() == "first-reconnected", "reconnected endpoint should rejoin selection"); + } + + [Test] + public async Task InvalidCustomSelectorShouldFailOnlyTheCurrentCall() + { + await using var first = await TcpServerScope.StartAsync(); + await using var second = await TcpServerScope.StartAsync(); + await using var client = SharpClientBuilder.Create() + .UseSerializer(MemoryPackCodec.Resolver) + .UseEndpoints( + [Endpoint("first", first.Port), Endpoint("second", second.Port)], + SharpLinkTransportFactories.Sockets()) + .UseEndpointSelector(new InvalidSelector()) + .Build(); + + await client.ConnectAsync(); + try + { + _ = await client.Get().PingAsync(1); + throw new Exception("invalid selector should fail the current call"); + } + catch (SharpLinkException exception) when (exception.Code == SharpLinkErrorCode.FailedPrecondition) + { + } + } + + [Test] + public async Task ThrowingCustomSelectorShouldLeaveTheClusterHealthyForLaterCalls() + { + await using var first = await TcpServerScope.StartAsync(); + await using var second = await TcpServerScope.StartAsync(); + await using var client = SharpClientBuilder.Create() + .UseSerializer(MemoryPackCodec.Resolver) + .UseEndpoints( + [Endpoint("first", first.Port), Endpoint("second", second.Port)], + SharpLinkTransportFactories.Sockets()) + .UseEndpointSelector(new ThrowOnceSelector()) + .Build(); + + await client.ConnectAsync(); + var service = client.Get(); + var exception = await EnsureThrowsSharpLink(service.PingAsync(1).AsTask(), "throwing custom selector"); + Ensure(exception.Code == SharpLinkErrorCode.FailedPrecondition, "throwing selector error code"); + Ensure(await service.PingAsync(1) == 2, "later RPC should remain healthy"); + Ensure(client.State == SharpLinkConnectionState.Ready, "selector failure must not change client state"); + } + + [Test] + public async Task StaticClusterShouldExpandWithinGlobalAndPerEndpointBudgets() + { + await using var first = await TcpServerScope.StartAsync(); + await using var second = await TcpServerScope.StartAsync(); + await using var client = SharpClientBuilder.Create() + .UseSerializer(MemoryPackCodec.Resolver) + .UseEndpoints( + [Endpoint("first", first.Port), Endpoint("second", second.Port)], + SharpLinkTransportFactories.Sockets()) + .UseCluster(options => + { + options.MinReadyEndpoints = 2; + options.MaxConnections = 4; + options.MaxConnectionsPerEndpoint = 2; + }) + .Build(); + + await client.ConnectAsync(); + await WaitUntilAsync(() => ((SharpLinkClient)client).ReadyConnectionCount == 2, TimeSpan.FromSeconds(2)); + var service = client.Get(); + var calls = new Task[32]; + for (var index = 0; index < calls.Length; index++) + calls[index] = service.SlowAsync(100, CancellationToken.None).AsTask(); + await Task.WhenAll(calls); + + var implementation = (SharpLinkClient)client; + await WaitUntilAsync(() => implementation.ReadyConnectionCount == 4, TimeSpan.FromSeconds(2)); + Ensure(implementation.ReadyConnectionCount == 4, "cluster should fill only the configured global budget"); + } + + [Test] + [NotInParallel] + public async Task CustomStaticSelectorShouldRejectTheOnlyNonMatchingReadyEndpoint() + { + await using var east = await TcpServerScope.StartAsync("east"); + await using var west = await TcpServerScope.StartAsync("west"); + await using var client = SharpClientBuilder.Create() + .UseSerializer(MemoryPackCodec.Resolver) + .UseEndpoints( + [Endpoint("east", east.Port, "east"), Endpoint("west", west.Port, "west")], + SharpLinkTransportFactories.Sockets()) + .UseEndpointSelector(new AttributeSelector("west")) + .UseCluster(options => + { + options.MinReadyEndpoints = 2; + options.MaxConnections = 2; + options.MaxConnectionsPerEndpoint = 1; + }) + .Build(); + + await client.ConnectAsync(); + await WaitUntilAsync(() => ((SharpLinkClient)client).ReadyConnectionCount == 2, TimeSpan.FromSeconds(2)); + await west.StopAsync(); + await WaitUntilAsync(() => ((SharpLinkClient)client).ReadyConnectionCount == 1, TimeSpan.FromSeconds(2)); + + var exception = await EnsureThrowsSharpLink( + client.Get().PingAsync(1).AsTask(), + "selector must reject the only non-matching static endpoint"); + Ensure(exception.Code == SharpLinkErrorCode.FailedPrecondition, + "a strict static selector must not be bypassed for one candidate"); + } + + [Test] + public async Task StaticNamedPipeEndpointsShouldServeRpc() + { + var firstName = $"sharplink-static-first-{Guid.NewGuid():N}"; + var secondName = $"sharplink-static-second-{Guid.NewGuid():N}"; + await using var first = await TcpServerScope.StartNamedPipeAsync(firstName); + await using var second = await TcpServerScope.StartNamedPipeAsync(secondName); + await using var client = SharpClientBuilder.Create() + .UseSerializer(MemoryPackCodec.Resolver) + .UseEndpoints( + [ + new SharpLinkEndpoint { Id = "first", Address = new SharpLinkNamedPipeAddress(firstName) }, + new SharpLinkEndpoint { Id = "second", Address = new SharpLinkNamedPipeAddress(secondName) } + ], + SharpLinkTransportFactories.NamedPipes()) + .Build(); + + await client.ConnectAsync(); + Ensure(await client.Get().PingAsync(4) == 5, "named-pipe static-cluster RPC"); + } + + [Test] + public async Task StaticSharedMemoryEndpointsShouldServeRpc() + { + var firstName = $"sharplink-static-first-{Guid.NewGuid():N}"; + var secondName = $"sharplink-static-second-{Guid.NewGuid():N}"; + await using var first = await TcpServerScope.StartSharedMemoryAsync(firstName); + await using var second = await TcpServerScope.StartSharedMemoryAsync(secondName); + await using var client = SharpClientBuilder.Create() + .UseSerializer(MemoryPackCodec.Resolver) + .UseEndpoints( + [ + new SharpLinkEndpoint { Id = "first", Address = new SharpLinkSharedMemoryAddress(firstName) }, + new SharpLinkEndpoint { Id = "second", Address = new SharpLinkSharedMemoryAddress(secondName) } + ], + SharpLinkTransportFactories.SharedMemory()) + .Build(); + + await client.ConnectAsync(); + Ensure(await client.Get().PingAsync(6) == 7, "shared-memory static-cluster RPC"); + } + + [Test] + public async Task StaticUdsEndpointsShouldServeRpc() + { + if (!Socket.OSSupportsUnixDomainSockets) + return; + var firstPath = Path.Combine(Path.GetTempPath(), $"sharplink-static-{Guid.NewGuid():N}.sock"); + var secondPath = Path.Combine(Path.GetTempPath(), $"sharplink-static-{Guid.NewGuid():N}.sock"); + await using var first = await TcpServerScope.StartUdsAsync(firstPath); + await using var second = await TcpServerScope.StartUdsAsync(secondPath); + await using var client = SharpClientBuilder.Create() + .UseSerializer(MemoryPackCodec.Resolver) + .UseEndpoints( + [ + new SharpLinkEndpoint { Id = "first", Address = new SharpLinkUnixDomainSocketAddress(firstPath) }, + new SharpLinkEndpoint { Id = "second", Address = new SharpLinkUnixDomainSocketAddress(secondPath) } + ], + SharpLinkTransportFactories.Sockets()) + .Build(); + + await client.ConnectAsync(); + Ensure(await client.Get().PingAsync(10) == 11, "UDS static-cluster RPC"); + } + + [Test] + public async Task StaticTcpEndpointsShouldSupportHostnameIpv4AndIpv6() + { + await using var hostname = await TcpServerScope.StartAsync("hostname"); + await using var ipv4 = await TcpServerScope.StartAsync("ipv4"); + await using var ipv6 = Socket.OSSupportsIPv6 + ? await TcpServerScope.StartAsync("ipv6", address: IPAddress.IPv6Loopback) + : null; + var endpoints = new List + { + new() + { + Id = "hostname", + Address = new SharpLinkTcpAddress("localhost", hostname.Port) + }, + Endpoint("ipv4", ipv4.Port) + }; + if (ipv6 is not null) + { + endpoints.Add(new SharpLinkEndpoint + { + Id = "ipv6", + Address = new SharpLinkTcpAddress(IPAddress.IPv6Loopback.ToString(), ipv6.Port) + }); + } + + await using var client = SharpClientBuilder.Create() + .UseSerializer(MemoryPackCodec.Resolver) + .UseEndpoints(endpoints, SharpLinkTransportFactories.Sockets()) + .UseCluster(options => + { + options.MinReadyEndpoints = endpoints.Count; + options.MaxConnections = endpoints.Count; + options.MaxConnectionsPerEndpoint = 1; + }) + .UseLoadBalancing(SharpLinkLoadBalancingStrategy.RoundRobin) + .Build(); + + await client.ConnectAsync(); + await WaitUntilAsync( + () => ((SharpLinkClient)client).ReadyConnectionCount == endpoints.Count, + TimeSpan.FromSeconds(3)); + var service = client.Get(); + var observed = new HashSet(StringComparer.Ordinal); + for (var index = 0; index < endpoints.Count * 2; index++) + observed.Add(await service.GetEndpointIdAsync()); + Ensure(observed.Contains("hostname") && observed.Contains("ipv4"), "hostname and IPv4 endpoints"); + if (ipv6 is not null) + Ensure(observed.Contains("ipv6"), "IPv6 endpoint"); + } + + [Test] + public async Task ConcurrentConnectAndStopShouldConvergeStaticClusterResources() + { + await using var first = await TcpServerScope.StartAsync(); + await using var second = await TcpServerScope.StartAsync(); + var client = SharpClientBuilder.Create() + .UseSerializer(MemoryPackCodec.Resolver) + .UseEndpoints( + [Endpoint("first", first.Port), Endpoint("second", second.Port)], + SharpLinkTransportFactories.Sockets()) + .Build(); + try + { + var connects = new Task[16]; + for (var index = 0; index < connects.Length; index++) + connects[index] = client.ConnectAsync().AsTask(); + await Task.WhenAll(connects); + + var stops = new Task[16]; + for (var index = 0; index < stops.Length; index++) + stops[index] = client.StopAsync().AsTask(); + await Task.WhenAll(stops); + + var implementation = (SharpLinkClient)client; + Ensure(implementation.State == SharpLinkConnectionState.Stopped, "static cluster must stop"); + Ensure(implementation.ReadyConnectionCount == 0, "static cluster connections must converge to zero"); + await EnsureThrows(client.ConnectAsync().AsTask(), "Connect after Stop"); + } + finally + { + await client.DisposeAsync(); + } + } + + [Test] + [NotInParallel] + public async Task StopShouldWaitForInitialSiblingDialsBeforeDisposingFactories() + { + await using var first = await TcpServerScope.StartAsync("first"); + var blocking = new BlockingConnectFactory(); + var sockets = SharpLinkTransportFactories.Sockets(); + var client = SharpClientBuilder.Create() + .UseSerializer(MemoryPackCodec.Resolver) + .UseEndpoints( + [Endpoint("first", first.Port), Endpoint("blocked", GetUnusedTcpPort())], + endpoint => endpoint.Id == "first" ? sockets(endpoint) : blocking) + .Build(); + + try + { + var connect = client.ConnectAsync().AsTask(); + await blocking.Entered.WaitAsync(TimeSpan.FromSeconds(2)); + await connect.WaitAsync(TimeSpan.FromSeconds(2)); + + var stop = client.StopAsync().AsTask(); + await Task.Delay(100); + Ensure(!stop.IsCompleted, "StopAsync must wait for an in-flight initial sibling dial"); + + blocking.Release(); + await stop.WaitAsync(TimeSpan.FromSeconds(2)); + Ensure(((SharpLinkClient)client).State == SharpLinkConnectionState.Stopped, + "cluster must stop after the sibling dial finishes"); + } + finally + { + blocking.Release(); + await client.DisposeAsync(); + } + } + + [Test] + [NotInParallel] + public async Task InitialStaticDialReservationsShouldPreventSurplusTargetFill() + { + await using var first = await TcpServerScope.StartAsync("first"); + var blocking = new BlockingConnectFactory(); + var surplus = new FailingConnectFactory(); + var sockets = SharpLinkTransportFactories.Sockets(); + var client = SharpClientBuilder.Create() + .UseSerializer(MemoryPackCodec.Resolver) + .UseEndpoints( + [Endpoint("first", first.Port), Endpoint("blocked", 1), Endpoint("surplus", 2)], + endpoint => endpoint.Id switch + { + "first" => sockets(endpoint), + "blocked" => blocking, + _ => surplus + }) + .UseCluster(options => + { + options.MinReadyEndpoints = 2; + options.MaxConnections = 3; + options.MaxConnectionsPerEndpoint = 1; + }) + .Build(); + + try + { + var connect = client.ConnectAsync().AsTask(); + await blocking.Entered.WaitAsync(TimeSpan.FromSeconds(2)); + await connect.WaitAsync(TimeSpan.FromSeconds(2)); + await Task.Delay(200); + + Ensure(surplus.ConnectCount == 0, + "a pending initial sibling must reserve the remaining ready target before reconciliation"); + } + finally + { + blocking.Release(); + await client.DisposeAsync(); + } + } + + [Test] + [NotInParallel] + public async Task ConnectAfterStaticClusterDisconnectShouldAwaitRecovery() + { + await using var first = await TcpServerScope.StartAsync("first"); + var sockets = SharpLinkTransportFactories.Sockets(); + var blocking = new BlockAfterFirstConnectFactory(sockets(Endpoint("first", first.Port))); + var unavailable = new FailingConnectFactory(); + await using var client = SharpClientBuilder.Create() + .UseSerializer(MemoryPackCodec.Resolver) + .UseEndpoints( + [Endpoint("first", first.Port), Endpoint("unavailable", 1)], + endpoint => endpoint.Id == "first" ? blocking : unavailable) + .UseCluster(options => + { + options.MinReadyEndpoints = 1; + options.MaxConnections = 2; + options.MaxConnectionsPerEndpoint = 1; + }) + .Build(); + + await client.ConnectAsync(); + await first.StopAsync(); + await WaitUntilAsync(() => ((SharpLinkClient)client).ReadyConnectionCount == 0, TimeSpan.FromSeconds(2)); + await blocking.Entered.WaitAsync(TimeSpan.FromSeconds(2)); + + var reconnect = client.ConnectAsync().AsTask(); + var completed = await Task.WhenAny(reconnect, Task.Delay(100)); + Ensure(!ReferenceEquals(completed, reconnect), + "ConnectAsync must wait for a new ready connection instead of returning stale initialization success"); + } + + [Test] + [NotInParallel] + public async Task FailedInitialSiblingDialShouldContinueFillingMinReadyEndpoints() + { + await using var first = await TcpServerScope.StartAsync("first"); + await using var recovered = await TcpServerScope.StartAsync("recovered"); + var sockets = SharpLinkTransportFactories.Sockets(); + var delayedFailure = new DeferredFailOnceFactory(sockets(Endpoint("recovered", recovered.Port))); + await using var client = SharpClientBuilder.Create() + .UseSerializer(MemoryPackCodec.Resolver) + .UseEndpoints( + [Endpoint("first", first.Port), Endpoint("recovered", recovered.Port)], + endpoint => endpoint.Id == "recovered" ? delayedFailure : sockets(endpoint)) + .UseCluster(options => + { + options.MinReadyEndpoints = 2; + options.MaxConnections = 2; + options.MaxConnectionsPerEndpoint = 1; + }) + .Build(); + + var connect = client.ConnectAsync().AsTask(); + await delayedFailure.Entered.WaitAsync(TimeSpan.FromSeconds(2)); + await connect.WaitAsync(TimeSpan.FromSeconds(2)); + Ensure(((SharpLinkClient)client).ReadyConnectionCount == 1, + "first endpoint should make initial connect available before the sibling finishes"); + + delayedFailure.FailInitialConnect(); + await WaitUntilAsync(() => ((SharpLinkClient)client).ReadyConnectionCount == 2, TimeSpan.FromSeconds(3)); + Ensure(delayedFailure.ConnectCount >= 2, + "failed initial sibling must be retried to restore the configured minimum ready count"); + Ensure(await client.Get().GetEndpointIdAsync() is "first" or "recovered", + "recovered static endpoint topology should remain usable"); + } + + [Test] + [NotInParallel] + public async Task StaticReconnectShouldProbeHealthyEndpointsAfterAFailingEndpoint() + { + await using var first = await TcpServerScope.StartAsync("first"); + await using var second = await TcpServerScope.StartAsync("second"); + var failing = new FailingConnectFactory(); + var sockets = SharpLinkTransportFactories.Sockets(); + await using var client = SharpClientBuilder.Create() + .UseSerializer(MemoryPackCodec.Resolver) + .UseEndpoints( + [Endpoint("bad", 1), Endpoint("first", first.Port), Endpoint("second", second.Port)], + endpoint => endpoint.Id == "bad" ? failing : sockets(endpoint)) + .UseCluster(options => + { + options.MinReadyEndpoints = 2; + options.MaxConnections = 2; + options.MaxConnectionsPerEndpoint = 1; + }) + .Build(); + + await client.ConnectAsync(); + await WaitUntilAsync(() => ((SharpLinkClient)client).ReadyConnectionCount == 2, TimeSpan.FromSeconds(3)); + + Ensure(failing.ConnectCount != 0, "the failing endpoint should have been probed"); + Ensure(((SharpLinkClient)client).ReadyConnectionCount == 2, + "a persistently failing endpoint must not monopolize the static ready target"); + } + + [Test] + [NotInParallel] + public async Task InitialStaticConnectShouldContinueFillingTargetsBeyondTheFirstBatch() + { + await using var first = await TcpServerScope.StartAsync("first"); + await using var second = await TcpServerScope.StartAsync("second"); + await using var third = await TcpServerScope.StartAsync("third"); + await using var fourth = await TcpServerScope.StartAsync("fourth"); + await using var fifth = await TcpServerScope.StartAsync("fifth"); + await using var client = SharpClientBuilder.Create() + .UseSerializer(MemoryPackCodec.Resolver) + .UseEndpoints( + [ + Endpoint("first", first.Port), + Endpoint("second", second.Port), + Endpoint("third", third.Port), + Endpoint("fourth", fourth.Port), + Endpoint("fifth", fifth.Port) + ], + SharpLinkTransportFactories.Sockets()) + .UseCluster(options => + { + options.MinReadyEndpoints = 5; + options.MaxConnections = 5; + options.MaxConnectionsPerEndpoint = 1; + }) + .Build(); + + await client.ConnectAsync(); + await WaitUntilAsync(() => ((SharpLinkClient)client).ReadyConnectionCount == 5, TimeSpan.FromSeconds(3)); + + Ensure(((SharpLinkClient)client).ReadyConnectionCount == 5, + "initial static connect must continue filling endpoints beyond its first parallel batch"); + } + + [Test] + public async Task RoundRobinAndCustomAttributeSelectorsShouldChooseExpectedEndpoints() + { + await using var first = await TcpServerScope.StartAsync("east"); + await using var second = await TcpServerScope.StartAsync("west"); + var endpoints = new[] + { + Endpoint("first", first.Port, "east"), + Endpoint("second", second.Port, "west") + }; + + await using (var roundRobin = SharpClientBuilder.Create() + .UseSerializer(MemoryPackCodec.Resolver) + .UseEndpoints(endpoints, SharpLinkTransportFactories.Sockets()) + .UseLoadBalancing(SharpLinkLoadBalancingStrategy.RoundRobin) + .Build()) + { + await roundRobin.ConnectAsync(); + await WaitUntilAsync(() => ((SharpLinkClient)roundRobin).ReadyConnectionCount == 2, TimeSpan.FromSeconds(2)); + Ensure(((SharpLinkClient)roundRobin).ReadyConnectionCount == 2, "round robin endpoints must both be ready"); + var service = roundRobin.Get(); + var ids = new[] + { + await service.GetEndpointIdAsync(), + await service.GetEndpointIdAsync(), + await service.GetEndpointIdAsync(), + await service.GetEndpointIdAsync() + }; + Ensure(ids[0] != ids[1] && ids[0] == ids[2] && ids[1] == ids[3], "round robin endpoint order"); + } + + await using var custom = SharpClientBuilder.Create() + .UseSerializer(MemoryPackCodec.Resolver) + .UseEndpoints(endpoints, SharpLinkTransportFactories.Sockets()) + .UseEndpointSelector(new AttributeSelector("west")) + .Build(); + await custom.ConnectAsync(); + await WaitUntilAsync(() => ((SharpLinkClient)custom).ReadyConnectionCount == 2, TimeSpan.FromSeconds(2)); + Ensure(((SharpLinkClient)custom).ReadyConnectionCount == 2, "custom selector endpoints must both be ready"); + Ensure(await custom.Get().GetEndpointIdAsync() == "west", "custom selector attributes"); + } + + [Test] + public async Task LeastPendingShouldAvoidEndpointWithAnActiveCall() + { + await using var first = await TcpServerScope.StartAsync("first"); + await using var second = await TcpServerScope.StartAsync("second"); + await using var client = SharpClientBuilder.Create() + .UseSerializer(MemoryPackCodec.Resolver) + .UseEndpoints( + [Endpoint("first", first.Port), Endpoint("second", second.Port)], + SharpLinkTransportFactories.Sockets()) + .UseLoadBalancing(SharpLinkLoadBalancingStrategy.LeastPending) + .Build(); + + await client.ConnectAsync(); + var service = client.Get(); + var slow = service.SlowAsync(200, CancellationToken.None).AsTask(); + var completed = await Task.WhenAny(first.Service.SlowCallStarted!.Task, second.Service.SlowCallStarted!.Task); + var busyId = await ((Task)completed); + var selectedId = await service.GetEndpointIdAsync(); + Ensure(selectedId != busyId, "least pending should select the non-busy endpoint"); + await slow; + } + + [Test] + public async Task LeastPendingShouldRotateTiesAcrossReadyEndpoints() + { + await using var first = await TcpServerScope.StartAsync("first"); + await using var second = await TcpServerScope.StartAsync("second"); + await using var client = SharpClientBuilder.Create() + .UseSerializer(MemoryPackCodec.Resolver) + .UseEndpoints( + [Endpoint("first", first.Port), Endpoint("second", second.Port)], + SharpLinkTransportFactories.Sockets()) + .UseLoadBalancing(SharpLinkLoadBalancingStrategy.LeastPending) + .Build(); + + await client.ConnectAsync(); + var service = client.Get(); + var ids = new[] + { + await service.GetEndpointIdAsync(), + await service.GetEndpointIdAsync(), + await service.GetEndpointIdAsync(), + await service.GetEndpointIdAsync() + }; + Ensure(ids[0] != ids[1] && ids[0] == ids[2] && ids[1] == ids[3], "least-pending tie rotation"); + } + + [Test] + [NotInParallel] + public async Task GoAwayShouldDrainExistingUnaryAndStreamWhileNewCallsUseAnotherEndpoint() + { + await using var first = await TcpServerScope.StartAsync("first"); + await using var second = await TcpServerScope.StartAsync("second"); + await using var client = SharpClientBuilder.Create() + .UseSerializer(MemoryPackCodec.Resolver) + .UseEndpoints( + [Endpoint("first", first.Port), Endpoint("second", second.Port)], + SharpLinkTransportFactories.Sockets()) + .UseCluster(options => + { + options.MinReadyEndpoints = 2; + options.MaxConnections = 2; + options.MaxConnectionsPerEndpoint = 1; + }) + .UseEndpointSelector(new PreferEndpointSelector("first")) + .Build(); + + await client.ConnectAsync(); + var service = client.Get(); + var longUnary = service.SlowAsync(600, CancellationToken.None).AsTask(); + Ensure(await first.Service.SlowUnaryStarted!.Task == "first", "accepted unary should start on first endpoint"); + await using var stream = service.SlowRangeAsync(3, 100, CancellationToken.None).GetAsyncEnumerator(); + Ensure(await stream.MoveNextAsync() && stream.Current == 0, "first stream item"); + Ensure(await first.Service.SlowCallStarted!.Task == "first", "existing stream should start on first endpoint"); + + var stopTask = first.StopAsync(TimeSpan.FromSeconds(2)).AsTask(); + var implementation = (SharpLinkClient)client; + await WaitUntilAsync(() => implementation.ReadyConnectionCount == 1, TimeSpan.FromSeconds(2)); + Ensure(implementation.ReadyConnectionCount == 1, "GoAway should retire the draining endpoint from selection"); + Ensure(await service.GetEndpointIdAsync() == "second", "new RPC should use the remaining endpoint"); + + Ensure(await stream.MoveNextAsync() && stream.Current == 1, "draining stream second item"); + Ensure(await stream.MoveNextAsync() && stream.Current == 2, "draining stream third item"); + Ensure(!await stream.MoveNextAsync(), "draining stream completion"); + Ensure(await longUnary == 600, "accepted unary should complete during graceful drain"); + await stopTask.WaitAsync(TimeSpan.FromSeconds(2)); + } + + private static SharpLinkEndpoint Endpoint(string id, int port, string? zone = null) => new() + { + Id = id, + Address = new SharpLinkTcpAddress(IPAddress.Loopback.ToString(), port), + Attributes = zone is null ? new Dictionary() : new Dictionary { ["zone"] = zone } + }; + + private static int GetUnusedTcpPort() + { + using var listener = new TcpListener(IPAddress.Loopback, 0); + listener.Start(); + return ((IPEndPoint)listener.LocalEndpoint).Port; + } + + private static void Ensure(bool condition, string message) + { + if (!condition) + throw new Exception(message); + } + + private static async Task WaitUntilAsync(Func condition, TimeSpan timeout) + { + var deadline = Stopwatch.GetTimestamp() + (long)(timeout.TotalSeconds * Stopwatch.Frequency); + while (!condition() && Stopwatch.GetTimestamp() < deadline) + await Task.Delay(20); + } + + private static async Task EnsureThrows(Task action, string name) where TException : Exception + { + try + { + await action; + throw new Exception($"{name} should throw {typeof(TException).Name}"); + } + catch (TException) + { + } + } + + private static async Task EnsureThrowsSharpLink(Task action, string name) + { + try + { + await action; + throw new Exception($"{name} should throw SharpLinkException"); + } + catch (SharpLinkException exception) + { + return exception; + } + } + + private sealed class TcpServerScope : IAsyncDisposable + { + private readonly ISharpLinkServer _server; + private readonly CancellationTokenSource _cancellation = new(); + private readonly Task _runTask; + private int _stopped; + + private TcpServerScope(ISharpLinkServer server, int port, ConnectionBehaviorService service) + { + _server = server; + Port = port; + Service = service; + _runTask = Task.Run(() => _server.RunAsync(_cancellation.Token).AsTask(), CancellationToken.None); + } + + public int Port { get; } + public ConnectionBehaviorService Service { get; } + + public static Task StartAsync( + string endpointId = "default", + int port = 0, + IPAddress? address = null) + { + var listenAddress = address ?? IPAddress.Loopback; + var builder = SharpLinkServerBuilder.Create() + .UseTcp(port, listenAddress.ToString()) + .UseSerializer(MemoryPackCodec.Resolver) + .UseHeartbeat(TimeSpan.FromMilliseconds(100), TimeSpan.FromMilliseconds(500)); + var service = new ConnectionBehaviorService + { + EndpointId = endpointId, + SlowCallStarted = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously), + SlowUnaryStarted = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously) + }; + builder.ReplaceService(service); + var boundPort = ((IPEndPoint)builder.Transport!.LocalEndPoint!).Port; + return Task.FromResult(new TcpServerScope(builder.Build(), boundPort, service)); + } + + public static Task StartNamedPipeAsync(string name) + { + var builder = SharpLinkServerBuilder.Create() + .UseNamedPipe(name) + .UseSerializer(MemoryPackCodec.Resolver) + .UseHeartbeat(TimeSpan.FromMilliseconds(100), TimeSpan.FromMilliseconds(500)); + return Task.FromResult(new TcpServerScope(builder.Build(), 0, new ConnectionBehaviorService())); + } + + public static Task StartSharedMemoryAsync(string name) + { + var builder = SharpLinkServerBuilder.Create() + .UseSharedMemory(name) + .UseSerializer(MemoryPackCodec.Resolver) + .UseHeartbeat(TimeSpan.FromMilliseconds(100), TimeSpan.FromMilliseconds(500)); + return Task.FromResult(new TcpServerScope(builder.Build(), 0, new ConnectionBehaviorService())); + } + + public static Task StartUdsAsync(string path) + { + var builder = SharpLinkServerBuilder.Create() + .UseUds(path) + .UseSerializer(MemoryPackCodec.Resolver) + .UseHeartbeat(TimeSpan.FromMilliseconds(100), TimeSpan.FromMilliseconds(500)); + return Task.FromResult(new TcpServerScope(builder.Build(), 0, new ConnectionBehaviorService())); + } + + public async ValueTask StopAsync(TimeSpan gracefulTimeout = default) + { + if (Interlocked.Exchange(ref _stopped, 1) != 0) + return; + await _server.StopAsync(gracefulTimeout); + await _cancellation.CancelAsync(); + await Task.WhenAny(_runTask, Task.Delay(1000)); + } + + public async ValueTask DisposeAsync() + { + await StopAsync(); + await _server.DisposeAsync(); + _cancellation.Dispose(); + } + } + + private sealed class InvalidSelector : ISharpLinkEndpointSelector + { + public int Select(in SharpLinkEndpointSelectionContext context) => context.Count; + } + + private sealed class BlockingConnectFactory : IClientTransportFactory + { + private readonly TaskCompletionSource _entered = new(TaskCreationOptions.RunContinuationsAsynchronously); + private readonly TaskCompletionSource _release = new(TaskCreationOptions.RunContinuationsAsynchronously); + + public Task Entered => _entered.Task; + + public ValueTask ConnectAsync(CancellationToken cancellationToken = default) + { + _entered.TrySetResult(); + return AwaitReleaseAsync(); + } + + public void Release() => _release.TrySetResult(); + + public ValueTask DisposeAsync() => ValueTask.CompletedTask; + + private async ValueTask AwaitReleaseAsync() + { + await _release.Task.ConfigureAwait(false); + throw new InvalidOperationException("test initial dial completed after release"); + } + } + + private sealed class DeferredFailOnceFactory(IClientTransportFactory inner) : IClientTransportFactory + { + private readonly TaskCompletionSource _entered = new(TaskCreationOptions.RunContinuationsAsynchronously); + private readonly TaskCompletionSource _release = new(TaskCreationOptions.RunContinuationsAsynchronously); + private int _connectCount; + + public Task Entered => _entered.Task; + public int ConnectCount => Volatile.Read(ref _connectCount); + + public ValueTask ConnectAsync(CancellationToken cancellationToken = default) + { + if (Interlocked.Increment(ref _connectCount) != 1) + return inner.ConnectAsync(cancellationToken); + _entered.TrySetResult(); + return FailAfterReleaseAsync(); + } + + public void FailInitialConnect() => _release.TrySetResult(); + + public ValueTask DisposeAsync() => inner.DisposeAsync(); + + private async ValueTask FailAfterReleaseAsync() + { + await _release.Task.ConfigureAwait(false); + throw new InvalidOperationException("test initial sibling failure"); + } + } + + private sealed class FailingConnectFactory : IClientTransportFactory + { + private int _connectCount; + + public int ConnectCount => Volatile.Read(ref _connectCount); + + public ValueTask ConnectAsync(CancellationToken cancellationToken = default) + { + Interlocked.Increment(ref _connectCount); + return ValueTask.FromException(new InvalidOperationException("test transport failure")); + } + + public ValueTask DisposeAsync() => ValueTask.CompletedTask; + } + + private sealed class BlockAfterFirstConnectFactory(IClientTransportFactory inner) : IClientTransportFactory + { + private readonly TaskCompletionSource _entered = new(TaskCreationOptions.RunContinuationsAsynchronously); + private int _connectCount; + + public Task Entered => _entered.Task; + + public async ValueTask ConnectAsync(CancellationToken cancellationToken = default) + { + if (Interlocked.Increment(ref _connectCount) == 1) + return await inner.ConnectAsync(cancellationToken).ConfigureAwait(false); + + _entered.TrySetResult(); + await Task.Delay(Timeout.InfiniteTimeSpan, cancellationToken).ConfigureAwait(false); + throw new UnreachableException(); + } + + public ValueTask DisposeAsync() => inner.DisposeAsync(); + } + + private sealed class AttributeSelector(string zone) : ISharpLinkEndpointSelector + { + public int Select(in SharpLinkEndpointSelectionContext context) + { + for (var index = 0; index < context.Count; index++) + { + if ((context.ExcludedMask & (1UL << index)) == 0 && + context[index].Endpoint.Attributes.TryGetValue("zone", out var value) && value == zone) + { + return index; + } + } + return -1; + } + } + + private sealed class ThrowOnceSelector : ISharpLinkEndpointSelector + { + private int _remainingFailures = 1; + + public int Select(in SharpLinkEndpointSelectionContext context) + { + if (Interlocked.Exchange(ref _remainingFailures, 0) != 0) + throw new InvalidOperationException("test selector failure"); + return 0; + } + } + + private sealed class PreferEndpointSelector(string endpointId) : ISharpLinkEndpointSelector + { + public int Select(in SharpLinkEndpointSelectionContext context) + { + for (var index = 0; index < context.Count; index++) + { + if ((context.ExcludedMask & (1UL << index)) == 0 && + context[index].Endpoint.Id == endpointId) + { + return index; + } + } + + for (var index = 0; index < context.Count; index++) + if ((context.ExcludedMask & (1UL << index)) == 0) + return index; + return -1; + } + } +} diff --git a/test/SharpLink.IntegrationTests/TlsTransportIntegrationTests.cs b/test/SharpLink.IntegrationTests/TlsTransportIntegrationTests.cs index 131e036..afbaef5 100644 --- a/test/SharpLink.IntegrationTests/TlsTransportIntegrationTests.cs +++ b/test/SharpLink.IntegrationTests/TlsTransportIntegrationTests.cs @@ -89,6 +89,29 @@ await EnsureTlsFailure( Ensure(await client.Get().AddAsync(2, 3) == 5, "mutual TLS RPC"); } + [Test] + public async Task MutualTlsShouldPreserveClientCertificateContext() + { + using var serverCertificate = CreateCertificate("localhost", serverAuthentication: true); + using var clientCertificate = CreateCertificate("sharplink-client", serverAuthentication: false); + var serverOptions = CreateServerOptions(serverCertificate); + serverOptions.ClientCertificateRequired = true; + serverOptions.EnabledSslProtocols = SslProtocols.Tls12; + serverOptions.RemoteCertificateValidationCallback = ValidateTestCertificate; + await using var server = await StartServerAsync(0, serverOptions); + + var clientOptions = CreateClientOptions("localhost"); + clientOptions.EnabledSslProtocols = SslProtocols.Tls12; + clientOptions.ClientCertificateContext = SslStreamCertificateContext.Create( + clientCertificate, + additionalCertificates: null, + offline: true); + await using var client = CreateClient(server.Port, clientOptions); + await client.ConnectAsync(); + Ensure(await client.Get().AddAsync(3, 4) == 7, + "client certificate context mutual TLS RPC"); + } + [Test] public async Task TlsHandshakeShouldHonorIndependentTimeout() { @@ -120,6 +143,41 @@ public async Task TlsHandshakeShouldHonorIndependentTimeout() } } + [Test] + public async Task StaticTlsEndpointsShouldUseEndpointAuthorityAndIsolateFailure() + { + using var certificate = CreateCertificate("localhost", serverAuthentication: true); + await using var first = await StartServerAsync(0, CreateServerOptions(certificate)); + await using var second = await StartServerAsync(0, CreateServerOptions(certificate)); + var tlsOptions = CreateClientOptions(string.Empty); + await using var client = SharpClientBuilder.Create() + .UseHeartbeat(TimeSpan.FromMilliseconds(100), TimeSpan.FromSeconds(2)) + .UseEndpoints( + [ + new SharpLinkEndpoint + { + Id = "first", + Address = new SharpLinkTcpAddress(IPAddress.Loopback.ToString(), first.Port), + Authority = "localhost" + }, + new SharpLinkEndpoint + { + Id = "second", + Address = new SharpLinkTcpAddress(IPAddress.Loopback.ToString(), second.Port), + Authority = "localhost" + } + ], + SharpLinkTransportFactories.Sockets(tlsOptions, tlsHandshakeTimeout: TimeSpan.FromSeconds(2))) + .Build(); + + await client.ConnectAsync(); + await WaitUntilAsync(() => ((SharpLinkClient)client).ReadyConnectionCount == 2); + Ensure(await client.Get().AddAsync(5, 6) == 11, "static TLS RPC"); + await first.StopAsync(); + await WaitUntilAsync(() => ((SharpLinkClient)client).ReadyConnectionCount == 1); + Ensure(await client.Get().AddAsync(7, 8) == 15, "remaining static TLS endpoint"); + } + private static ISharpLinkClient CreateClient(int port, SslClientAuthenticationOptions options) => SharpClientBuilder.Create() .UseTcp(IPAddress.Loopback.ToString(), port, options, TimeSpan.FromSeconds(2)) diff --git a/test/SharpLink.IntegrationTests/TransportConnectionIntegrationTests.cs b/test/SharpLink.IntegrationTests/TransportConnectionIntegrationTests.cs index 602f9d2..d70f409 100644 --- a/test/SharpLink.IntegrationTests/TransportConnectionIntegrationTests.cs +++ b/test/SharpLink.IntegrationTests/TransportConnectionIntegrationTests.cs @@ -1542,8 +1542,11 @@ public interface IConnectionBehaviorService : IService [NonCancellable] ValueTask EchoAsync(string value); [NonCancellable] + ValueTask GetEndpointIdAsync(); + [NonCancellable] ValueTask CreatePayloadAsync(int length); ValueTask SlowAsync(int delayMs, CancellationToken cancellationToken = default); + IAsyncEnumerable SlowRangeAsync(int count, int delayMs, CancellationToken cancellationToken = default); [NonCancellable] ValueTask GetAuthenticationSummaryAsync(); [NonCancellable] @@ -1559,19 +1562,40 @@ public interface IConnectionBehaviorService : IService [RpcService] public sealed class ConnectionBehaviorService : IConnectionBehaviorService { + public string EndpointId { get; set; } = "default"; + public TaskCompletionSource? SlowCallStarted { get; set; } + public TaskCompletionSource? SlowUnaryStarted { get; set; } + public ValueTask PingAsync(int value) => ValueTask.FromResult(value + 1); public ValueTask EchoAsync(string value) => ValueTask.FromResult(value); + public ValueTask GetEndpointIdAsync() => ValueTask.FromResult(EndpointId); + public ValueTask CreatePayloadAsync(int length) => ValueTask.FromResult(new string('x', length)); public async ValueTask SlowAsync(int delayMs, CancellationToken cancellationToken = default) { + SlowCallStarted?.TrySetResult(EndpointId); + SlowUnaryStarted?.TrySetResult(EndpointId); await Task.Delay(delayMs, cancellationToken); return delayMs; } + public async IAsyncEnumerable SlowRangeAsync( + int count, + int delayMs, + [System.Runtime.CompilerServices.EnumeratorCancellation] CancellationToken cancellationToken = default) + { + SlowCallStarted?.TrySetResult(EndpointId); + for (var value = 0; value < count; value++) + { + yield return value; + await Task.Delay(delayMs, cancellationToken); + } + } + public ValueTask GetAuthenticationSummaryAsync() { var context = SharpLinkCallContext.Current?.Authentication; diff --git a/test/SharpLink.LoadTest/Program.cs b/test/SharpLink.LoadTest/Program.cs index 965eb50..e3ae605 100644 --- a/test/SharpLink.LoadTest/Program.cs +++ b/test/SharpLink.LoadTest/Program.cs @@ -61,6 +61,7 @@ private static void PrintConfig(LoadTestOptions options) $"[Config] mode={options.Mode} transport={options.Transport} operation={options.Operation} " + $"duration={options.DurationSeconds}s warmup={options.WarmupSeconds}s concurrency=[{string.Join(",", options.ConcurrencyConfig)}] " + $"payload={options.PayloadSize}B pool={options.MinConnections}/{options.MaxConnections} " + + $"staticEndpoints={options.StaticEndpointCount} dynamicEndpoints={options.DynamicEndpointCount} dynamicResolver={options.UseDynamicResolver} lb={options.StaticLoadBalancingStrategy} " + $"profile={options.PerformanceProfile} requestTimeout={options.RequestTimeoutMode} " + $"admission={options.AdmissionMode} compression={options.CompressionAlgorithm}/{options.CompressionLevel} " + $"thresholds={options.CompressionMinimumPayloadBytes}B/{options.CompressionMinimumSavingsBytes}B/{options.CompressionMinimumSavingsRatio:P0} " + @@ -86,6 +87,7 @@ private static void PrintHelp() Console.WriteLine(" --duration 20 --warmup 5 --concurrency 1,2,4,8,16,32"); Console.WriteLine(" --operation empty|add|echo|oneway|yield|delay --payload-size 64"); Console.WriteLine(" --min-connections 1 --max-connections 1"); + Console.WriteLine(" --static-endpoints 1 | --dynamic-endpoints 1 --load-balancing p2c|random|roundrobin|leastpending (local TCP only)"); Console.WriteLine(" --profile balanced|lowlatency|throughput"); Console.WriteLine(" --request-timeout default|disabled|1ms|10ms|100ms"); Console.WriteLine(" --admission disabled|immediate|queue|reject"); @@ -109,6 +111,12 @@ private static void PrintHelp() private static async Task RunLocalAsync(LoadTestOptions options, MetricsRegistry metrics) { + if (options.UseStaticEndpoints || options.UseDynamicResolver) + { + await RunStaticTcpLocalAsync(options, metrics); + return; + } + await using var harness = await LoadTestTransportFactory.CreateLocalHarness( options.Transport, options.Host, @@ -147,6 +155,99 @@ private static async Task RunLocalAsync(LoadTestOptions options, MetricsRegistry } } + private static async Task RunStaticTcpLocalAsync(LoadTestOptions options, MetricsRegistry metrics) + { + var servers = new ISharpLinkServer?[options.EndpointCount]; + var serverTasks = new Task?[servers.Length]; + var endpoints = new SharpLinkEndpoint[servers.Length]; + using var serverCancellation = new CancellationTokenSource(); + try + { + for (var index = 0; index < servers.Length; index++) + { + var builder = ConfigureServer(SharpLinkServerBuilder.Create(), options) + .UseSerializer(MemoryPackCodec.Resolver) + .UseRuntime(runtime => + { + runtime.PerformanceProfile = options.PerformanceProfile; + ConfigureCompression(runtime, options); + }) + .UseHeartbeat( + TimeSpan.FromSeconds(options.HeartbeatCheckIntervalSeconds), + TimeSpan.FromSeconds(options.HeartbeatTimeoutSeconds)) + .UseTcp(0, options.BindIp); + var port = ((IPEndPoint)builder.Transport!.LocalEndPoint!).Port; + servers[index] = builder.Build(); + endpoints[index] = new SharpLinkEndpoint + { + Id = $"local-{index}", + Address = new SharpLinkTcpAddress(options.Host, port) + }; + serverTasks[index] = RunServerLoopAsync(servers[index]!, serverCancellation.Token); + } + + await Task.Delay(200, serverCancellation.Token); + var clientBuilder = SharpClientBuilder.Create() + .UseSerializer(MemoryPackCodec.Resolver) + .UseRuntime(runtime => + { + runtime.PerformanceProfile = options.PerformanceProfile; + ConfigureCompression(runtime, options); + }) + .UseHeartbeat( + TimeSpan.FromSeconds(options.HeartbeatIntervalSeconds), + TimeSpan.FromSeconds(options.HeartbeatTimeoutSeconds)); + if (options.UseDynamicResolver) + { + var snapshot = new SharpLinkEndpointSnapshot(1, endpoints); + clientBuilder + .UseEndpointResolver( + new DelegateSharpLinkEndpointResolver(_ => ValueTask.FromResult(snapshot)), + SharpLinkTransportFactories.Sockets()) + .UseCluster(cluster => + { + cluster.MinReadyEndpoints = endpoints.Length; + cluster.MaxConnections = endpoints.Length; + cluster.MaxConnectionsPerEndpoint = 1; + }) + .UseLoadBalancing(options.StaticLoadBalancingStrategy); + } + else if (endpoints.Length == 1) + { + clientBuilder.UseEndpoint(endpoints[0], SharpLinkTransportFactories.Sockets()); + } + else + { + clientBuilder + .UseEndpoints(endpoints, SharpLinkTransportFactories.Sockets()) + .UseCluster(cluster => + { + cluster.MinReadyEndpoints = endpoints.Length; + cluster.MaxConnections = endpoints.Length; + cluster.MaxConnectionsPerEndpoint = 1; + }) + .UseLoadBalancing(options.StaticLoadBalancingStrategy); + } + if (options.DisableRequestTimeout) + clientBuilder.DisableRequestTimeout(); + else if (options.RequestTimeout is { } requestTimeout) + clientBuilder.UseRequestTimeout(requestTimeout); + await using var client = clientBuilder.Build(); + await RunClientOnlyAsync(options, metrics, client); + } + finally + { + await serverCancellation.CancelAsync(); + for (var index = 0; index < servers.Length; index++) + { + var server = servers[index]; + if (server is not null) + await server.DisposeAsync(); + } + await Task.WhenAll(serverTasks.Select(static task => task ?? Task.CompletedTask)); + } + } + private static async Task RunServerLoopAsync(ISharpLinkServer server, CancellationToken token) { try @@ -509,6 +610,12 @@ public sealed class LoadTestOptions public int HeartbeatTimeoutSeconds { get; private init; } = 120; public int MinConnections { get; private init; } = 1; public int MaxConnections { get; private init; } = 1; + public bool UseStaticEndpoints { get; private init; } + public int StaticEndpointCount { get; private init; } = 1; + public bool UseDynamicResolver { get; private init; } + public int DynamicEndpointCount { get; private init; } = 1; + public int EndpointCount => UseDynamicResolver ? DynamicEndpointCount : StaticEndpointCount; + public SharpLinkLoadBalancingStrategy StaticLoadBalancingStrategy { get; private init; } = SharpLinkLoadBalancingStrategy.PowerOfTwoChoices; public SharpLinkPerformanceProfile PerformanceProfile { get; private init; } = SharpLinkPerformanceProfile.Balanced; public string RequestTimeoutMode { get; private init; } = "default"; public string AdmissionMode { get; private init; } = "disabled"; @@ -549,6 +656,29 @@ public static LoadTestOptions Parse(string[] args) var transport = map.TryGetValue("transport", out var transportStr) && TransportDefaults.TryParseTransport(transportStr, out var parsedTransport) ? parsedTransport : TransportMode.Tcp; + var staticEndpointCount = int.Parse(map.GetValueOrDefault("static-endpoints", "1")); + if (staticEndpointCount is < 1 or > SharpLinkClusterOptions.MaximumEndpoints) + throw new ArgumentOutOfRangeException(nameof(staticEndpointCount)); + var useStaticEndpoints = map.ContainsKey("static-endpoints"); + var dynamicEndpointCount = int.Parse(map.GetValueOrDefault("dynamic-endpoints", "1")); + if (dynamicEndpointCount is < 1 or > SharpLinkClusterOptions.MaximumEndpoints) + throw new ArgumentOutOfRangeException(nameof(dynamicEndpointCount)); + var useDynamicResolver = map.ContainsKey("dynamic-endpoints"); + if (useStaticEndpoints && useDynamicResolver) + throw new ArgumentException("Static and dynamic endpoint load-test modes are mutually exclusive."); + if ((useStaticEndpoints || useDynamicResolver) && (mode != RunMode.Local || transport != TransportMode.Tcp)) + { + throw new ArgumentException( + "Endpoint topology load tests currently support only --mode local --transport tcp."); + } + var staticLoadBalancingStrategy = map.GetValueOrDefault("load-balancing", "p2c").ToLowerInvariant() switch + { + "p2c" => SharpLinkLoadBalancingStrategy.PowerOfTwoChoices, + "random" => SharpLinkLoadBalancingStrategy.Random, + "roundrobin" => SharpLinkLoadBalancingStrategy.RoundRobin, + "leastpending" => SharpLinkLoadBalancingStrategy.LeastPending, + _ => throw new ArgumentException("Unsupported static load-balancing strategy.") + }; var concurrencyNum = map.TryGetValue("concurrency", out var concurrencyStr) ? concurrencyStr.Split(',', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries) @@ -653,6 +783,11 @@ public static LoadTestOptions Parse(string[] args) HeartbeatTimeoutSeconds = int.Parse(map.GetValueOrDefault("heartbeat-timeout", "120")), MinConnections = minConnections, MaxConnections = maxConnections, + UseStaticEndpoints = useStaticEndpoints, + StaticEndpointCount = staticEndpointCount, + UseDynamicResolver = useDynamicResolver, + DynamicEndpointCount = dynamicEndpointCount, + StaticLoadBalancingStrategy = staticLoadBalancingStrategy, PerformanceProfile = profile, RequestTimeoutMode = requestTimeoutMode, AdmissionMode = admissionMode, diff --git a/test/SharpLink.PackageSmoke/Program.cs b/test/SharpLink.PackageSmoke/Program.cs index 62d82f6..33f46ac 100644 --- a/test/SharpLink.PackageSmoke/Program.cs +++ b/test/SharpLink.PackageSmoke/Program.cs @@ -41,6 +41,7 @@ public static async Task Main() using var timeout = new CancellationTokenSource(TimeSpan.FromSeconds(30)); await RunTransportSmokeAsync(useSharedMemory: false, timeout.Token); await RunTransportSmokeAsync(useSharedMemory: true, timeout.Token); + await RunStaticEndpointSmokeAsync(timeout.Token); } private static async Task RunTransportSmokeAsync( @@ -97,6 +98,79 @@ private static async Task RunTransportSmokeAsync( } } + private static async Task RunStaticEndpointSmokeAsync(CancellationToken cancellationToken) + { + var firstBuilder = SharpLinkServerBuilder.Create() + .UseRuntime(ConfigureCompression) + .UseAdmissionControl(options => options.Global.UseConcurrency(64)) + .UseTcp(0, IPAddress.Loopback.ToString()); + var secondBuilder = SharpLinkServerBuilder.Create() + .UseRuntime(ConfigureCompression) + .UseAdmissionControl(options => options.Global.UseConcurrency(64)) + .UseTcp(0, IPAddress.Loopback.ToString()); + var firstPort = ((IPEndPoint)firstBuilder.Transport!.LocalEndPoint!).Port; + var secondPort = ((IPEndPoint)secondBuilder.Transport!.LocalEndPoint!).Port; + var firstServer = firstBuilder.Build(); + var secondServer = secondBuilder.Build(); + var firstTask = RunServerAsync(firstServer, cancellationToken); + var secondTask = RunServerAsync(secondServer, cancellationToken); + var endpoints = new[] + { + new SharpLinkEndpoint + { + Id = "first", + Address = new SharpLinkTcpAddress(IPAddress.Loopback.ToString(), firstPort), + Attributes = new Dictionary { ["zone"] = "a" } + }, + new SharpLinkEndpoint + { + Id = "second", + Address = new SharpLinkTcpAddress(IPAddress.Loopback.ToString(), secondPort), + Attributes = new Dictionary { ["zone"] = "b" } + } + }; + var client = SharpClientBuilder.Create() + .UseRuntime(ConfigureCompression) + .UseEndpoints( + endpoints, + SharpLinkTransportFactories.Sockets()) + .UseCluster(options => + { + options.MinReadyEndpoints = 2; + options.MaxConnections = 2; + options.MaxConnectionsPerEndpoint = 1; + }) + .UseLoadBalancing(SharpLinkLoadBalancingStrategy.RoundRobin) + .Build(); + + try + { + await client.ConnectAsync(cancellationToken); + if (await client.Get().AddAsync(20, 22) != 42) + throw new InvalidOperationException("Static endpoint package smoke returned an unexpected result."); + + await using var dynamicClient = SharpClientBuilder.Create() + .UseRuntime(ConfigureCompression) + .UseEndpointResolver( + new DelegateSharpLinkEndpointResolver( + _ => ValueTask.FromResult(new SharpLinkEndpointSnapshot(1, endpoints))), + SharpLinkTransportFactories.Sockets()) + .UseLoadBalancing(SharpLinkLoadBalancingStrategy.RoundRobin) + .Build(); + await dynamicClient.ConnectAsync(cancellationToken); + if (await dynamicClient.Get().AddAsync(20, 22) != 42) + throw new InvalidOperationException("Dynamic endpoint package smoke returned an unexpected result."); + } + finally + { + await client.DisposeAsync(); + await firstServer.DisposeAsync(); + await secondServer.DisposeAsync(); + await Task.WhenAny(firstTask, Task.Delay(TimeSpan.FromSeconds(2), CancellationToken.None)); + await Task.WhenAny(secondTask, Task.Delay(TimeSpan.FromSeconds(2), CancellationToken.None)); + } + } + private static async Task RunServerAsync(ISharpLinkServer server, CancellationToken cancellationToken) { try diff --git a/test/SharpLink.UnitTests/Abstractions/SharpLinkTelemetryTests.cs b/test/SharpLink.UnitTests/Abstractions/SharpLinkTelemetryTests.cs index 3c6ef9b..529692e 100644 --- a/test/SharpLink.UnitTests/Abstractions/SharpLinkTelemetryTests.cs +++ b/test/SharpLink.UnitTests/Abstractions/SharpLinkTelemetryTests.cs @@ -202,6 +202,62 @@ public void AdmissionMetricsShouldExposeStableNamesAndLowCardinalityReasons() "admission rejection low-cardinality tags"); } + [Test] + public void ClientTopologyMetricsShouldExposeStableLowCardinalityInstruments() + { + var resolverUpdates = 0L; + var resolverFailures = 0L; + var instruments = new HashSet(StringComparer.Ordinal); + using var listener = new MeterListener(); + listener.InstrumentPublished = (instrument, meterListener) => + { + if (instrument.Meter.Name == "SharpLink" && + instrument.Name.StartsWith("sharplink.client.", StringComparison.Ordinal)) + { + instruments.Add(instrument.Name); + meterListener.EnableMeasurementEvents(instrument); + } + }; + listener.SetMeasurementEventCallback((instrument, measurement, _, _) => + { + switch (instrument.Name) + { + case "sharplink.client.resolver.updates": + Interlocked.Add(ref resolverUpdates, measurement); + break; + case "sharplink.client.resolver.failures": + Interlocked.Add(ref resolverFailures, measurement); + break; + } + }); + listener.Start(); + + SharpLinkTelemetry.AddClientActiveEndpoints(2); + SharpLinkTelemetry.AddClientReadyEndpoints(1); + SharpLinkTelemetry.AddClientDrainingEndpoints(1); + SharpLinkTelemetry.RecordClientResolverUpdate(); + SharpLinkTelemetry.RecordClientResolverFailure(); + SharpLinkTelemetry.ConnectionOpened("client"); + SharpLinkTelemetry.AddClientRetiringConnections(1); + listener.RecordObservableInstruments(); + SharpLinkTelemetry.AddClientActiveEndpoints(-2); + SharpLinkTelemetry.AddClientReadyEndpoints(-1); + SharpLinkTelemetry.AddClientDrainingEndpoints(-1); + SharpLinkTelemetry.AddClientRetiringConnections(-1); + SharpLinkTelemetry.ConnectionClosed("client"); + listener.RecordObservableInstruments(); + + Ensure(instruments.Contains("sharplink.client.endpoints.active"), "active endpoints instrument"); + Ensure(instruments.Contains("sharplink.client.endpoints.ready"), "ready endpoints instrument"); + Ensure(instruments.Contains("sharplink.client.endpoints.draining"), "draining endpoints instrument"); + Ensure(instruments.Contains("sharplink.client.resolver.updates"), "resolver updates instrument"); + Ensure(instruments.Contains("sharplink.client.resolver.failures"), "resolver failures instrument"); + Ensure(instruments.Contains("sharplink.client.connections.active"), "active connections instrument"); + Ensure(instruments.Contains("sharplink.client.connections.retiring"), "retiring connections instrument"); + Ensure(Volatile.Read(ref resolverUpdates) == 1, "resolver updates metric"); + Ensure(Volatile.Read(ref resolverFailures) == 1, "resolver failures metric"); + } + private static string? FindTag( ReadOnlySpan> tags, string key) diff --git a/test/SharpLink.UnitTests/Client/ClientInvokerTestHelper.cs b/test/SharpLink.UnitTests/Client/ClientInvokerTestHelper.cs index 0498bc0..bbb83f4 100644 --- a/test/SharpLink.UnitTests/Client/ClientInvokerTestHelper.cs +++ b/test/SharpLink.UnitTests/Client/ClientInvokerTestHelper.cs @@ -16,6 +16,16 @@ internal static class ClientInvokerTestHelper HasMethodTimeout: false, MethodTimeout: null); + private static readonly RpcMethodDescriptor SIdempotentUnaryMethod = new( + 1, + 4, + RpcMethodKind.Unary, + HasResponsePayload: true, + HasClientStreams: false, + HasMethodTimeout: false, + MethodTimeout: null, + IsIdempotent: true); + private static readonly RpcMethodDescriptor SOneWayMethod = new( 1, 2, @@ -50,6 +60,22 @@ public static ValueTask InvokeUnaryAsync( cancellationToken); } + public static ValueTask InvokeIdempotentUnaryAsync( + SharpLinkClient client, + SharpLinkCallOptions options = default, + CancellationToken cancellationToken = default) + { + var channel = (IRpcChannel)client; + var request = default(RpcEmptyRequest); + return channel.InvokeUnaryAsync( + SIdempotentUnaryMethod, + in request, + RpcEmptyRequestCodec.Instance, + channel.RuntimeContext.Codecs.GetCodec(), + options, + cancellationToken); + } + public static ValueTask InvokeOneWayAsync(SharpLinkClient client) { var channel = (IRpcChannel)client; diff --git a/test/SharpLink.UnitTests/Client/DynamicEndpointResolverTests.cs b/test/SharpLink.UnitTests/Client/DynamicEndpointResolverTests.cs new file mode 100644 index 0000000..fb9cf0f --- /dev/null +++ b/test/SharpLink.UnitTests/Client/DynamicEndpointResolverTests.cs @@ -0,0 +1,236 @@ +using System; +using System.Collections.Generic; +using System.Net; +using System.Net.Sockets; +using System.Runtime.CompilerServices; +using System.Threading; +using System.Threading.Tasks; +using SharpLink.Client; + +namespace SharpLink.UnitTests.Client; + +public sealed class DynamicEndpointResolverTests +{ + [Test] + public async Task DnsResolverShouldNormalizeOrderAndRetainTheLastGoodSnapshot() + { + var query = new TestDnsQuery + { + Addresses = + [ + IPAddress.Parse("::1"), + IPAddress.Parse("127.0.0.1"), + IPAddress.Parse("127.0.0.1") + ] + }; + await using var resolver = new SharpLinkDnsEndpointResolver( + "service.example", + 5001, + new SharpLinkDnsResolverOptions + { + RefreshInterval = TimeSpan.FromMilliseconds(1), + MinimumRefreshInterval = TimeSpan.FromMilliseconds(1), + MaximumRefreshInterval = TimeSpan.FromSeconds(1), + JitterRatio = 0 + }, + query); + + var first = await resolver.ResolveAsync(CancellationToken.None); + Ensure(first.Version == 1, "first DNS snapshot version"); + Ensure(first.Endpoints.Count == 2, "DNS endpoint de-duplication"); + Ensure(first.Endpoints[0].Authority == "service.example", "DNS authority"); + + query.Addresses = [IPAddress.Parse("127.0.0.1"), IPAddress.Parse("::1")]; + var reordered = await resolver.ResolveAsync(CancellationToken.None); + Ensure(reordered.Version == first.Version, "DNS order-only change must not publish a topology update"); + + query.Throw = true; + var retained = await resolver.ResolveAsync(CancellationToken.None); + Ensure(retained.Version == first.Version, "DNS lookup failure must retain the last good topology"); + } + + [Test] + public async Task DnsResolverShouldBoundGeneratedIdsForAValidLongHostname() + { + var label = new string('a', 63); + var host = $"{label}.{label}.{label}.{new string('a', 61)}"; + var query = new TestDnsQuery { Addresses = [IPAddress.Loopback] }; + await using var resolver = new SharpLinkDnsEndpointResolver( + host, + 5001, + new SharpLinkDnsResolverOptions(), + query); + + var snapshot = await resolver.ResolveAsync(CancellationToken.None); + + Ensure(snapshot.Endpoints.Count == 1, "long-host DNS endpoint count"); + Ensure(snapshot.Endpoints[0].Id.Length <= 256, "long-host DNS endpoint ID limit"); + Ensure(snapshot.Endpoints[0].Authority == host, "long-host DNS authority preservation"); + } + + [Test] + public async Task DnsResolverShouldFilterMappedIpv4AddressesByTheirNormalizedFamily() + { + var query = new TestDnsQuery { Addresses = [IPAddress.Parse("::ffff:127.0.0.1")] }; + await using var ipv4 = new SharpLinkDnsEndpointResolver( + "service.example", + 5001, + new SharpLinkDnsResolverOptions { AddressFamily = AddressFamily.InterNetwork }, + query); + await using var ipv6 = new SharpLinkDnsEndpointResolver( + "service.example", + 5001, + new SharpLinkDnsResolverOptions { AddressFamily = AddressFamily.InterNetworkV6 }, + query); + + var ipv4Snapshot = await ipv4.ResolveAsync(CancellationToken.None); + var ipv6Snapshot = await ipv6.ResolveAsync(CancellationToken.None); + + Ensure(ipv4Snapshot.Endpoints.Count == 1, "mapped IPv4 must satisfy an IPv4 family filter"); + Ensure(ipv4Snapshot.Endpoints[0].Address is SharpLinkTcpAddress { Host: "127.0.0.1" }, + "mapped IPv4 endpoint must publish its normalized address"); + Ensure(ipv6Snapshot.Endpoints.Count == 0, "mapped IPv4 must not satisfy an IPv6 family filter"); + } + + [Test] + public async Task DynamicBuilderShouldOwnResolverAndRejectFixedTransportConflict() + { + var resolver = new TrackingResolver(); + await using (var client = SharpClientBuilder.Create() + .UseEndpointResolver(resolver, _ => new TrackingFactory()) + .Build()) + { + } + Ensure(resolver.DisposeCount == 1, "resolver should be disposed exactly once"); + + await EnsureThrows(() => + { + _ = SharpClientBuilder.Create() + .UseTransport(new TrackingFactory()) + .UseEndpointResolver(new TrackingResolver(), _ => new TrackingFactory()) + .Build(); + return Task.CompletedTask; + }); + } + + [Test] + public async Task DynamicBuilderShouldCapMinReadyByMaxEndpoints() + { + var resolver = new TrackingResolver(); + await using var client = SharpClientBuilder.Create() + .UseEndpointResolver(resolver, _ => new TrackingFactory()) + .UseCluster(options => + { + options.MaxEndpoints = 1; + options.MinReadyEndpoints = 2; + options.MaxConnections = 1; + options.MaxConnectionsPerEndpoint = 1; + }) + .Build(); + } + + [Test] + public async Task DynamicClusterShouldRejectAnonymousPipeFactories() + { + var resolver = new SingleSnapshotResolver(new SharpLinkEndpointSnapshot(0, + [ + new SharpLinkEndpoint + { + Id = "pipe", + Address = new SharpLinkAnonymousPipeAddress("in-handle", "out-handle") + } + ])); + await using var client = SharpClientBuilder.Create() + .UseEndpointResolver(resolver, _ => new AnonymousPipeClientTransportFactory("in-handle", "out-handle")) + .Build(); + + try + { + await client.ConnectAsync(); + throw new Exception("expected anonymous-pipe dynamic cluster rejection"); + } + catch (SharpLinkException exception) + { + Ensure(exception.Code == SharpLinkErrorCode.Unavailable, "dynamic cluster rejection code"); + Ensure(exception.InnerException is InvalidOperationException { + Message: "The endpoint resolver returned an invalid initial topology." + }, "dynamic cluster must reject the anonymous-pipe factory before attempting a connection"); + } + } + + private static async Task EnsureThrows(Func action) where TException : Exception + { + try + { + await action(); + throw new Exception($"expected {typeof(TException).Name}"); + } + catch (TException) + { + } + } + + private static void Ensure(bool condition, string message) + { + if (!condition) + throw new Exception(message); + } + + private sealed class TestDnsQuery : ISharpLinkDnsQuery + { + public IPAddress[] Addresses { get; set; } = []; + public bool Throw { get; set; } + + public ValueTask QueryAsync(string host, CancellationToken cancellationToken) + { + if (Throw) + return ValueTask.FromException(new SocketException()); + return ValueTask.FromResult(Addresses); + } + } + + private sealed class TrackingResolver : ISharpLinkEndpointResolver + { + private int _disposeCount; + public int DisposeCount => Volatile.Read(ref _disposeCount); + + public ValueTask ResolveAsync(CancellationToken cancellationToken) + => ValueTask.FromResult(new SharpLinkEndpointSnapshot(0, [])); + + public async IAsyncEnumerable WatchAsync( + [EnumeratorCancellation] CancellationToken cancellationToken) + { + await Task.Delay(Timeout.InfiniteTimeSpan, cancellationToken); + yield break; + } + + public ValueTask DisposeAsync() + { + Interlocked.Increment(ref _disposeCount); + return ValueTask.CompletedTask; + } + } + + private sealed class SingleSnapshotResolver(SharpLinkEndpointSnapshot snapshot) : ISharpLinkEndpointResolver + { + public ValueTask ResolveAsync(CancellationToken cancellationToken) + => ValueTask.FromResult(snapshot); + + public async IAsyncEnumerable WatchAsync( + [EnumeratorCancellation] CancellationToken cancellationToken) + { + await Task.Delay(Timeout.InfiniteTimeSpan, cancellationToken); + yield break; + } + + public ValueTask DisposeAsync() => ValueTask.CompletedTask; + } + + private sealed class TrackingFactory : IClientTransportFactory + { + public ValueTask ConnectAsync(CancellationToken cancellationToken = default) + => ValueTask.FromException(new NotSupportedException()); + + public ValueTask DisposeAsync() => ValueTask.CompletedTask; + } +} diff --git a/test/SharpLink.UnitTests/Client/SharpLinkClientCallOptionsTests.cs b/test/SharpLink.UnitTests/Client/SharpLinkClientCallOptionsTests.cs index b825d46..ee76255 100644 --- a/test/SharpLink.UnitTests/Client/SharpLinkClientCallOptionsTests.cs +++ b/test/SharpLink.UnitTests/Client/SharpLinkClientCallOptionsTests.cs @@ -1,4 +1,7 @@ using System.Threading; +using System.Collections.Generic; +using System.Diagnostics; +using System.Buffers.Binary; using SharpLink.Client; using SharpLink.Sdk; @@ -67,6 +70,298 @@ public async Task WaitForReadyDeadlineShouldMapToDeadlineExceeded() Ensure(exception.Code == SharpLinkErrorCode.DeadlineExceeded, "wait deadline error code"); } + [Test] + public async Task WaitForReadyShouldRetryZeroAdmissionDelayWithABoundedYield() + { + var transport = new TestClientTransportFactory(); + var policy = new RejectOnceWithZeroDelayPolicy(); + await using var client = new SharpLinkClient( + transport, + TimeSpan.FromSeconds(10), + TimeSpan.FromSeconds(30), + fixedEndpoint: FixedEndpoint, + endpointAdmissionPolicy: policy); + await client.ConnectAsync(); + + var invocation = ClientInvokerTestHelper.InvokeUnaryAsync( + client, + new SharpLinkCallOptions { WaitForReady = true }).AsTask(); + var request = await transport.Connection.WaitForSentPacket(ProtocolV2FrameType.Request); + await transport.Connection.InjectPacketAsync( + ProtocolV2FrameType.Response, + ProtocolV2FrameFlags.None, + unchecked((long)request.RequestId)); + + Ensure(await invocation == 0, "zero-delay admission rejection should retry"); + Ensure(policy.AcquireCount >= 2, "admission should be retried after the bounded yield"); + Ensure(policy.ReportCount == 1, "only the granted admission lease should report"); + } + + [Test] + public async Task ClientStopShouldCancelWaitForReadyAdmissionDelayPromptly() + { + var transport = new TestClientTransportFactory(); + var policy = new SignaledRejectWithDelayPolicy(TimeSpan.MaxValue); + await using var client = new SharpLinkClient( + transport, + TimeSpan.FromSeconds(10), + TimeSpan.FromSeconds(30), + fixedEndpoint: FixedEndpoint, + endpointAdmissionPolicy: policy); + await client.ConnectAsync(); + + var invocation = ClientInvokerTestHelper.InvokeUnaryAsync( + client, new SharpLinkCallOptions { WaitForReady = true }).AsTask(); + await policy.RejectionStarted.WaitAsync(TimeSpan.FromSeconds(2)); + + var stoppedAt = Stopwatch.GetTimestamp(); + var stop = client.StopAsync().AsTask(); + var exception = await CaptureSharpLinkException(invocation.WaitAsync(TimeSpan.FromSeconds(2))); + await stop.WaitAsync(TimeSpan.FromSeconds(2)); + + Ensure(exception.Code == SharpLinkErrorCode.ConnectionClosed, "stopped admission delay error code"); + Ensure(Stopwatch.GetElapsedTime(stoppedAt) < TimeSpan.FromSeconds(1), + "client stop must cancel the wait-for-ready admission delay promptly"); + } + + [Test] + public async Task WaitForReadyShouldDiscardAStaleAdmissionDelayAfterAnAdmittedEndpointDisconnects() + { + var first = new TestClientTransportFactory(); + var second = new TestClientTransportFactory(); + var blockingSecond = new ReconnectBlockingFactory(second); + var policy = new RejectFirstThenBlockSecondAdmissionPolicy(); + var endpoints = new[] + { + new StaticEndpointConfiguration(Endpoint("first", 5001), first), + new StaticEndpointConfiguration(Endpoint("second", 5002), blockingSecond) + }; + await using var client = new SharpLinkClient( + first, + TimeSpan.FromSeconds(10), + TimeSpan.FromSeconds(30), + staticEndpoints: endpoints, + clusterOptions: new SharpLinkClusterOptions { MinReadyEndpoints = 2, MaxConnections = 2 }, + endpointSelector: new FirstUnexcludedSelector(), + endpointAdmissionPolicy: policy); + await client.ConnectAsync(); + await WaitForReadyConnectionCountAsync(client, 2); + + var invocation = Task.Run(async () => await ClientInvokerTestHelper.InvokeUnaryAsync( + client, new SharpLinkCallOptions { WaitForReady = true })); + await policy.SecondAdmissionEntered.WaitAsync(TimeSpan.FromSeconds(2)); + await InjectGoAwayAsync(second.Connection); + await blockingSecond.ReconnectStarted.WaitAsync(TimeSpan.FromSeconds(2)); + await WaitForReadyConnectionCountAsync(client, 1); + var freshRejectionDelay = TimeSpan.FromMilliseconds(120); + policy.RejectNextFirstAdmission(freshRejectionDelay); + var releasedAt = Stopwatch.GetTimestamp(); + policy.ReleaseSecondAdmission(); + + var request = await first.Connection.WaitForSentPacket(ProtocolV2FrameType.Request) + .WaitAsync(TimeSpan.FromSeconds(5)); + await first.Connection.InjectPacketAsync( + ProtocolV2FrameType.Response, + ProtocolV2FrameFlags.None, + unchecked((long)request.RequestId)); + + Ensure(await invocation == 0, "a granted endpoint disconnect must not retain a previous rejection delay"); + Ensure(Stopwatch.GetElapsedTime(releasedAt) >= freshRejectionDelay - TimeSpan.FromMilliseconds(25), + "a fresh all-rejected selection must honor its retry delay after the lost grant"); + Ensure(policy.FreshFirstRejectionCount == 1, "fresh rejection should be sampled exactly once"); + } + + [Test] + public async Task EndpointOutcomeElapsedShouldExcludeWaitForReadyTime() + { + var transport = new TestClientTransportFactory(); + var policy = new RecordingAdmissionPolicy(); + await using var client = new SharpLinkClient( + transport, + TimeSpan.FromSeconds(10), + TimeSpan.FromSeconds(30), + fixedEndpoint: FixedEndpoint, + endpointAdmissionPolicy: policy); + + var invocation = ClientInvokerTestHelper.InvokeUnaryAsync( + client, + new SharpLinkCallOptions + { + Timeout = TimeSpan.FromSeconds(2), + WaitForReady = true + }).AsTask(); + await Task.Delay(200); + await client.ConnectAsync(); + var request = await transport.Connection.WaitForSentPacket(ProtocolV2FrameType.Request); + await transport.Connection.InjectPacketAsync( + ProtocolV2FrameType.Response, + ProtocolV2FrameFlags.None, + unchecked((long)request.RequestId)); + + Ensure(await invocation == 0, "wait-for-ready response"); + var measuredEndpointInterval = Stopwatch.GetElapsedTime(policy.LastAdmissionTimestamp, policy.LastReportTimestamp); + var difference = (policy.LastOutcome.Elapsed - measuredEndpointInterval).Duration(); + Ensure(difference < TimeSpan.FromMilliseconds(100), + "endpoint outcome elapsed must measure the interval after admission rather than pre-ready waiting"); + } + + [Test] + public async Task EndpointAdmissionShouldReportMalformedResponsesAsRemoteErrors() + { + var transport = new TestClientTransportFactory(); + var policy = new RecordingAdmissionPolicy(); + await using var client = new SharpLinkClient( + transport, + TimeSpan.FromSeconds(10), + TimeSpan.FromSeconds(30), + fixedEndpoint: FixedEndpoint, + endpointAdmissionPolicy: policy); + await client.ConnectAsync(); + + var invocation = ClientInvokerTestHelper.InvokeUnaryAsync(client).AsTask(); + var request = await transport.Connection.WaitForSentPacket(ProtocolV2FrameType.Request); + await transport.Connection.InjectFrameAsync( + ProtocolV2FrameType.Response, + ProtocolV2FrameFlags.None, + request.RequestId, + new byte[] { 0 }); + + var exception = await CaptureSharpLinkException(invocation); + Ensure(exception.Code == SharpLinkErrorCode.DataLoss, "malformed response error code"); + Ensure(policy.ReportCount == 1, "malformed response admission report count"); + Ensure(policy.LastOutcome.Kind == SharpLinkEndpointOutcomeKind.RemoteError, + "malformed response must report a remote error rather than success"); + Ensure(policy.LastOutcome.ErrorCode == SharpLinkErrorCode.DataLoss, + "malformed response outcome error code"); + Ensure(policy.LastOutcome.ResponseObserved, + "malformed response must still record that the endpoint sent a response"); + } + + [Test] + public async Task WaitForReadyAdmissionDelayBeyondDeadlineShouldNotOverflow() + { + var transport = new TestClientTransportFactory(); + var policy = new RejectWithDelayPolicy(TimeSpan.MaxValue); + await using var client = new SharpLinkClient( + transport, + TimeSpan.FromSeconds(10), + TimeSpan.FromSeconds(30), + fixedEndpoint: FixedEndpoint, + endpointAdmissionPolicy: policy); + await client.ConnectAsync(); + + var exception = await CaptureSharpLinkException(ClientInvokerTestHelper.InvokeUnaryAsync( + client, + new SharpLinkCallOptions { WaitForReady = true, Timeout = TimeSpan.FromSeconds(1) })); + Ensure(exception.Code == SharpLinkErrorCode.DeadlineExceeded, + "oversized admission retry delay must map to deadline exceeded"); + Ensure(policy.AcquireCount == 1, "oversized delay should not loop or issue a request"); + } + + [Test] + public async Task StreamRegistrationFailuresShouldReportAcquiredAdmissionLeases() + { + var transport = new TestClientTransportFactory(); + var policy = new RecordingAdmissionPolicy(); + await using var client = new SharpLinkClient( + transport, + TimeSpan.FromSeconds(10), + TimeSpan.FromSeconds(30), + protocolOptions: new SharpLinkProtocolOptions { MaxPendingRequestsPerConnection = 1 }, + fixedEndpoint: FixedEndpoint, + endpointAdmissionPolicy: policy); + await client.ConnectAsync(); + + var occupied = ClientInvokerTestHelper.InvokeUnaryAsync(client).AsTask(); + var occupiedRequest = await transport.Connection.WaitForSentPacket(ProtocolV2FrameType.Request); + var channel = (IRpcChannel)client; + var request = default(RpcEmptyRequest); + var streams = default(RpcNoClientStreams); + var oneWayFailure = await CaptureSharpLinkException(channel.InvokeOneWayAsync( + OneWayClientStreamingMethod, + in request, + RpcEmptyRequestCodec.Instance, + in streams, + default).AsTask()); + Ensure(oneWayFailure.Code == SharpLinkErrorCode.ResourceExhausted, "one-way stream registration failure"); + + var stream = channel.InvokeServerStreamingAsync( + ServerStreamingMethod, + in request, + RpcEmptyRequestCodec.Instance, + channel.RuntimeContext.Codecs.GetCodec(), + default); + await using var enumerator = stream.GetAsyncEnumerator(); + var serverFailure = await CaptureSharpLinkException(enumerator.MoveNextAsync().AsTask()); + Ensure(serverFailure.Code == SharpLinkErrorCode.ResourceExhausted, "server stream registration failure"); + + Ensure(policy.ReportCount == 2, "both pre-registration failures must report their admission permits"); + Ensure(policy.Outcomes.TrueForAll(static outcome => outcome.Kind == SharpLinkEndpointOutcomeKind.SendFailure), + "pre-registration failures must report send failure outcomes"); + await transport.Connection.InjectPacketAsync( + ProtocolV2FrameType.Response, + ProtocolV2FrameFlags.None, + unchecked((long)occupiedRequest.RequestId)); + Ensure(await occupied == 0, "occupied pending call completion"); + } + + private static readonly RpcMethodDescriptor OneWayClientStreamingMethod = new( + 1, + 31, + RpcMethodKind.OneWay, + HasResponsePayload: false, + HasClientStreams: true, + HasMethodTimeout: false, + MethodTimeout: null); + + private static readonly RpcMethodDescriptor ServerStreamingMethod = new( + 1, + 32, + RpcMethodKind.ServerStreaming, + HasResponsePayload: true, + HasClientStreams: false, + HasMethodTimeout: false, + MethodTimeout: null); + + private static readonly SharpLinkEndpoint FixedEndpoint = new() + { + Id = "fixed", + Address = new SharpLinkTcpAddress("127.0.0.1", 5001) + }; + + private static SharpLinkEndpoint Endpoint(string id, int port) => new() + { + Id = id, + Address = new SharpLinkTcpAddress("127.0.0.1", port) + }; + + private static async Task WaitForReadyConnectionCountAsync(SharpLinkClient client, int expected) + { + var deadline = DateTime.UtcNow + TimeSpan.FromSeconds(5); + while (client.ReadyConnectionCount != expected && DateTime.UtcNow < deadline) + await Task.Delay(10); + Ensure(client.ReadyConnectionCount == expected, $"expected {expected} ready connections"); + } + + private static async Task InjectGoAwayAsync(TestTransportConnection connection) + { + var payload = new PooledByteBufferWriter(); + var lastAccepted = payload.GetSpan(sizeof(ulong)); + BinaryPrimitives.WriteUInt64LittleEndian(lastAccepted, 0); + payload.Advance(sizeof(ulong)); + ProtocolV2PayloadCodec.WriteError( + payload, + SharpLinkErrorCode.Unavailable, + "test endpoint disconnect", + 1024, + out _); + await connection.InjectFrameAsync( + ProtocolV2FrameType.GoAway, + ProtocolV2FrameFlags.Error, + 0, + payload.WrittenMemory); + } + private static async Task CaptureSharpLinkException(ValueTask invocation) { try @@ -80,9 +375,180 @@ private static async Task CaptureSharpLinkException(ValueTas } } + private static async Task CaptureSharpLinkException(Task invocation) + { + try + { + await invocation; + throw new Exception("expected SharpLinkException"); + } + catch (SharpLinkException exception) + { + return exception; + } + } + private static void Ensure(bool condition, string message) { if (!condition) throw new Exception(message); } + + private sealed class RejectOnceWithZeroDelayPolicy : ISharpLinkEndpointAdmissionPolicy + { + public int AcquireCount { get; private set; } + public int ReportCount { get; private set; } + + public SharpLinkEndpointAdmissionDecision TryAcquire( + in SharpLinkEndpointCandidate endpoint, + in RpcMethodDescriptor method) + { + AcquireCount++; + return AcquireCount == 1 + ? new SharpLinkEndpointAdmissionDecision(false, Token: 0, RetryAfter: TimeSpan.Zero) + : new SharpLinkEndpointAdmissionDecision(true, Token: 1, RetryAfter: null); + } + + public void Report(in SharpLinkEndpointOutcome outcome, long token) + { + ReportCount++; + Ensure(token == 1, "zero-delay retry admission token"); + } + } + + private sealed class RejectWithDelayPolicy(TimeSpan retryAfter) : ISharpLinkEndpointAdmissionPolicy + { + public int AcquireCount { get; private set; } + + public SharpLinkEndpointAdmissionDecision TryAcquire( + in SharpLinkEndpointCandidate endpoint, + in RpcMethodDescriptor method) + { + AcquireCount++; + return new SharpLinkEndpointAdmissionDecision(false, Token: 0, RetryAfter: retryAfter); + } + + public void Report(in SharpLinkEndpointOutcome outcome, long token) + { + } + } + + private sealed class SignaledRejectWithDelayPolicy(TimeSpan retryAfter) : ISharpLinkEndpointAdmissionPolicy + { + private readonly TaskCompletionSource _rejectionStarted = + new(TaskCreationOptions.RunContinuationsAsynchronously); + + public Task RejectionStarted => _rejectionStarted.Task; + + public SharpLinkEndpointAdmissionDecision TryAcquire( + in SharpLinkEndpointCandidate endpoint, + in RpcMethodDescriptor method) + { + _rejectionStarted.TrySetResult(); + return new SharpLinkEndpointAdmissionDecision(false, Token: 0, RetryAfter: retryAfter); + } + + public void Report(in SharpLinkEndpointOutcome outcome, long token) + { + } + } + + private sealed class FirstUnexcludedSelector : ISharpLinkEndpointSelector + { + public int Select(in SharpLinkEndpointSelectionContext context) + { + for (var index = 0; index < context.Count; index++) + if ((context.ExcludedMask & (1UL << index)) == 0) + return index; + return -1; + } + } + + private sealed class RejectFirstThenBlockSecondAdmissionPolicy : ISharpLinkEndpointAdmissionPolicy + { + private readonly TaskCompletionSource _secondAdmissionEntered = new(TaskCreationOptions.RunContinuationsAsynchronously); + private readonly TaskCompletionSource _secondAdmissionRelease = new(TaskCreationOptions.RunContinuationsAsynchronously); + private int _firstAdmissions; + private long _nextFirstAdmissionRetryAfterTicks; + private int _freshFirstRejectionCount; + + public Task SecondAdmissionEntered => _secondAdmissionEntered.Task; + + public int FreshFirstRejectionCount => Volatile.Read(ref _freshFirstRejectionCount); + + public SharpLinkEndpointAdmissionDecision TryAcquire( + in SharpLinkEndpointCandidate endpoint, + in RpcMethodDescriptor method) + { + if (endpoint.Endpoint.Id == "first") + { + var freshRetryAfterTicks = Interlocked.Exchange(ref _nextFirstAdmissionRetryAfterTicks, 0); + if (freshRetryAfterTicks != 0) + { + Interlocked.Increment(ref _freshFirstRejectionCount); + return new SharpLinkEndpointAdmissionDecision(false, Token: 0, + RetryAfter: TimeSpan.FromTicks(freshRetryAfterTicks)); + } + return Interlocked.Increment(ref _firstAdmissions) == 1 + ? new SharpLinkEndpointAdmissionDecision(false, Token: 0, RetryAfter: TimeSpan.FromSeconds(30)) + : new SharpLinkEndpointAdmissionDecision(true, Token: 1, RetryAfter: null); + } + + _secondAdmissionEntered.TrySetResult(); + _secondAdmissionRelease.Task.GetAwaiter().GetResult(); + return new SharpLinkEndpointAdmissionDecision(true, Token: 2, RetryAfter: null); + } + + public void RejectNextFirstAdmission(TimeSpan retryAfter) + => Interlocked.Exchange(ref _nextFirstAdmissionRetryAfterTicks, retryAfter.Ticks); + + public void ReleaseSecondAdmission() => _secondAdmissionRelease.TrySetResult(); + + public void Report(in SharpLinkEndpointOutcome outcome, long token) + { + } + } + + private sealed class ReconnectBlockingFactory(TestClientTransportFactory inner) : IClientTransportFactory + { + private readonly TaskCompletionSource _reconnectStarted = new(TaskCreationOptions.RunContinuationsAsynchronously); + private int _connectCount; + + public Task ReconnectStarted => _reconnectStarted.Task; + + public async ValueTask ConnectAsync(CancellationToken cancellationToken = default) + { + if (Interlocked.Increment(ref _connectCount) == 1) + return await inner.ConnectAsync(cancellationToken).ConfigureAwait(false); + + _reconnectStarted.TrySetResult(); + await Task.Delay(Timeout.InfiniteTimeSpan, cancellationToken).ConfigureAwait(false); + throw new UnreachableException(); + } + + public ValueTask DisposeAsync() => inner.DisposeAsync(); + } + + private sealed class RecordingAdmissionPolicy : ISharpLinkEndpointAdmissionPolicy + { + public List Outcomes { get; } = []; + public int ReportCount => Outcomes.Count; + public SharpLinkEndpointOutcome LastOutcome => Outcomes[^1]; + public long LastAdmissionTimestamp { get; private set; } + public long LastReportTimestamp { get; private set; } + + public SharpLinkEndpointAdmissionDecision TryAcquire( + in SharpLinkEndpointCandidate endpoint, + in RpcMethodDescriptor method) + { + LastAdmissionTimestamp = Stopwatch.GetTimestamp(); + return new SharpLinkEndpointAdmissionDecision(true, Token: method.MethodId, RetryAfter: null); + } + + public void Report(in SharpLinkEndpointOutcome outcome, long token) + { + LastReportTimestamp = Stopwatch.GetTimestamp(); + Outcomes.Add(outcome); + } + } } diff --git a/test/SharpLink.UnitTests/Client/SharpLinkClientLifecycleStateTests.cs b/test/SharpLink.UnitTests/Client/SharpLinkClientLifecycleStateTests.cs index b5a1d4c..c3ef191 100644 --- a/test/SharpLink.UnitTests/Client/SharpLinkClientLifecycleStateTests.cs +++ b/test/SharpLink.UnitTests/Client/SharpLinkClientLifecycleStateTests.cs @@ -1,6 +1,7 @@ using System.Buffers.Binary; using System.Collections.Generic; using System.Net.Sockets; +using System.Reflection; using System.Threading; using SharpLink.Client; @@ -183,6 +184,88 @@ public async Task PowerOfTwoChoiceShouldSelectLowerActiveConnection() await ObserveFailureAsync(secondCall.AsValueTask()); } + [Test] + public async Task ClusterSelectionShouldFallBackFromAStalePooledConnection() + { + await using var owner = new SharpLinkClient( + new TestClientTransportFactory(), + TimeSpan.FromSeconds(10), + TimeSpan.FromSeconds(30)); + var context = new SharpLinkRuntimeContextBuilder().Build(); + await using var stale = new ClientConnection( + owner, + new RpcSession(new TestTransportConnection()), + new CancellationTokenSource(), + 8, + context.Codecs); + await using var ready = new ClientConnection( + owner, + new RpcSession(new TestTransportConnection()), + new CancellationTokenSource(), + 8, + context.Codecs); + stale.Session.NotifyConnected(); + ready.Session.NotifyConnected(); + Ensure(ready.TryBeginUntrackedCall(), "ready connection active-call setup"); + stale.MarkDraining(); + + try + { + Ensure(ReferenceEquals( + SelectClusterConnection("StaticClusterRuntime", 0, stale, ready), + ready), + "static cluster should fall back to an accepting pooled connection"); + Ensure(ReferenceEquals( + SelectClusterConnection("DynamicClusterRuntime", 0L, stale, ready), + ready), + "dynamic cluster should fall back to an accepting pooled connection"); + } + finally + { + ready.EndUntrackedCall(); + } + } + + [Test] + public async Task AdmissionRetryAfterShouldSurviveAStaleGrantedConnection() + { + var policy = new AdmitFirstRejectSecondPolicy(TimeSpan.FromMilliseconds(100)); + await using var client = new SharpLinkClient( + new TestClientTransportFactory(), + TimeSpan.FromSeconds(10), + TimeSpan.FromSeconds(30), + endpointAdmissionPolicy: policy); + var stateType = typeof(SharpLinkClient).GetNestedType("AttemptOutcomeState", BindingFlags.NonPublic) + ?? throw new Exception("cannot find attempt outcome state"); + var method = new RpcMethodDescriptor(1, 2, RpcMethodKind.Unary, true, false, false, null); + var state = Activator.CreateInstance( + stateType, + BindingFlags.Instance | BindingFlags.Public | BindingFlags.NonPublic, + binder: null, + args: [client, method], + culture: null) + ?? throw new Exception("cannot create attempt outcome state"); + var tryAcquire = stateType.GetMethod("TryAcquire", BindingFlags.Instance | BindingFlags.Public) + ?? throw new Exception("cannot find attempt acquisition"); + var complete = stateType.GetMethod("CompleteWithoutPending", BindingFlags.Instance | BindingFlags.Public) + ?? throw new Exception("cannot find attempt completion"); + var shouldHonor = stateType.GetProperty("ShouldHonorAdmissionRetryAfter", BindingFlags.Instance | BindingFlags.Public) + ?? throw new Exception("cannot find retry-after predicate"); + var first = new SharpLinkEndpointCandidate(CreateEndpoint("first", 5001), 1, 0, generation: 1); + var second = new SharpLinkEndpointCandidate(CreateEndpoint("second", 5002), 1, 0, generation: 1); + + Ensure((bool)(tryAcquire.Invoke(state, [first]) ?? false), "first endpoint should be admitted"); + complete.Invoke( + state, + [ + PendingCallCompletionReason.ConnectionClosed, + new SharpLinkException(SharpLinkErrorCode.ConnectionClosed, "selected connection became stale") + ]); + Ensure(!(bool)(tryAcquire.Invoke(state, [second]) ?? true), "second endpoint should be rejected"); + Ensure((bool)(shouldHonor.GetValue(state) ?? false), + "a stale admitted endpoint must not suppress the current selection retry-after"); + } + [Test] public async Task GoAwayShouldDrainOnlyItsConnectionAndRefillMinimumPool() { @@ -198,6 +281,48 @@ public async Task GoAwayShouldDrainOnlyItsConnectionAndRefillMinimumPool() }); await client.ConnectAsync(); var drainingConnection = await transport.WaitForConnectionAsync(0); + await InjectGoAwayAsync(drainingConnection); + await WaitUntilAsync(() => transport.ConnectCount >= 3 && client.ReadyConnectionCount == 2); + + Ensure(client.State == SharpLinkConnectionState.Ready, "another ready connection should keep the client ready"); + } + + [Test] + public async Task GoAwayShouldCountAsBreakerFailureWithoutAnActiveCall() + { + var transport = new TestClientTransportFactory(); + var endpoint = new SharpLinkEndpoint + { + Id = "breaker", + Address = new SharpLinkTcpAddress("127.0.0.1", 5001) + }; + var breaker = new SharpLinkCircuitBreaker(new SharpLinkCircuitBreakerOptions + { + MinimumThroughput = 1, + FailureRatio = 1, + SamplingDuration = TimeSpan.FromSeconds(10), + BreakDuration = TimeSpan.FromSeconds(5), + HalfOpenMaxCalls = 1 + }.CloneValidated()); + await using var client = new SharpLinkClient( + transport, + TimeSpan.FromSeconds(10), + TimeSpan.FromSeconds(30), + fixedEndpoint: endpoint, + endpointAdmissionPolicy: breaker); + await client.ConnectAsync(); + + await InjectGoAwayAsync(transport.Connection); + + var candidate = new SharpLinkEndpointCandidate(endpoint, 0, 0, generation: 0); + var method = new RpcMethodDescriptor(1, 2, RpcMethodKind.Unary, true, false, false, null); + await WaitUntilAsync( + () => !breaker.TryAcquire(candidate, method).IsAllowed, + () => "GoAway was not recorded as an endpoint infrastructure failure"); + } + + private static async Task InjectGoAwayAsync(TestTransportConnection connection) + { var payload = new PooledByteBufferWriter(); var lastAccepted = payload.GetSpan(sizeof(ulong)); BinaryPrimitives.WriteUInt64LittleEndian(lastAccepted, 0); @@ -209,14 +334,11 @@ public async Task GoAwayShouldDrainOnlyItsConnectionAndRefillMinimumPool() 1024, out _); - await drainingConnection.InjectFrameAsync( + await connection.InjectFrameAsync( ProtocolV2FrameType.GoAway, ProtocolV2FrameFlags.Error, 0, payload.WrittenMemory); - await WaitUntilAsync(() => transport.ConnectCount >= 3 && client.ReadyConnectionCount == 2); - - Ensure(client.State == SharpLinkConnectionState.Ready, "another ready connection should keep the client ready"); } private static async Task WaitUntilAsync(Func condition, Func? timeoutMessage = null) @@ -350,4 +472,67 @@ public async ValueTask DisposeAsync() await connections[index].DisposeAsync(); } } + + private static ClientConnection? SelectClusterConnection( + string runtimeName, + object stateIndex, + ClientConnection stale, + ClientConnection ready) + { + var flags = BindingFlags.NonPublic | BindingFlags.Public; + var runtimeType = typeof(SharpLinkClient).GetNestedType(runtimeName, BindingFlags.NonPublic) + ?? throw new Exception($"cannot find {runtimeName}"); + var endpointType = runtimeType.GetNestedType("EndpointState", flags) + ?? throw new Exception($"cannot find {runtimeName}.EndpointState"); + var configuration = new StaticEndpointConfiguration( + new SharpLinkEndpoint + { + Id = "selection", + Address = new SharpLinkTcpAddress("127.0.0.1", 5001) + }, + new NonConnectingFactory()); + var endpoint = Activator.CreateInstance( + endpointType, + BindingFlags.Instance | flags, + binder: null, + args: [configuration, stateIndex], + culture: null) + ?? throw new Exception($"cannot create {runtimeName}.EndpointState"); + var readyConnections = endpointType.GetField("_readyConnections", BindingFlags.Instance | BindingFlags.NonPublic) + ?? throw new Exception($"cannot find {runtimeName} ready connection field"); + readyConnections.SetValue(endpoint, new[] { stale, ready }); + var selectConnection = runtimeType.GetMethod( + "SelectConnection", + BindingFlags.Static | BindingFlags.NonPublic) + ?? throw new Exception($"cannot find {runtimeName} selection method"); + return (ClientConnection?)selectConnection.Invoke(null, [endpoint]); + } + + private static SharpLinkEndpoint CreateEndpoint(string id, int port) => new() + { + Id = id, + Address = new SharpLinkTcpAddress("127.0.0.1", port) + }; + + private sealed class NonConnectingFactory : IClientTransportFactory + { + public ValueTask ConnectAsync(CancellationToken cancellationToken = default) + => ValueTask.FromException(new NotSupportedException()); + + public ValueTask DisposeAsync() => ValueTask.CompletedTask; + } + + private sealed class AdmitFirstRejectSecondPolicy(TimeSpan retryAfter) : ISharpLinkEndpointAdmissionPolicy + { + public SharpLinkEndpointAdmissionDecision TryAcquire( + in SharpLinkEndpointCandidate endpoint, + in RpcMethodDescriptor method) + => endpoint.Endpoint.Id == "first" + ? new SharpLinkEndpointAdmissionDecision(true, Token: 1, RetryAfter: null) + : new SharpLinkEndpointAdmissionDecision(false, Token: 0, RetryAfter: retryAfter); + + public void Report(in SharpLinkEndpointOutcome outcome, long token) + { + } + } } diff --git a/test/SharpLink.UnitTests/Client/SharpLinkClientRetryTests.cs b/test/SharpLink.UnitTests/Client/SharpLinkClientRetryTests.cs new file mode 100644 index 0000000..a9f7851 --- /dev/null +++ b/test/SharpLink.UnitTests/Client/SharpLinkClientRetryTests.cs @@ -0,0 +1,716 @@ +using System.Threading; +using System.Diagnostics; +using SharpLink.Client; +using SharpLink.Sdk; + +namespace SharpLink.UnitTests.Client; + +public class SharpLinkClientRetryTests +{ + [Test] + public async Task IdempotentUnaryShouldRetryRemoteUnavailableAndExposeResponseObservation() + { + var transport = new TestClientTransportFactory(); + var policy = new RecordingRetryPolicy(); + await using var client = CreateRetryClient(transport, policy, maxAttempts: 2); + await client.ConnectAsync(); + + var invocation = ClientInvokerTestHelper.InvokeIdempotentUnaryAsync(client).AsTask(); + var first = await transport.Connection.WaitForSentPacket(ProtocolV2FrameType.Request); + await InjectErrorAsync(transport, first, SharpLinkErrorCode.Unavailable); + var second = await transport.Connection.WaitForSentPacket(ProtocolV2FrameType.Request); + await transport.Connection.InjectPacketAsync( + ProtocolV2FrameType.Response, ProtocolV2FrameFlags.None, unchecked((long)second.RequestId)); + + Ensure(await invocation == 0, "second attempt result"); + Ensure(policy.Count == 1, "policy invocation count"); + Ensure(policy.LastContext.Attempt == 1, "first completed attempt"); + Ensure(policy.LastContext.ErrorCode == SharpLinkErrorCode.Unavailable, "remote unavailable code"); + Ensure(policy.LastContext.ResponseObserved, "remote error is an observed response"); + } + + [Test] + public async Task NonIdempotentUnaryAndResourceExhaustedShouldNotRetry() + { + var transport = new TestClientTransportFactory(); + await using var client = CreateRetryClient(transport, policy: null, maxAttempts: 3); + await client.ConnectAsync(); + + var nonIdempotent = ClientInvokerTestHelper.InvokeUnaryAsync(client).AsTask(); + var first = await transport.Connection.WaitForSentPacket(ProtocolV2FrameType.Request); + await InjectErrorAsync(transport, first, SharpLinkErrorCode.Unavailable); + var nonIdempotentError = await EnsureThrows(nonIdempotent); + Ensure(nonIdempotentError.Code == SharpLinkErrorCode.Unavailable, "non-idempotent result"); + Ensure(!await transport.Connection.TryWaitForSentPacket( + ProtocolV2FrameType.Request, TimeSpan.FromMilliseconds(100)), "non-idempotent no second request"); + + var idempotent = ClientInvokerTestHelper.InvokeIdempotentUnaryAsync(client).AsTask(); + var resourceExhausted = await transport.Connection.WaitForSentPacket(ProtocolV2FrameType.Request); + await InjectErrorAsync(transport, resourceExhausted, SharpLinkErrorCode.ResourceExhausted); + var resourceError = await EnsureThrows(idempotent); + Ensure(resourceError.Code == SharpLinkErrorCode.ResourceExhausted, "resource exhausted result"); + Ensure(!await transport.Connection.TryWaitForSentPacket( + ProtocolV2FrameType.Request, TimeSpan.FromMilliseconds(100)), "resource exhausted no second request"); + } + + [Test] + public async Task RetryShouldHonorAbsoluteDeadlineAndCancellationDuringDelay() + { + var deadlineTransport = new TestClientTransportFactory(); + await using var deadlineClient = CreateRetryClient( + deadlineTransport, policy: null, maxAttempts: 2, initialBackoff: TimeSpan.FromMilliseconds(50)); + await deadlineClient.ConnectAsync(); + + var deadlineInvocation = ClientInvokerTestHelper.InvokeIdempotentUnaryAsync( + deadlineClient, new SharpLinkCallOptions { Timeout = TimeSpan.FromMilliseconds(20) }).AsTask(); + var deadlineRequest = await deadlineTransport.Connection.WaitForSentPacket(ProtocolV2FrameType.Request); + await InjectErrorAsync(deadlineTransport, deadlineRequest, SharpLinkErrorCode.Unavailable); + var deadlineError = await EnsureThrows(deadlineInvocation); + Ensure(deadlineError.Code == SharpLinkErrorCode.DeadlineExceeded, "retry delay deadline result"); + Ensure(!await deadlineTransport.Connection.TryWaitForSentPacket( + ProtocolV2FrameType.Request, TimeSpan.FromMilliseconds(100)), "deadline no second request"); + + var cancellationTransport = new TestClientTransportFactory(); + await using var cancellationClient = CreateRetryClient( + cancellationTransport, policy: null, maxAttempts: 2, initialBackoff: TimeSpan.FromSeconds(1)); + await cancellationClient.ConnectAsync(); + using var cancellation = new CancellationTokenSource(); + var cancellationInvocation = ClientInvokerTestHelper.InvokeIdempotentUnaryAsync( + cancellationClient, cancellationToken: cancellation.Token).AsTask(); + var cancellationRequest = await cancellationTransport.Connection.WaitForSentPacket(ProtocolV2FrameType.Request); + await InjectErrorAsync(cancellationTransport, cancellationRequest, SharpLinkErrorCode.Unavailable); + cancellation.Cancel(); + await EnsureThrows(cancellationInvocation); + Ensure(!await cancellationTransport.Connection.TryWaitForSentPacket( + ProtocolV2FrameType.Request, TimeSpan.FromMilliseconds(100)), "cancel no second request"); + } + + [Test] + public async Task ClientStopShouldCancelCustomRetryBackoffPromptly() + { + var transport = new TestClientTransportFactory(); + var policy = new DelayingRetryPolicy(TimeSpan.MaxValue); + await using var client = CreateRetryClient(transport, policy, maxAttempts: 2); + await client.ConnectAsync(); + + var invocation = ClientInvokerTestHelper.InvokeIdempotentUnaryAsync(client).AsTask(); + var request = await transport.Connection.WaitForSentPacket(ProtocolV2FrameType.Request); + await InjectErrorAsync(transport, request, SharpLinkErrorCode.Unavailable); + await policy.EvaluationStarted.WaitAsync(TimeSpan.FromSeconds(2)); + + var stoppedAt = Stopwatch.GetTimestamp(); + var stop = client.StopAsync().AsTask(); + var exception = await EnsureThrows( + invocation.WaitAsync(TimeSpan.FromSeconds(2))); + await stop.WaitAsync(TimeSpan.FromSeconds(2)); + + Ensure(exception.Code == SharpLinkErrorCode.ConnectionClosed, "stopped retry backoff error code"); + Ensure(Stopwatch.GetElapsedTime(stoppedAt) < TimeSpan.FromSeconds(1), + "client stop must cancel the custom retry backoff promptly"); + } + + [Test] + public async Task HugeBuiltInJitteredRetryDelayShouldRemainCancellable() + { + for (var iteration = 0; iteration < 32; iteration++) + { + var transport = new TestClientTransportFactory(); + await using var client = new SharpLinkClient( + transport, + TimeSpan.FromSeconds(10), + TimeSpan.FromSeconds(30), + retryOptions: new SharpLinkRetryOptions + { + MaxAttempts = 2, + InitialBackoff = TimeSpan.MaxValue, + MaxBackoff = TimeSpan.MaxValue, + JitterRatio = 1 + }); + await client.ConnectAsync(); + + var invocation = ClientInvokerTestHelper.InvokeIdempotentUnaryAsync(client).AsTask(); + var request = await transport.Connection.WaitForSentPacket(ProtocolV2FrameType.Request); + await InjectErrorAsync(transport, request, SharpLinkErrorCode.Unavailable); + await Task.Delay(20); + + var stop = client.StopAsync().AsTask(); + var exception = await EnsureThrows( + invocation.WaitAsync(TimeSpan.FromSeconds(2))); + await stop.WaitAsync(TimeSpan.FromSeconds(2)); + Ensure(exception.Code == SharpLinkErrorCode.ConnectionClosed, + $"huge jittered retry delay cancellation iteration {iteration}"); + } + } + + [Test] + public async Task RetryDelayBeyondDeadlineShouldNotOverflow() + { + var transport = new TestClientTransportFactory(); + var policy = new HugeDelayPolicy(); + await using var client = CreateRetryClient(transport, policy, maxAttempts: 2); + await client.ConnectAsync(); + + var invocation = ClientInvokerTestHelper.InvokeIdempotentUnaryAsync( + client, + new SharpLinkCallOptions { Timeout = TimeSpan.FromSeconds(1) }).AsTask(); + var request = await transport.Connection.WaitForSentPacket(ProtocolV2FrameType.Request); + await InjectErrorAsync(transport, request, SharpLinkErrorCode.Unavailable); + + var exception = await EnsureThrows(invocation); + Ensure(exception.Code == SharpLinkErrorCode.DeadlineExceeded, + "oversized retry delay must map to deadline exceeded instead of overflowing"); + Ensure(policy.Count == 1, "custom retry policy should be evaluated once"); + } + + [Test] + public async Task RetryShouldRunInterceptorOnceAndRejectInvalidCustomPolicyDelay() + { + var transport = new TestClientTransportFactory(); + var interceptor = new CountingInterceptor(); + var invalidPolicy = new NegativeDelayPolicy(); + await using var client = new SharpLinkClient( + transport, + TimeSpan.FromSeconds(10), + TimeSpan.FromSeconds(30), + clientInterceptors: [interceptor], + retryOptions: RetryOptions(2, TimeSpan.Zero), + retryPolicy: invalidPolicy); + await client.ConnectAsync(); + + var invocation = ClientInvokerTestHelper.InvokeIdempotentUnaryAsync(client).AsTask(); + var request = await transport.Connection.WaitForSentPacket(ProtocolV2FrameType.Request); + await InjectErrorAsync(transport, request, SharpLinkErrorCode.Unavailable); + var error = await EnsureThrows(invocation); + + Ensure(error.Code == SharpLinkErrorCode.FailedPrecondition, "negative delay rejected"); + Ensure(interceptor.Count == 1, "interceptor runs once for logical call"); + Ensure(invalidPolicy.Count == 1, "custom policy receives failed attempt"); + } + + [Test] + public async Task RetryShouldExcludeTriedEndpointsThenResetAfterAllCandidates() + { + var first = new TestClientTransportFactory(); + var second = new TestClientTransportFactory(); + var endpoints = new[] + { + new StaticEndpointConfiguration(Endpoint("first", 5001), first), + new StaticEndpointConfiguration(Endpoint("second", 5002), second) + }; + await using var client = new SharpLinkClient( + first, + TimeSpan.FromSeconds(10), + TimeSpan.FromSeconds(30), + staticEndpoints: endpoints, + clusterOptions: new SharpLinkClusterOptions(), + endpointSelector: new FirstAvailableSelector(), + retryOptions: RetryOptions(3, TimeSpan.Zero)); + await client.ConnectAsync(); + await WaitForReadyConnectionCountAsync(client, 2); + + var invocation = ClientInvokerTestHelper.InvokeIdempotentUnaryAsync(client).AsTask(); + var firstAttempt = await first.Connection.WaitForSentPacket(ProtocolV2FrameType.Request); + await InjectErrorAsync(first, firstAttempt, SharpLinkErrorCode.Unavailable); + var secondAttempt = await second.Connection.WaitForSentPacket(ProtocolV2FrameType.Request); + await InjectErrorAsync(second, secondAttempt, SharpLinkErrorCode.Unavailable); + var thirdAttempt = await first.Connection.WaitForSentPacket(ProtocolV2FrameType.Request); + await first.Connection.InjectPacketAsync( + ProtocolV2FrameType.Response, ProtocolV2FrameFlags.None, unchecked((long)thirdAttempt.RequestId)); + + Ensure(await invocation == 0, "candidate reset third attempt result"); + } + + [Test] + public async Task EndpointAdmissionShouldRejectOneCandidateAndReportTheSelectedAttemptOnce() + { + var first = new TestClientTransportFactory(); + var second = new TestClientTransportFactory(); + var policy = new RejectFirstEndpointPolicy(); + var endpoints = new[] + { + new StaticEndpointConfiguration(Endpoint("first", 5001), first), + new StaticEndpointConfiguration(Endpoint("second", 5002), second) + }; + await using var client = new SharpLinkClient( + first, + TimeSpan.FromSeconds(10), + TimeSpan.FromSeconds(30), + staticEndpoints: endpoints, + clusterOptions: new SharpLinkClusterOptions(), + endpointSelector: new FirstAvailableSelector(), + endpointAdmissionPolicy: policy); + await client.ConnectAsync(); + await WaitForReadyConnectionCountAsync(client, 2); + + var invocation = ClientInvokerTestHelper.InvokeUnaryAsync(client).AsTask(); + var request = await second.Connection.WaitForSentPacket(ProtocolV2FrameType.Request); + await second.Connection.InjectPacketAsync( + ProtocolV2FrameType.Response, ProtocolV2FrameFlags.None, unchecked((long)request.RequestId)); + + Ensure(await invocation == 0, "admitted endpoint response"); + Ensure(policy.AcquireCount == 2, "both candidates evaluated"); + Ensure(policy.ReportCount == 1, $"only selected endpoint reported: {policy.ReportCount}"); + Ensure(policy.LastOutcome.Endpoint.Endpoint.Id == "second", "selected endpoint report identity"); + Ensure(policy.LastOutcome.Kind == SharpLinkEndpointOutcomeKind.Success, "selected endpoint report outcome"); + Ensure(!await first.Connection.TryWaitForSentPacket( + ProtocolV2FrameType.Request, TimeSpan.FromMilliseconds(100)), "rejected endpoint has no request"); + } + + [Test] + public async Task RetryShouldHonorAdmissionRetryAfterBeforeTheNextAttempt() + { + var transport = new TestClientTransportFactory(); + var admission = new RejectOnceWithRetryAfterPolicy(TimeSpan.FromMilliseconds(100)); + await using var client = new SharpLinkClient( + transport, + TimeSpan.FromSeconds(10), + TimeSpan.FromSeconds(30), + fixedEndpoint: Endpoint("retry", 5001), + retryOptions: RetryOptions(2, TimeSpan.Zero), + endpointAdmissionPolicy: admission); + await client.ConnectAsync(); + + var started = Stopwatch.GetTimestamp(); + var invocation = ClientInvokerTestHelper.InvokeIdempotentUnaryAsync(client).AsTask(); + var request = await transport.Connection.WaitForSentPacket(ProtocolV2FrameType.Request); + Ensure(Stopwatch.GetElapsedTime(started) >= TimeSpan.FromMilliseconds(75), + "retry must wait for the admission retry delay rather than consume the next attempt immediately"); + await transport.Connection.InjectPacketAsync( + ProtocolV2FrameType.Response, ProtocolV2FrameFlags.None, unchecked((long)request.RequestId)); + + Ensure(await invocation == 0, "admitted retry result"); + Ensure(admission.AcquireCount == 2, "admission should be retried once after its requested delay"); + Ensure(admission.ReportCount == 1, "only the admitted retry should report"); + } + + [Test] + public async Task ClientStopShouldCancelRetryAdmissionDelayPromptly() + { + var transport = new TestClientTransportFactory(); + var admission = new SignaledRejectWithRetryAfterPolicy(TimeSpan.MaxValue); + await using var client = new SharpLinkClient( + transport, + TimeSpan.FromSeconds(10), + TimeSpan.FromSeconds(30), + fixedEndpoint: Endpoint("retry-admission", 5001), + retryOptions: RetryOptions(2, TimeSpan.Zero), + endpointAdmissionPolicy: admission); + await client.ConnectAsync(); + + var invocation = ClientInvokerTestHelper.InvokeIdempotentUnaryAsync( + client, new SharpLinkCallOptions { WaitForReady = true }).AsTask(); + await admission.RejectionStarted.WaitAsync(TimeSpan.FromSeconds(2)); + + var stoppedAt = Stopwatch.GetTimestamp(); + var stop = client.StopAsync().AsTask(); + var exception = await EnsureThrows( + invocation.WaitAsync(TimeSpan.FromSeconds(2))); + await stop.WaitAsync(TimeSpan.FromSeconds(2)); + + Ensure(exception.Code == SharpLinkErrorCode.ConnectionClosed, "stopped retry admission error code"); + Ensure(Stopwatch.GetElapsedTime(stoppedAt) < TimeSpan.FromSeconds(1), + "client stop must cancel the retry admission delay promptly"); + } + + [Test] + public async Task RetryShouldNotDelayUntriedEndpointsAfterAnAdmittedAttemptFails() + { + var first = new TestClientTransportFactory(); + var second = new TestClientTransportFactory(); + var third = new TestClientTransportFactory(); + var endpoints = new[] + { + new StaticEndpointConfiguration(Endpoint("first", 5001), first), + new StaticEndpointConfiguration(Endpoint("second", 5002), second), + new StaticEndpointConfiguration(Endpoint("third", 5003), third) + }; + await using var client = new SharpLinkClient( + first, + TimeSpan.FromSeconds(10), + TimeSpan.FromSeconds(30), + staticEndpoints: endpoints, + clusterOptions: new SharpLinkClusterOptions { MinReadyEndpoints = 3, MaxConnections = 3 }, + endpointSelector: new FirstUnexcludedSelector(), + retryOptions: RetryOptions(2, TimeSpan.Zero), + endpointAdmissionPolicy: new RejectFirstEndpointWithDelayPolicy(TimeSpan.FromSeconds(30))); + await client.ConnectAsync(); + await WaitForReadyConnectionCountAsync(client, 3); + + var invocation = ClientInvokerTestHelper.InvokeIdempotentUnaryAsync(client).AsTask(); + var secondRequest = await second.Connection.WaitForSentPacket(ProtocolV2FrameType.Request); + await InjectErrorAsync(second, secondRequest, SharpLinkErrorCode.Unavailable); + var thirdRequest = await third.Connection.WaitForSentPacket(ProtocolV2FrameType.Request) + .WaitAsync(TimeSpan.FromSeconds(5)); + await third.Connection.InjectPacketAsync( + ProtocolV2FrameType.Response, ProtocolV2FrameFlags.None, unchecked((long)thirdRequest.RequestId)); + + Ensure(await invocation == 0, "untried endpoint retry response"); + } + + [Test] + public void CircuitBreakerShouldRejectZeroFailureRatio() + { + try + { + _ = new SharpLinkCircuitBreakerOptions { FailureRatio = 0 }.CloneValidated(); + throw new Exception("zero failure ratio should be rejected"); + } + catch (ArgumentOutOfRangeException) + { + } + } + + [Test] + public void CircuitBreakerShouldOpenPerGenerationForInfrastructureFailures() + { + var breaker = new SharpLinkCircuitBreaker(new SharpLinkCircuitBreakerOptions + { + MinimumThroughput = 2, + FailureRatio = 0.5, + SamplingDuration = TimeSpan.FromSeconds(10), + BreakDuration = TimeSpan.FromSeconds(10), + HalfOpenMaxCalls = 1 + }.CloneValidated()); + var method = new RpcMethodDescriptor(1, 2, RpcMethodKind.Unary, true, false, false, null); + var generationOne = new SharpLinkEndpointCandidate(Endpoint("breaker", 5001), 1, 0, generation: 1); + var failure = new SharpLinkEndpointOutcome( + generationOne, + method, + SharpLinkEndpointOutcomeKind.RemoteError, + SharpLinkErrorCode.Unavailable, + ResponseObserved: true, + TimeSpan.Zero); + + var first = breaker.TryAcquire(generationOne, method); + Ensure(first.IsAllowed, "closed breaker first acquisition"); + breaker.Report(failure, first.Token); + var second = breaker.TryAcquire(generationOne, method); + Ensure(second.IsAllowed, "closed breaker second acquisition"); + breaker.Report(failure, second.Token); + + var open = breaker.TryAcquire(generationOne, method); + Ensure(!open.IsAllowed && open.RetryAfter > TimeSpan.Zero, "breaker opens after failure ratio threshold"); + var replacementGeneration = new SharpLinkEndpointCandidate(Endpoint("breaker", 5002), 1, 0, generation: 2); + Ensure(breaker.TryAcquire(replacementGeneration, method).IsAllowed, "replacement generation starts closed"); + } + + [Test] + public void CircuitBreakerShouldIgnoreLocalResourceExhaustionDuringHalfOpenProbe() + { + var breaker = new SharpLinkCircuitBreaker(new SharpLinkCircuitBreakerOptions + { + MinimumThroughput = 1, + FailureRatio = 1, + SamplingDuration = TimeSpan.FromSeconds(10), + BreakDuration = TimeSpan.FromMilliseconds(1), + HalfOpenMaxCalls = 1 + }.CloneValidated()); + var method = new RpcMethodDescriptor(1, 2, RpcMethodKind.Unary, true, false, false, null); + var endpoint = new SharpLinkEndpointCandidate(Endpoint("breaker", 5001), 1, 0, generation: 1); + var infrastructureFailure = new SharpLinkEndpointOutcome( + endpoint, method, SharpLinkEndpointOutcomeKind.RemoteError, SharpLinkErrorCode.Unavailable, true, TimeSpan.Zero); + var localCapacityFailure = new SharpLinkEndpointOutcome( + endpoint, method, SharpLinkEndpointOutcomeKind.SendFailure, SharpLinkErrorCode.ResourceExhausted, false, TimeSpan.Zero); + + breaker.Report(infrastructureFailure, breaker.TryAcquire(endpoint, method).Token); + Thread.Sleep(20); + var probe = breaker.TryAcquire(endpoint, method); + Ensure(probe.IsAllowed && probe.Token != 0, "half-open probe should be admitted"); + breaker.Report(localCapacityFailure, probe.Token); + + var nextProbe = breaker.TryAcquire(endpoint, method); + Ensure(nextProbe.IsAllowed && nextProbe.Token != 0, + "local capacity pressure must not close the breaker as a successful probe"); + } + + [Test] + public void CircuitBreakerShouldIgnoreLocalSendFailures() + { + var breaker = new SharpLinkCircuitBreaker(new SharpLinkCircuitBreakerOptions + { + MinimumThroughput = 1, + FailureRatio = 1, + SamplingDuration = TimeSpan.FromSeconds(10), + BreakDuration = TimeSpan.FromSeconds(10), + HalfOpenMaxCalls = 1 + }.CloneValidated()); + var method = new RpcMethodDescriptor(1, 2, RpcMethodKind.Unary, true, false, false, null); + var endpoint = new SharpLinkEndpointCandidate(Endpoint("breaker", 5001), 1, 0, generation: 1); + var codecFailure = new SharpLinkEndpointOutcome( + endpoint, method, SharpLinkEndpointOutcomeKind.SendFailure, null, false, TimeSpan.Zero); + var validationFailure = new SharpLinkEndpointOutcome( + endpoint, method, SharpLinkEndpointOutcomeKind.SendFailure, SharpLinkErrorCode.InvalidArgument, false, TimeSpan.Zero); + + breaker.Report(codecFailure, breaker.TryAcquire(endpoint, method).Token); + breaker.Report(validationFailure, breaker.TryAcquire(endpoint, method).Token); + + Ensure(breaker.TryAcquire(endpoint, method).IsAllowed, + "local serialization and validation failures must not open the endpoint breaker"); + } + + [Test] + public void CircuitBreakerShouldIgnoreReportsFromAnExpiredHalfOpenEpoch() + { + var breaker = new SharpLinkCircuitBreaker(new SharpLinkCircuitBreakerOptions + { + MinimumThroughput = 1, + FailureRatio = 1, + SamplingDuration = TimeSpan.FromSeconds(10), + BreakDuration = TimeSpan.FromMilliseconds(1), + HalfOpenMaxCalls = 2 + }.CloneValidated()); + var method = new RpcMethodDescriptor(1, 2, RpcMethodKind.Unary, true, false, false, null); + var endpoint = new SharpLinkEndpointCandidate(Endpoint("breaker", 5001), 1, 0, generation: 1); + var failure = new SharpLinkEndpointOutcome( + endpoint, method, SharpLinkEndpointOutcomeKind.RemoteError, SharpLinkErrorCode.Unavailable, true, TimeSpan.Zero); + var success = new SharpLinkEndpointOutcome( + endpoint, method, SharpLinkEndpointOutcomeKind.Success, null, true, TimeSpan.Zero); + + breaker.Report(failure, breaker.TryAcquire(endpoint, method).Token); + Thread.Sleep(20); + var firstEpochFirstProbe = breaker.TryAcquire(endpoint, method); + var firstEpochSecondProbe = breaker.TryAcquire(endpoint, method); + Ensure(firstEpochFirstProbe.IsAllowed && firstEpochFirstProbe.Token != 0, "first half-open probe token"); + Ensure(firstEpochSecondProbe.IsAllowed && firstEpochSecondProbe.Token == firstEpochFirstProbe.Token, + "same half-open epoch token"); + + breaker.Report(failure, firstEpochFirstProbe.Token); + Thread.Sleep(20); + var currentEpochProbe = breaker.TryAcquire(endpoint, method); + Ensure(currentEpochProbe.IsAllowed && currentEpochProbe.Token != 0, "current half-open probe token"); + + breaker.Report(success, firstEpochSecondProbe.Token); + var stillHalfOpen = breaker.TryAcquire(endpoint, method); + Ensure(stillHalfOpen.IsAllowed && stillHalfOpen.Token != 0, + "stale success must not close a newer half-open epoch"); + } + + private static SharpLinkClient CreateRetryClient( + TestClientTransportFactory transport, + ISharpLinkRetryPolicy? policy, + int maxAttempts, + TimeSpan? initialBackoff = null) + => new( + transport, + TimeSpan.FromSeconds(10), + TimeSpan.FromSeconds(30), + retryOptions: RetryOptions(maxAttempts, initialBackoff ?? TimeSpan.Zero), + retryPolicy: policy); + + private static SharpLinkRetryOptions RetryOptions(int maxAttempts, TimeSpan initialBackoff) + => new() + { + MaxAttempts = maxAttempts, + InitialBackoff = initialBackoff, + MaxBackoff = initialBackoff, + JitterRatio = 0 + }; + + private static SharpLinkEndpoint Endpoint(string id, int port) + => new() + { + Id = id, + Address = new SharpLinkTcpAddress("127.0.0.1", port) + }; + + private static Task InjectErrorAsync( + TestClientTransportFactory transport, + ProtocolV2FrameHeader request, + SharpLinkErrorCode code) + { + var payload = new PooledByteBufferWriter(); + ProtocolV2PayloadCodec.WriteError(payload, code, code.ToString(), 1024, out _); + return transport.Connection.InjectFrameAsync( + ProtocolV2FrameType.Response, + ProtocolV2FrameFlags.Error, + request.RequestId, + payload.WrittenMemory); + } + + private static async Task EnsureThrows(Task invocation) + where TException : Exception + { + try + { + await invocation; + throw new Exception($"expected {typeof(TException).Name}"); + } + catch (TException exception) + { + return exception; + } + } + + private static async Task WaitForReadyConnectionCountAsync(SharpLinkClient client, int expected) + { + var deadline = DateTime.UtcNow + TimeSpan.FromSeconds(5); + while (client.ReadyConnectionCount < expected && DateTime.UtcNow < deadline) + await Task.Delay(10); + Ensure(client.ReadyConnectionCount >= expected, $"expected {expected} ready connections"); + } + + private static void Ensure(bool condition, string message) + { + if (!condition) + throw new Exception(message); + } + + private sealed class RecordingRetryPolicy : ISharpLinkRetryPolicy + { + public int Count { get; private set; } + public SharpLinkRetryContext LastContext { get; private set; } + + public SharpLinkRetryDecision Evaluate(in SharpLinkRetryContext context) + { + Count++; + LastContext = context; + return new SharpLinkRetryDecision(true, TimeSpan.Zero); + } + } + + private sealed class DelayingRetryPolicy(TimeSpan delay) : ISharpLinkRetryPolicy + { + private readonly TaskCompletionSource _evaluationStarted = + new(TaskCreationOptions.RunContinuationsAsynchronously); + + public Task EvaluationStarted => _evaluationStarted.Task; + + public SharpLinkRetryDecision Evaluate(in SharpLinkRetryContext context) + { + _evaluationStarted.TrySetResult(); + return new SharpLinkRetryDecision(true, delay); + } + } + + private sealed class NegativeDelayPolicy : ISharpLinkRetryPolicy + { + public int Count { get; private set; } + + public SharpLinkRetryDecision Evaluate(in SharpLinkRetryContext context) + { + Count++; + return new SharpLinkRetryDecision(true, TimeSpan.FromMilliseconds(-1)); + } + } + + private sealed class CountingInterceptor : ISharpLinkClientInterceptor + { + public int Count { get; private set; } + + public async ValueTask InvokeAsync( + SharpLinkClientInvocationContext context, + SharpLinkClientInvocationDelegate next) + { + Count++; + return await next(context); + } + } + + private sealed class HugeDelayPolicy : ISharpLinkRetryPolicy + { + public int Count { get; private set; } + + public SharpLinkRetryDecision Evaluate(in SharpLinkRetryContext context) + { + Count++; + return new SharpLinkRetryDecision(true, TimeSpan.MaxValue); + } + } + + private sealed class FirstAvailableSelector : ISharpLinkEndpointSelector + { + public int Select(in SharpLinkEndpointSelectionContext context) + => (context.ExcludedMask & 1UL) == 0 ? 0 : 1; + } + + private sealed class FirstUnexcludedSelector : ISharpLinkEndpointSelector + { + public int Select(in SharpLinkEndpointSelectionContext context) + { + for (var index = 0; index < context.Count; index++) + if ((context.ExcludedMask & (1UL << index)) == 0) + return index; + return -1; + } + } + + private sealed class RejectFirstEndpointPolicy : ISharpLinkEndpointAdmissionPolicy + { + public int AcquireCount { get; private set; } + public int ReportCount { get; private set; } + public SharpLinkEndpointOutcome LastOutcome { get; private set; } + + public SharpLinkEndpointAdmissionDecision TryAcquire( + in SharpLinkEndpointCandidate endpoint, + in RpcMethodDescriptor method) + { + AcquireCount++; + return endpoint.Endpoint.Id == "first" + ? new SharpLinkEndpointAdmissionDecision(false, Token: 0, RetryAfter: null) + : new SharpLinkEndpointAdmissionDecision(true, Token: 7, RetryAfter: null); + } + + public void Report(in SharpLinkEndpointOutcome outcome, long token) + { + ReportCount++; + LastOutcome = outcome; + Ensure(token == 7, "admission token preserved"); + } + } + + private sealed class RejectOnceWithRetryAfterPolicy(TimeSpan retryAfter) : ISharpLinkEndpointAdmissionPolicy + { + public int AcquireCount { get; private set; } + public int ReportCount { get; private set; } + + public SharpLinkEndpointAdmissionDecision TryAcquire( + in SharpLinkEndpointCandidate endpoint, + in RpcMethodDescriptor method) + { + AcquireCount++; + return AcquireCount == 1 + ? new SharpLinkEndpointAdmissionDecision(false, Token: 0, RetryAfter: retryAfter) + : new SharpLinkEndpointAdmissionDecision(true, Token: 1, RetryAfter: null); + } + + public void Report(in SharpLinkEndpointOutcome outcome, long token) + { + ReportCount++; + Ensure(token == 1, "admitted retry token"); + } + } + + private sealed class SignaledRejectWithRetryAfterPolicy(TimeSpan retryAfter) : ISharpLinkEndpointAdmissionPolicy + { + private readonly TaskCompletionSource _rejectionStarted = + new(TaskCreationOptions.RunContinuationsAsynchronously); + + public Task RejectionStarted => _rejectionStarted.Task; + + public SharpLinkEndpointAdmissionDecision TryAcquire( + in SharpLinkEndpointCandidate endpoint, + in RpcMethodDescriptor method) + { + _rejectionStarted.TrySetResult(); + return new SharpLinkEndpointAdmissionDecision(false, Token: 0, RetryAfter: retryAfter); + } + + public void Report(in SharpLinkEndpointOutcome outcome, long token) + { + } + } + + private sealed class RejectFirstEndpointWithDelayPolicy(TimeSpan retryAfter) : ISharpLinkEndpointAdmissionPolicy + { + public SharpLinkEndpointAdmissionDecision TryAcquire( + in SharpLinkEndpointCandidate endpoint, + in RpcMethodDescriptor method) + => endpoint.Endpoint.Id == "first" + ? new SharpLinkEndpointAdmissionDecision(false, Token: 0, RetryAfter: retryAfter) + : new SharpLinkEndpointAdmissionDecision(true, Token: 1, RetryAfter: null); + + public void Report(in SharpLinkEndpointOutcome outcome, long token) + { + } + } +} diff --git a/test/SharpLink.UnitTests/Client/StaticEndpointBuilderTests.cs b/test/SharpLink.UnitTests/Client/StaticEndpointBuilderTests.cs new file mode 100644 index 0000000..fd3c4ed --- /dev/null +++ b/test/SharpLink.UnitTests/Client/StaticEndpointBuilderTests.cs @@ -0,0 +1,367 @@ +using System.Collections.Generic; +using System.Linq; +using System.Reflection; +using System.Threading; +using SharpLink.Client; + +namespace SharpLink.UnitTests.Client; + +public class StaticEndpointBuilderTests +{ + [Test] + public async Task AddressValidationAndAnonymousPipeRedactionShouldBeStable() + { + await EnsureThrows(() => + { + _ = new SharpLinkTcpAddress("localhost", 0); + return Task.CompletedTask; + }); + var address = new SharpLinkAnonymousPipeAddress("in-secret", "out-secret"); + Ensure(!address.ToString().Contains("secret", StringComparison.Ordinal), "anonymous handles must not be rendered"); + } + + [Test] + public async Task SingleEndpointShouldFreezeAttributesAndDisposeItsFactoryOnce() + { + var attributes = new Dictionary { ["zone"] = "a" }; + SharpLinkEndpoint? received = null; + var factory = new TrackingFactory(); + var client = SharpClientBuilder.Create() + .UseEndpoint( + new SharpLinkEndpoint + { + Id = "one", + Address = new SharpLinkTcpAddress("127.0.0.1", 5001), + Attributes = attributes + }, + endpoint => + { + received = endpoint; + return factory; + }) + .Build(); + + attributes["zone"] = "changed"; + Ensure(received is not null && received.Attributes["zone"] == "a", "endpoint attributes must be frozen"); + var endpointField = client.GetType().GetField("_fixedEndpoint", BindingFlags.Instance | BindingFlags.NonPublic); + Ensure((endpointField?.GetValue(client) as SharpLinkEndpoint)?.Id == "one", "fixed mode must retain endpoint identity"); + await client.DisposeAsync(); + Ensure(factory.DisposeCount == 1, "single endpoint factory disposal count"); + } + + [Test] + public async Task SingleEndpointFactoryShouldBeReleasedWhenLaterBuildValidationFails() + { + var factory = new TrackingFactory(); + await EnsureThrows(() => + { + _ = SharpClientBuilder.Create() + .UseEndpoint(Endpoint("one", 5001), _ => factory) + .UseConnectionPool(static options => options.MaxConnections = 0) + .Build(); + return Task.CompletedTask; + }); + Ensure(factory.DisposeCount == 1, "factory disposal after a fixed-client build failure"); + } + + [Test] + public async Task SingleEndpointAnonymousPipeFactoryShouldRejectExpandedConnectionPools() + { + await EnsureThrows(() => + { + _ = SharpClientBuilder.Create() + .UseEndpoint( + new SharpLinkEndpoint + { + Id = "pipe", + Address = new SharpLinkAnonymousPipeAddress("in-handle", "out-handle") + }, + _ => new AnonymousPipeClientTransportFactory("in-handle", "out-handle")) + .UseConnectionPool(options => options.MaxConnections = 2) + .Build(); + return Task.CompletedTask; + }); + } + + [Test] + public async Task StaticClusterShouldRejectAnonymousPipeFactories() + { + await EnsureThrows(() => + { + _ = SharpClientBuilder.Create() + .UseEndpoints( + [Endpoint("first", 5001), Endpoint("second", 5002)], + _ => new AnonymousPipeClientTransportFactory("in-handle", "out-handle")) + .Build(); + return Task.CompletedTask; + }); + } + + [Test] + public async Task EndpointFactoryShouldBeDisposedWhenProfileBindingFails() + { + var factory = new ProfileBindingFailureFactory(); + await EnsureThrows(() => + { + _ = SharpClientBuilder.Create() + .UseEndpoints([Endpoint("one", 5001), Endpoint("two", 5002)], _ => factory) + .Build(); + return Task.CompletedTask; + }); + Ensure(factory.DisposeCount == 1, "profile binding failure must release the newly created factory"); + } + + [Test] + public async Task StaticClusterShouldOwnEveryFactoryExactlyOnce() + { + var first = new TrackingFactory(); + var second = new TrackingFactory(); + var client = SharpClientBuilder.Create() + .UseEndpoints( + [Endpoint("first", 5001), Endpoint("second", 5002)], + endpoint => endpoint.Id == "first" ? first : second) + .Build(); + + await client.DisposeAsync(); + Ensure(first.DisposeCount == 1, "first cluster factory disposal count"); + Ensure(second.DisposeCount == 1, "second cluster factory disposal count"); + } + + [Test] + public async Task ClusterBuildCleanupShouldReleaseEveryFactoryWhenOneDisposalFails() + { + var throwing = new TrackingFactory(throwOnDispose: true); + var remaining = new TrackingFactory(); + await EnsureThrows(() => + { + _ = SharpClientBuilder.Create() + .UseEndpoints( + [Endpoint("first", 5001), Endpoint("second", 5002), Endpoint("duplicate", 5003)], + endpoint => endpoint.Id switch + { + "first" => throwing, + "second" => remaining, + _ => remaining + }) + .Build(); + return Task.CompletedTask; + }); + + Ensure(throwing.DisposeCount == 1, "throwing factory must be attempted once"); + Ensure(remaining.DisposeCount == 1, "later factory cleanup must run after an earlier disposal failure"); + } + + [Test] + public async Task ClusterStopShouldReachStoppedWhenFactoryCleanupFails() + { + var throwing = new TrackingFactory(throwOnDispose: true); + var remaining = new TrackingFactory(); + var client = SharpClientBuilder.Create() + .UseEndpoints( + [Endpoint("first", 5001), Endpoint("second", 5002)], + endpoint => endpoint.Id == "first" ? throwing : remaining) + .Build(); + + await EnsureThrows(() => client.StopAsync().AsTask()); + + Ensure(((SharpLinkClient)client).State == SharpLinkConnectionState.Stopped, + "outer client cleanup must reach Stopped after cluster cleanup fails"); + Ensure(remaining.DisposeCount == 1, "cluster cleanup must still dispose later factories"); + } + + [Test] + public async Task BuilderShouldRejectConflictingModesAndOptions() + { + await EnsureThrows(() => + { + _ = SharpClientBuilder.Create() + .UseTransport(new TrackingFactory()) + .UseEndpoints([Endpoint("first", 5001), Endpoint("second", 5002)], _ => new TrackingFactory()) + .Build(); + return Task.CompletedTask; + }); + + await EnsureThrows(() => + { + _ = SharpClientBuilder.Create() + .UseEndpoints([Endpoint("first", 5001), Endpoint("second", 5002)], _ => new TrackingFactory()) + .UseConnectionPool(static options => options.MaxConnections = 2) + .Build(); + return Task.CompletedTask; + }); + + await EnsureThrows(() => + { + _ = SharpClientBuilder.Create() + .UseLoadBalancing(SharpLinkLoadBalancingStrategy.Random) + .UseEndpointSelector(new FirstSelector()); + return Task.CompletedTask; + }); + } + + [Test] + public async Task BuilderShouldValidateEndpointIdsAndClusterBounds() + { + await EnsureThrows(() => + { + _ = SharpClientBuilder.Create() + .UseEndpoints([Endpoint("duplicate", 5001), Endpoint("duplicate", 5002)], _ => new TrackingFactory()) + .Build(); + return Task.CompletedTask; + }); + + await EnsureThrows(() => + { + _ = SharpClientBuilder.Create() + .UseEndpoints([Endpoint("one", 5001), Endpoint("two", 5002)], _ => new TrackingFactory()) + .UseCluster(static options => + { + options.MinReadyEndpoints = 5; + options.MaxConnections = 1; + options.MaxConnectionsPerEndpoint = 1; + }) + .Build(); + return Task.CompletedTask; + }); + + await EnsureThrows(() => + { + _ = SharpClientBuilder.Create() + .UseEndpoints([], _ => new TrackingFactory()) + .Build(); + return Task.CompletedTask; + }); + + await EnsureThrows(() => + { + _ = SharpClientBuilder.Create() + .UseEndpoints( + Enumerable.Range(0, SharpLinkClusterOptions.MaximumEndpoints + 1) + .Select(index => Endpoint($"endpoint-{index}", 5001 + index)), + _ => new TrackingFactory()) + .Build(); + return Task.CompletedTask; + }); + + await EnsureThrows(() => + { + _ = SharpClientBuilder.Create() + .UseEndpoints( + [new SharpLinkEndpoint + { + Id = "one", + Address = new SharpLinkTcpAddress("127.0.0.1", 5001), + Attributes = new Dictionary { [" "] = "invalid" } + }], + _ => new TrackingFactory()) + .Build(); + return Task.CompletedTask; + }); + + await EnsureThrows(() => + { + _ = SharpClientBuilder.Create() + .UseEndpoints([Endpoint("one", 5001), Endpoint("two", 5002)], _ => new TrackingFactory()) + .UseCluster(static options => options.MaxRetiringConnections = -1) + .Build(); + return Task.CompletedTask; + }); + } + + [Test] + public async Task ClusterMinReadyShouldUseTheEndpointCountAsItsEffectiveUpperBound() + { + var first = new TrackingFactory(); + var second = new TrackingFactory(); + await using var client = SharpClientBuilder.Create() + .UseEndpoints( + [Endpoint("one", 5001), Endpoint("two", 5002)], + endpoint => endpoint.Id == "one" ? first : second) + .UseCluster(options => + { + options.MinReadyEndpoints = 5; + options.MaxConnections = 4; + options.MaxConnectionsPerEndpoint = 2; + }) + .Build(); + + Ensure(first.DisposeCount == 0 && second.DisposeCount == 0, "factories remain client-owned until stop"); + } + + [Test] + public async Task ClusterShouldRejectAFactoryInstanceSharedAcrossEndpoints() + { + var shared = new TrackingFactory(); + await EnsureThrows(() => + { + _ = SharpClientBuilder.Create() + .UseEndpoints([Endpoint("one", 5001), Endpoint("two", 5002)], _ => shared) + .Build(); + return Task.CompletedTask; + }); + Ensure(shared.DisposeCount == 1, "rejected shared factory must be disposed exactly once"); + } + + private static SharpLinkEndpoint Endpoint(string id, int port) => new() + { + Id = id, + Address = new SharpLinkTcpAddress("127.0.0.1", port) + }; + + private static async Task EnsureThrows(Func action) where TException : Exception + { + try + { + await action(); + throw new Exception($"expected {typeof(TException).Name}"); + } + catch (TException) + { + } + } + + private static void Ensure(bool condition, string message) + { + if (!condition) + throw new Exception(message); + } + + private sealed class TrackingFactory(bool throwOnDispose = false) : IClientTransportFactory + { + private int _disposeCount; + public int DisposeCount => Volatile.Read(ref _disposeCount); + public ValueTask ConnectAsync(CancellationToken cancellationToken = default) + => ValueTask.FromException(new NotSupportedException()); + public ValueTask DisposeAsync() + { + Interlocked.Increment(ref _disposeCount); + if (throwOnDispose) + return ValueTask.FromException(new InvalidOperationException("test disposal failure")); + return ValueTask.CompletedTask; + } + } + + private sealed class ProfileBindingFailureFactory : IClientTransportFactory, IPerformanceProfileAwareTransport + { + private int _disposeCount; + + public int DisposeCount => Volatile.Read(ref _disposeCount); + + public ValueTask ConnectAsync(CancellationToken cancellationToken = default) + => ValueTask.FromException(new NotSupportedException()); + + public void BindPerformanceProfile(SharpLinkPerformanceProfile profile) + => throw new InvalidOperationException("test profile binding failure"); + + public ValueTask DisposeAsync() + { + Interlocked.Increment(ref _disposeCount); + return ValueTask.CompletedTask; + } + } + + private sealed class FirstSelector : ISharpLinkEndpointSelector + { + public int Select(in SharpLinkEndpointSelectionContext context) => 0; + } +} diff --git a/test/SharpLink.UnitTests/Client/StaticEndpointSelectionTests.cs b/test/SharpLink.UnitTests/Client/StaticEndpointSelectionTests.cs new file mode 100644 index 0000000..fc186e5 --- /dev/null +++ b/test/SharpLink.UnitTests/Client/StaticEndpointSelectionTests.cs @@ -0,0 +1,53 @@ +using System.Collections.Concurrent; +using System.Linq; +using SharpLink.Client; + +namespace SharpLink.UnitTests.Client; + +public sealed class StaticEndpointSelectionTests +{ + [Test] + public void PowerOfTwoComparisonShouldUseExactCrossMultiplication() + { + Ensure( + StaticEndpointSelection.CompareNormalizedLoad(3, 2, 4, 3) > 0, + "3/2 should be greater than 4/3"); + Ensure( + StaticEndpointSelection.CompareNormalizedLoad(4, 2, 6, 3) == 0, + "equal normalized loads"); + Ensure( + StaticEndpointSelection.CompareNormalizedLoad( + int.MaxValue, + int.MaxValue, + int.MaxValue - 1, + int.MaxValue) > 0, + "large operands must not overflow the 64-bit cross product"); + } + + [Test] + public void RandomSelectionShouldOnlyReturnNonExcludedIndexes() + { + const ulong excluded = (1UL << 1) | (1UL << 3); + Ensure(StaticEndpointSelection.SelectRandomIndex(5, excluded, 3, 0) == 0, "first available index"); + Ensure(StaticEndpointSelection.SelectRandomIndex(5, excluded, 3, 1) == 2, "middle available index"); + Ensure(StaticEndpointSelection.SelectRandomIndex(5, excluded, 3, 2) == 4, "last available index"); + Ensure(StaticEndpointSelection.SelectRandomIndex(5, excluded, 3, 3) == -1, "out-of-range random target"); + } + + [Test] + public void RoundRobinCursorShouldRemainBalancedUnderConcurrency() + { + var cursor = -1; + var selections = new ConcurrentBag(); + Parallel.For(0, 400, _ => selections.Add(StaticEndpointSelection.SelectRoundRobinIndex(ref cursor, 4, 0))); + + for (var index = 0; index < 4; index++) + Ensure(selections.Count(selection => selection == index) == 100, "round-robin concurrent balance"); + } + + private static void Ensure(bool condition, string message) + { + if (!condition) + throw new Exception(message); + } +} diff --git a/test/SharpLink.UnitTests/Runtime/RequestManagerTests.cs b/test/SharpLink.UnitTests/Runtime/RequestManagerTests.cs index eb24367..ba714d3 100644 --- a/test/SharpLink.UnitTests/Runtime/RequestManagerTests.cs +++ b/test/SharpLink.UnitTests/Runtime/RequestManagerTests.cs @@ -214,6 +214,41 @@ public async Task CompletionRaceShouldHaveExactlyOneWinnerAndReleaseOneSlot() } } + [Test] + public async Task StreamingResponseObservationShouldPrecedeTerminalCompletion() + { + using var manager = new PendingRequestTable(8); + using var observer = new BlockingStreamingCompletionObserver(); + var requestId = manager.RegisterStream( + PendingCallKind.ServerStreaming, + new NoopStreamDispatcher(), + deadlineTimestamp: 0, + CancellationToken.None, + observer); + + var response = Task.Run(() => + { + var payload = ReadOnlySequence.Empty; + return manager.Dispatch(requestId, ref payload); + }); + await observer.ResponseObservationEntered.WaitAsync(TimeSpan.FromSeconds(2)); + + var terminal = Task.Run(() => manager.TryComplete( + requestId, + PendingCallCompletionReason.ConnectionClosed, + new SharpLinkException(SharpLinkErrorCode.ConnectionClosed, "test disconnect"))); + await Task.Delay(50); + Ensure(!observer.TerminalCompletionObserved.IsCompleted, + "terminal completion must wait until the matched streaming response is observed"); + + observer.ReleaseResponseObservation(); + Ensure(await response, "streaming response acknowledgement"); + Ensure(await terminal, "terminal completion"); + await observer.TerminalCompletionObserved.WaitAsync(TimeSpan.FromSeconds(2)); + Ensure(observer.TerminalSawResponseObservation, + "terminal observer must see the prior response acknowledgement"); + } + [Test] public async Task MonotonicDeadlineScanShouldCompleteWithoutCompletionPathRemoval() { @@ -397,4 +432,50 @@ public void Dispose() AllowRegistration.Dispose(); } } + + private sealed class NoopStreamDispatcher : IStreamDispatcher + { + public ValueTask DispatchAsync(ReadOnlySequence payload) => ValueTask.CompletedTask; + + public void Complete(bool isError, string? errorMessage) + { + } + + public void Complete(Exception? exception) + { + } + } + + private sealed class BlockingStreamingCompletionObserver : IPendingCallCompletionObserver, IDisposable + { + private readonly TaskCompletionSource _responseObservationEntered = + new(TaskCreationOptions.RunContinuationsAsynchronously); + private readonly TaskCompletionSource _responseObservationRelease = + new(TaskCreationOptions.RunContinuationsAsynchronously); + private readonly TaskCompletionSource _terminalCompletionObserved = + new(TaskCreationOptions.RunContinuationsAsynchronously); + private int _responseObserved; + private int _terminalSawResponse; + + public Task ResponseObservationEntered => _responseObservationEntered.Task; + public Task TerminalCompletionObserved => _terminalCompletionObserved.Task; + public bool TerminalSawResponseObservation => Volatile.Read(ref _terminalSawResponse) != 0; + + public void OnResponseObserved() + { + Volatile.Write(ref _responseObserved, 1); + _responseObservationEntered.TrySetResult(); + _responseObservationRelease.Task.GetAwaiter().GetResult(); + } + + public void OnPendingCallCompleted(in PendingCallCompletion completion) + { + Volatile.Write(ref _terminalSawResponse, Volatile.Read(ref _responseObserved)); + _terminalCompletionObserved.TrySetResult(); + } + + public void ReleaseResponseObservation() => _responseObservationRelease.TrySetResult(); + + public void Dispose() => _responseObservationRelease.TrySetResult(); + } }