From eada685191fbdfb42a83065b50f79027da6f9bd3 Mon Sep 17 00:00:00 2001 From: pavlos Date: Mon, 28 Sep 2026 12:19:18 +0300 Subject: [PATCH 01/20] feat(storage): define outbound stage contracts Task 1.1 - Define stage contracts, ownership transitions, and the composition boundary. Add single-destination lifecycle interfaces, typed work identities, and embedding contracts. Document stage-consistent recovery and verify public contract boundaries. Refs #338 Signed-off-by: pavlos --- docs/14-outbound-storage.md | 207 ++++++++++++++++++ docs/DocumentationMap.md | 2 + .../core/outbound/OutboundCoordinator.java | 85 +++++++ .../sendium/core/outbound/OutboundWork.java | 115 ++++++++++ .../sendium/core/storage/OutboundStage.java | 42 ++++ .../storage/OutboundStorageException.java | 31 +++ .../core/storage/PendingMessageStore.java | 22 ++ .../sendium/core/storage/RoutedWorkStore.java | 64 ++++++ .../core/storage/SelectedRouterStore.java | 50 +++++ .../core/outbound/OutboundWorkTest.java | 80 +++++++ .../consumer/OutboundCompositionTest.java | 53 +++++ 11 files changed, 751 insertions(+) create mode 100644 docs/14-outbound-storage.md create mode 100644 sendium-core/src/main/java/gr/cytech/sendium/core/outbound/OutboundCoordinator.java create mode 100644 sendium-core/src/main/java/gr/cytech/sendium/core/outbound/OutboundWork.java create mode 100644 sendium-core/src/main/java/gr/cytech/sendium/core/storage/OutboundStage.java create mode 100644 sendium-core/src/main/java/gr/cytech/sendium/core/storage/OutboundStorageException.java create mode 100644 sendium-core/src/main/java/gr/cytech/sendium/core/storage/PendingMessageStore.java create mode 100644 sendium-core/src/main/java/gr/cytech/sendium/core/storage/RoutedWorkStore.java create mode 100644 sendium-core/src/main/java/gr/cytech/sendium/core/storage/SelectedRouterStore.java create mode 100644 sendium-core/src/test/java/gr/cytech/sendium/core/outbound/OutboundWorkTest.java create mode 100644 sendium-core/src/test/java/gr/cytech/sendium/core/outbound/consumer/OutboundCompositionTest.java diff --git a/docs/14-outbound-storage.md b/docs/14-outbound-storage.md new file mode 100644 index 00000000..3a709927 --- /dev/null +++ b/docs/14-outbound-storage.md @@ -0,0 +1,207 @@ +# Outbound storage contracts + +This describes the initial contracts for [#338](https://github.com/cytechmobile/sendium/issues/338), +under [#337](https://github.com/cytechmobile/sendium/issues/337). At this step these are library APIs: +the coordinator, memory stage implementations, configuration selector, and production wiring are +subsequent tasks. Existing ingress and queue behavior has not changed. + +## Ownership model + +```text +accepted pending source + | bounded, eligible, priority-aware select-and-stage + v +selected router work ---- take/release ---- router execution + | record one destination assignment, then mark routed + v +destination work ------- take/release ---- worker execution + | | forward: transfer to one successor + | all provider parts terminal AND successful required handoffs + v +current destination work terminal + | complete pending sources + | complete selected work + v +forget routed terminal bookkeeping +``` + +Pending sources remain authoritative throughout. Taking work does not delete source or stage +ownership. Source IDs, selection IDs, and destination work IDs are distinct opaque UUID-backed types; +none is inferred from mutable message equality or a protocol serial. One selected item may retain +several source IDs, allowing later multipart integration without treating an aggregate as a new +independent admission. Detailed multipart admission and reconstruction are owned by #343. + +There is at most one active destination work item per selection. Forwarding transfers that ownership +to a successor with a new work ID under the same selection; it does not create simultaneous destinations. + +## Copied routing boundary + +The existing `+vendor`/`copied` routing feature and its public API remain available on the existing +routing path. The new storage abstraction does not support copied routing. Its routing integration +must reject a copied-route request as `UNSUPPORTED` before assignment or dispatch, even if the +lookup happens to produce only one destination. It must not silently choose one result, strip the +copy flag, or emulate fan-out using several assignments or forwarding calls. + +This guard belongs to the upcoming routing integration task; this contract-only step does not change +existing routing behavior. The single-destination scope is the owner's revision to the copied-route +requirements previously described in #337/#338. Those issue bodies have not been edited here. + +## Components + +The public APIs are grouped by responsibility under `gr.cytech.sendium.core`: + +```text +outbound/ + OutboundCoordinator.java + OutboundWork.java +storage/ + OutboundStage.java + PendingMessageStore.java + SelectedRouterStore.java + RoutedWorkStore.java + OutboundStorageException.java +``` + +Memory implementations will live under `storage.memory` when introduced. No backend implementation +package is needed for this contract-only step. + +| Contract | Responsibility | +|---|---| +| `PendingMessageStore` | Record accepted source state under a stable caller-supplied identity and complete sources idempotently. | +| `SelectedRouterStore` | Own bounded select-and-stage, exclusive runtime takes, return-to-router, and retained selected state. | +| `RoutedWorkStore` | Record one destination, own its scheduling, transfer responsibility on forwarding, and report terminal source ownership. | +| `OutboundCoordinator` | Application-facing admission, transitions, required handoff, source completion, and lifecycle coordination. | +| `OutboundWork` | Source/selection/work identities and typed execution, assignment, and completion values. | +| `OutboundStage` | Explicit open/close, backend identity, durability declaration, and lifecycle availability. | +| `OutboundStorageException` | Identify the failing stage and distinguish unavailable, capacity, ownership, unsupported, and invalid-transition failures. | + +The selected-router implementation receives its source access and selection/preparation collaborators +through construction. It can use Sendium's pending implementation or application-owned queries and +transactions. The shared API does not expose a SQL query language or require the application to +materialize its entire backlog. A selected source is not eligible for another concurrent selection. +Stage implementations enforce state transitions; malformed arguments fail immediately, while invalid +ownership/state transitions use the categorized storage failure. + +`OutboundWork.Assignment` carries exactly one `Destination`, and `Routed` identifies its execution +with a `WorkId`. The work values freeze source identity sets, not message objects. Worker and filter +mutations are intentional execution state. They continue through transitions, returns, and forwarding; +they must not silently change a retained earlier-stage record through a shared mutable reference. + +The contract requires consistent ownership and stored payload/stage, not a public copying service or +a copy on every operation. A backend may use independent snapshots, encoding/mapping, or explicit +ownership transitions to achieve that consistency. The concrete mechanism will be chosen with the +memory implementation. Custom message subtypes and their fields must be preserved; flattening a +subclass to `StandardMessage` is not acceptable. Codec/schema design comes later. + +## Recovery follows recorded stage ownership + +After reconciling an interrupted transition by stable identity, recovery resumes from the latest +safely recorded stage for each source: + +| Surviving state | Resume behavior | +|---|---| +| Destination and execution payload recorded | Restore work at that destination's worker queue. Do not rerun routing or routing filters. | +| Selected work, without recorded destination work | Resume routing from the selected state. | +| Accepted pending source only | Repeat selection and routing. | + +Earlier retained records must not independently schedule the same work. Keeping a pending record +through completion does not mean routing it alongside its recorded worker-stage message. + +A stored payload must match the processing stage that will resume it. For example, silently replacing +a pre-routing payload with its routing-filter output while retaining the pre-routing stage could apply +the transformation twice. Correctly recorded routed work instead resumes with its saved destination +and execution payload, without running that routing filter again. + +Backends may retain separate accepted/execution representations or explicitly advance payload and +stage together, provided they retain the recoverable source information required by the configured +profile and can reconcile interrupted transitions. This does not require permanently immutable +copies of every intermediate message. Work performed after the last recorded boundary may repeat; +provider-outcome checkpoints and exactly-once submission remain outside the guarantee. + +These are requirements for the later durable implementations, not a restart guarantee for memory. + +## Completion and retry boundaries + +- Admission retries reuse the same source ID and do not overwrite the accepted state while the + source remains pending. A new ID is a new admission; protocol-level deduplication and deduplication + after terminal removal are not implied. +- Returning taken work includes its updated execution projection, preserving current-process filter + and retry mutations without silently changing earlier-stage state. The take's identities and + destination cannot change on return. This does not promise durable retry counters or timing. +- `record` publishes a single assignment before its work can be taken. Retrying the same selection + returns its original work ID without reactivating completed/transferred work. Conflicting source + identities or destinations fail as `INVALID_TRANSITION`, rather than creating another assignment. +- Marking selected work routed stops router scheduling but retains the selected record until terminal + source cleanup. This leaves room for a future durable router with memory-backed routed work. +- `transfer` atomically moves responsibility to one successor before that successor can be taken. + Retrying with the previous ID returns the original successor; a conflicting successor destination + fails as `INVALID_TRANSITION`. Completion of the previous work is ignored, even after its successor + completes. A late previous-worker callback must never complete the successor. +- Provider multipart is aggregated by provider-processing integration: all parts must have terminal + outcomes and all required handoffs must succeed before completing the one destination work item. + Part submissions are not represented as multiple routed destinations. +- `complete(work, requiredHandoff)` is called after all provider parts have terminal outcomes. The supplied + `CompletionStage` represents successful acceptance by the required DLR/tracking subsystem, + not eventual delivery of the downstream receipt. Use an already-successful stage when no handoff + is required. For multipart, it covers all required part handoffs. Failure/cancellation retains ownership; + a later retry supplies another handoff attempt. +- Source cleanup occurs only when the active destination work satisfies that rule. Retry failed source cleanup without + dispatching terminal provider work again in the active process. The routed store retains a repeatable + completion result until cleanup succeeds and it can be forgotten. The coordinator must suppress + duplicate terminal callbacks, including callbacks arriving after store bookkeeping was forgotten. +- A terminal filter drop before routing uses `discard` with the same handoff requirement. A routing + miss uses return-to-router; it is not a terminal drop. Assignments without a destination are invalid. + +These contracts do not assume atomic transactions across independently selected stores. Physical +backends and the coordinator must reconcile interrupted transitions by stable identity. Refinement +of durable batch/transition semantics belongs to #344. No durable provider-outcome checkpoint or +exactly-once provider submission is promised. + +## Embedding and lifecycle + +An application assembles compatible `PendingMessageStore`, `SelectedRouterStore`, and +`RoutedWorkStore` instances and passes them directly as three constructor dependencies to its +coordinator implementation. There is no dependency-bundle type or public copier contract. The +coordinator remains an interface at this step; the concrete constructor and wiring arrive with its +implementation. Constructors must have no activation side effects. Standalone CDI assembly is a +separate integration task; these APIs require no CDI annotations or container. + +For example, mCore's `Message extends StandardMessage`, billing-aware batch preparation, +asynchronous committed tracking, and own lifecycle orchestration can be adapted without importing +its domain classes into Sendium. An application's tracker returning after an in-memory enqueue is +not a committed handoff; its completion signal must correspond to the promised storage acceptance. +mCore adapters and adoption are owned by its team, not #338. + +Graceful shutdown is two-phase: `quiesce()` stops new admission, selection, and takes; the application +stops and joins execution, resolves or drains in-flight provider operations, and returns unfinished +work; `close()` then closes owned stages. Completion and return operations remain available during +quiescence. Unrouted work returns to the router; already-routed work retains its destination. Runtime +worker removal is a separate policy, not an alias for application shutdown. Closure is idempotent, +cannot mark unfinished work terminal, and does not imply that memory survives a restart. + +## Planned standalone profile support + +The following selectors will be introduced by the standalone wiring task: + +```properties +sendium.sms.pending.backend=memory +sendium.sms.router-queue.backend=memory +sendium.sms.routed-work.backend=memory +``` + +| Pending / router / routed | Availability | Restart behavior | +|---|---|---| +| `memory/memory/memory` | Target of #338; not yet wired at this contract-only step | Non-durable. | +| `file/memory/memory` | #339 | Recover accepted work; reselect and reroute. | +| `file/file/memory` | #345 | Recover selected work; reroute. | +| `file/memory/file` | #345 | Reselect not-yet-routed work; restore recorded destinations. | +| `file/file/file` | #345 | Restore selected work and recorded destinations. | +| PostgreSQL profiles | #333 | Only explicitly implemented combinations will be supported. | + +Standalone validation will accept only the implemented profile and reject unsupported requests before +admission, without falling back to memory. A custom backend identity in the library does not register +a supported standalone profile. DLR storage remains independent of outbound stage selection. +Future durable guarantees require surviving storage and remain at least once. + +This milestone includes actionable errors and startup profile/non-durable logging. Metrics, +readiness endpoints, and broader observability are deferred. Contract-level status is not a health endpoint. diff --git a/docs/DocumentationMap.md b/docs/DocumentationMap.md index 7e089502..59788409 100644 --- a/docs/DocumentationMap.md +++ b/docs/DocumentationMap.md @@ -22,6 +22,7 @@ Sendium is an open-source, headless SMS gateway for high-throughput messaging. I | Understand releases and publishing | [11. Release Process](11-release-process.md) | | Review current features and roadmap | [12. Features And Roadmap](12-features-roadmap.md) | | Configure PostgreSQL DLR persistence and review durability | [13. DLR Persistence](13-dlr-persistence.md) | +| Review outbound storage extension contracts | [14. Outbound Storage Contracts](14-outbound-storage.md) | | Contribute code or docs | [Contributing](../.github/CONTRIBUTING.md) | ## Core Concepts @@ -63,6 +64,7 @@ Sendium expects these files in the configured `conf` directory. The Docker quick | [11. Release Process](11-release-process.md) | Release Please flow, Conventional Commit rules, release PR handling, GitHub Packages, and Docker publishing. | | [12. Features And Roadmap](12-features-roadmap.md) | Current product capabilities, planned roadmap phases, and related feature documentation. | | [13. DLR Persistence](13-dlr-persistence.md) | PostgreSQL setup, retention, restart guarantees, and durability limits. | +| [14. Outbound Storage Contracts](14-outbound-storage.md) | Initial stage APIs, ownership transitions, embedding boundaries, and planned standalone profiles. | | [Kannel migration converter](https://cytechmobile.github.io/sendium/kannelconverter/) | Browser-only helper for turning a legacy `kannel.conf` into Sendium starter files. | ## API Discovery diff --git a/sendium-core/src/main/java/gr/cytech/sendium/core/outbound/OutboundCoordinator.java b/sendium-core/src/main/java/gr/cytech/sendium/core/outbound/OutboundCoordinator.java new file mode 100644 index 00000000..037372d9 --- /dev/null +++ b/sendium-core/src/main/java/gr/cytech/sendium/core/outbound/OutboundCoordinator.java @@ -0,0 +1,85 @@ +package gr.cytech.sendium.core.outbound; + +import gr.cytech.sendium.core.message.StandardMessage; +import gr.cytech.sendium.core.outbound.OutboundWork.Destination; +import gr.cytech.sendium.core.outbound.OutboundWork.Routed; +import gr.cytech.sendium.core.outbound.OutboundWork.Selected; +import gr.cytech.sendium.core.outbound.OutboundWork.SelectionId; +import gr.cytech.sendium.core.outbound.OutboundWork.SourceId; +import gr.cytech.sendium.core.outbound.OutboundWork.WorkId; + +import java.time.Duration; +import java.util.Optional; +import java.util.concurrent.CompletionStage; + +/** + * Application-facing lifecycle boundary. Implementations receive pending, selected-router, and + * routed-work stores as three direct constructor dependencies. Embedders own construction and + * activation; this contract requires no CDI container. Store methods are implementation ports, + * not an alternative way for workers to bypass coordinated completion. + */ +public interface OutboundCoordinator extends AutoCloseable { + /** + * Opens selected stores before allowing admission; failed startup must clean up opened resources. + * Future durable implementations reconcile by stable identity and resume the latest safely recorded + * stage. Recorded routed work resumes at its worker without rerouting or repeating routing filters; + * earlier retained records must not independently schedule the same work. + */ + void start(); + + /** + * Stops new admission, selection, and takes. Existing routing, handoff, completion, and return + * operations remain valid while the application stops and joins execution. Idempotent. + */ + void quiesce(); + + void admit(SourceId source, M message); + + int selectAndStage(int limit); + + Optional> takeForRouting(Duration timeout) throws InterruptedException; + + void returnToRouter(Selected selected); + + /** + * Records one destination before marking selected work routed; dispatch uses takeForDestination. + * The selection and its source set must match work owned by this coordinator. Routing integration + * must reject copied-route requests as UNSUPPORTED before assignment or dispatch, even if lookup + * yields only one destination. It must not drop copies or submit several assignments for one selection. + */ + Routed route(Selected selected, Destination destination); + + Optional> takeForDestination(String destination, Duration timeout) throws InterruptedException; + + void returnToDestination(Routed work); + + /** Transfers responsibility to one successor without treating the previous work as terminal delivery. */ + Routed forward(WorkId previous, Destination destination); + + /** + * Reports terminal processing of all provider parts for this work, conditional on successful required + * handoff for all parts. The provider-processing integration aggregates part outcomes and handoffs. + * Pass an already successful stage when no handoff is required. Failure or cancellation retains work + * and source ownership; retrying completion must not redispatch provider work. The returned stage + * completes after work completion and pending/selected cleanup. Duplicate callbacks are harmless, + * including after cleanup. A late callback for transferred work is a successful no-op, not a signal + * that its successor or source has completed. + */ + CompletionStage complete(WorkId work, CompletionStage requiredHandoff); + + /** + * Terminal filter rejection/drop before a destination exists, with the same handoff rule as complete. + * A routing miss is not terminal and must instead return to the router. Routed selections cannot + * be discarded through this operation. + */ + CompletionStage discard(SelectionId selection, CompletionStage requiredHandoff); + + /** + * After quiesce and application-owned stop/join, returns unfinished work to its owning stage and + * closes stores. In-flight provider operations must first be resolved or drained by the application; + * closing may not classify them as terminal. Closure is idempotent. Memory closure promises no + * restart recovery, and a closed coordinator cannot be restarted. + */ + @Override + void close(); +} diff --git a/sendium-core/src/main/java/gr/cytech/sendium/core/outbound/OutboundWork.java b/sendium-core/src/main/java/gr/cytech/sendium/core/outbound/OutboundWork.java new file mode 100644 index 00000000..3b8d2ff4 --- /dev/null +++ b/sendium-core/src/main/java/gr/cytech/sendium/core/outbound/OutboundWork.java @@ -0,0 +1,115 @@ +package gr.cytech.sendium.core.outbound; + +import gr.cytech.sendium.core.message.StandardMessage; + +import java.util.Objects; +import java.util.Set; +import java.util.UUID; + +/** + * Storage identities are independent of protocol message IDs. Message-bearing values contain + * mutable execution projections. Mutations are intentional but must not silently change an + * earlier-stage record; implementations keep recorded payload and stage consistent. + */ +public final class OutboundWork { + private OutboundWork() { + } + + private static Set copySources(Set sources) { + Set copy = Set.copyOf(sources); + if (copy.isEmpty()) { + throw new IllegalArgumentException("Work must retain at least one source"); + } + return copy; + } + + private static String requireDestination(String destination) { + Objects.requireNonNull(destination, "destination"); + if (destination.isBlank()) { + throw new IllegalArgumentException("Destination must not be blank"); + } + return destination; + } + + public record SourceId(UUID value) { + public SourceId { + Objects.requireNonNull(value, "value"); + } + } + + public record SelectionId(UUID value) { + public SelectionId { + Objects.requireNonNull(value, "value"); + } + } + + public record WorkId(UUID value) { + public WorkId { + Objects.requireNonNull(value, "value"); + } + } + + public record Selected(SelectionId id, Set sources, M message) { + public Selected { + Objects.requireNonNull(id, "id"); + sources = copySources(sources); + Objects.requireNonNull(message, "message"); + } + + @Override + public String toString() { + return "Selected[sourceCount=" + sources.size() + "]"; + } + } + + public record Destination(String name, M message) { + public Destination { + name = requireDestination(name); + Objects.requireNonNull(message, "message"); + } + + @Override + public String toString() { + return "Destination[message=redacted]"; + } + } + + /** + * A single-destination routing decision. Stores must record its payload and destination consistently + * before acknowledging the assignment. Copied routes are not representable by this contract. + */ + public record Assignment(SelectionId selection, Set sources, + Destination destination) { + public Assignment { + Objects.requireNonNull(selection, "selection"); + sources = copySources(sources); + Objects.requireNonNull(destination, "destination"); + } + + @Override + public String toString() { + return "Assignment[sourceCount=" + sources.size() + "]"; + } + } + + public record Routed(WorkId id, SelectionId selection, String destination, M message) { + public Routed { + Objects.requireNonNull(id, "id"); + Objects.requireNonNull(selection, "selection"); + destination = requireDestination(destination); + Objects.requireNonNull(message, "message"); + } + + @Override + public String toString() { + return "Routed[message=redacted]"; + } + } + + public record Completed(SelectionId selection, Set sources) { + public Completed { + Objects.requireNonNull(selection, "selection"); + sources = copySources(sources); + } + } +} diff --git a/sendium-core/src/main/java/gr/cytech/sendium/core/storage/OutboundStage.java b/sendium-core/src/main/java/gr/cytech/sendium/core/storage/OutboundStage.java new file mode 100644 index 00000000..7d01d9fb --- /dev/null +++ b/sendium-core/src/main/java/gr/cytech/sendium/core/storage/OutboundStage.java @@ -0,0 +1,42 @@ +package gr.cytech.sendium.core.storage; + +import java.util.Objects; + +/** + * An explicitly owned, single-use storage lifecycle. Construction does not start a stage. + * Operations require READY; closing is idempotent and does not imply terminal completion. + * Implementations must support concurrent operation calls while open. + * + *

Worker/filter mutations belong to execution state and are preserved through explicit transitions + * and returns. They must not silently change an earlier-stage record through a shared mutable object. + * Recorded payload and processing stage must stay consistent. Snapshot/mapping/ownership mechanisms + * are implementation choices, not a requirement to copy on every operation. Custom message subtypes + * and their fields must be preserved. + */ +public interface OutboundStage extends AutoCloseable { + void open(); + + Status status(); + + @Override + void close(); + + enum Role { + PENDING, ROUTER_QUEUE, ROUTED_WORK + } + + enum State { + NEW, READY, STOPPING, CLOSED, FAILED + } + + /** Backend names are implementation identities, not the list of supported standalone profiles. */ + record Status(String backend, boolean durable, State state) { + public Status { + Objects.requireNonNull(backend, "backend"); + Objects.requireNonNull(state, "state"); + if (backend.isBlank()) { + throw new IllegalArgumentException("Backend identity must not be blank"); + } + } + } +} diff --git a/sendium-core/src/main/java/gr/cytech/sendium/core/storage/OutboundStorageException.java b/sendium-core/src/main/java/gr/cytech/sendium/core/storage/OutboundStorageException.java new file mode 100644 index 00000000..c18c803b --- /dev/null +++ b/sendium-core/src/main/java/gr/cytech/sendium/core/storage/OutboundStorageException.java @@ -0,0 +1,31 @@ +package gr.cytech.sendium.core.storage; + +import java.util.Objects; + +/** Operation failures must describe the boundary without including message content or credentials. */ +public class OutboundStorageException extends RuntimeException { + private final OutboundStage.Role stage; + private final Reason reason; + + public OutboundStorageException(OutboundStage.Role stage, Reason reason, String message) { + this(stage, reason, message, null); + } + + public OutboundStorageException(OutboundStage.Role stage, Reason reason, String message, Throwable cause) { + super(Objects.requireNonNull(message, "message"), cause); + this.stage = Objects.requireNonNull(stage, "stage"); + this.reason = Objects.requireNonNull(reason, "reason"); + } + + public OutboundStage.Role stage() { + return stage; + } + + public Reason reason() { + return reason; + } + + public enum Reason { + UNAVAILABLE, CAPACITY_EXCEEDED, OWNERSHIP_CONFLICT, UNSUPPORTED, INVALID_TRANSITION + } +} diff --git a/sendium-core/src/main/java/gr/cytech/sendium/core/storage/PendingMessageStore.java b/sendium-core/src/main/java/gr/cytech/sendium/core/storage/PendingMessageStore.java new file mode 100644 index 00000000..383bb641 --- /dev/null +++ b/sendium-core/src/main/java/gr/cytech/sendium/core/storage/PendingMessageStore.java @@ -0,0 +1,22 @@ +package gr.cytech.sendium.core.storage; + +import gr.cytech.sendium.core.message.StandardMessage; +import gr.cytech.sendium.core.outbound.OutboundWork.SourceId; + +import java.util.Set; + +public interface PendingMessageStore extends OutboundStage { + /** + * Records accepted source state before protocol success. Later execution mutations must not + * silently change that state. Repeating the same source ID while it remains pending is idempotent + * and never overwrites its accepted content. A failure must + * not be acknowledged. Deduplication after terminal source removal is not promised. + */ + void admit(SourceId source, M message); + + /** + * Called only after all work and required handoffs are terminal. Missing sources are already + * complete. Partial multi-source completion is allowed; retrying the full set must be safe. + */ + void complete(Set sources); +} diff --git a/sendium-core/src/main/java/gr/cytech/sendium/core/storage/RoutedWorkStore.java b/sendium-core/src/main/java/gr/cytech/sendium/core/storage/RoutedWorkStore.java new file mode 100644 index 00000000..cd16e4a7 --- /dev/null +++ b/sendium-core/src/main/java/gr/cytech/sendium/core/storage/RoutedWorkStore.java @@ -0,0 +1,64 @@ +package gr.cytech.sendium.core.storage; + +import gr.cytech.sendium.core.message.StandardMessage; +import gr.cytech.sendium.core.outbound.OutboundWork.Assignment; +import gr.cytech.sendium.core.outbound.OutboundWork.Completed; +import gr.cytech.sendium.core.outbound.OutboundWork.Destination; +import gr.cytech.sendium.core.outbound.OutboundWork.Routed; +import gr.cytech.sendium.core.outbound.OutboundWork.SelectionId; +import gr.cytech.sendium.core.outbound.OutboundWork.WorkId; + +import java.time.Duration; +import java.util.Optional; + +public interface RoutedWorkStore extends OutboundStage { + /** + * Records the execution payload and one destination consistently before its work may be taken. + * Future recovery resumes this recorded work at its worker, not through routing again. + * Repeating a selection returns its original assignment and work ID without reactivating + * completed/transferred work or overwriting content. + * Callers retry with the same assignment; changing an active destination requires transfer, not + * another assignment. There is at most one active destination work item per selection. + * Conflicting source identities or destinations for an existing selection fail as INVALID_TRANSITION. + */ + Routed record(Assignment assignment); + + /** + * Takes one work projection without completing it or silently changing earlier-stage records. + * Zero timeout polls; negative timeouts are invalid. Destination must be nonblank. + * A work item has at most one active taker. + */ + Optional> take(String destination, Duration timeout) throws InterruptedException; + + /** + * Retains the recorded destination and updated execution projection when retrying or draining + * in-flight work. Identity, selection, and destination must match the take. Repeated release cannot + * duplicate work or reactivate completed/transferred work. Earlier-stage state cannot silently change; + * preserving runtime mutations does not promise durable retry counters or timing. + */ + void release(Routed work); + + /** + * Atomically transfers active responsibility to exactly one successor with a new work ID under + * the same selection. Retrying with the same previous ID returns the original successor without + * reactivating it. The previous item is no longer schedulable and its late callbacks cannot complete + * the successor. Transfer must not transiently complete the source or leave two active destinations. + * A retry naming a different successor destination fails as INVALID_TRANSITION. + */ + Routed transfer(WorkId previous, Destination destination); + + /** + * Called only after all provider parts and required handoffs for the active work are terminal. + * Provider-part aggregation belongs to the provider-processing boundary, not separate routed items. + * Returns the completed selection for source cleanup. A transferred previous item returns empty, + * even after its successor completes. Until forget, repeating completion of the terminal work + * returns the same result so cleanup can be retried without sending again. Unknown IDs fail explicitly. + */ + Optional complete(WorkId work); + + /** + * Discards terminal bookkeeping only after pending and selected completion succeed. Nonterminal + * selections cannot be forgotten; repeating forget for an absent selection is harmless. + */ + void forget(SelectionId selection); +} diff --git a/sendium-core/src/main/java/gr/cytech/sendium/core/storage/SelectedRouterStore.java b/sendium-core/src/main/java/gr/cytech/sendium/core/storage/SelectedRouterStore.java new file mode 100644 index 00000000..e6cc627a --- /dev/null +++ b/sendium-core/src/main/java/gr/cytech/sendium/core/storage/SelectedRouterStore.java @@ -0,0 +1,50 @@ +package gr.cytech.sendium.core.storage; + +import gr.cytech.sendium.core.message.StandardMessage; +import gr.cytech.sendium.core.outbound.OutboundWork.Selected; +import gr.cytech.sendium.core.outbound.OutboundWork.SelectionId; + +import java.time.Duration; +import java.util.Optional; + +/** + * The implementation receives its pending source and selection/preparation policy through composition. + * Selection is not constrained to scanning Sendium-owned records or exposing SQL through this API. + */ +public interface SelectedRouterStore extends OutboundStage { + /** + * Selects and stages at most limit eligible, priority-aware items, returning the newly staged count. + * Limit must be positive. Selected, taken, and routed sources are ineligible for another selection. + * Source ownership is retained, including after partial publication failure. Stable selection IDs + * allow retry/reconciliation without repeating already-committed preparation. No cross-store + * transaction is implied. The implementation must not load the entire pending backlog. + */ + int selectAndStage(int limit); + + /** + * Takes an execution projection without completing selected or pending ownership. + * Execution mutations must not silently modify retained earlier-stage state. + * Zero timeout polls; negative timeouts are invalid. An item has at most one active taker. + */ + Optional> take(Duration timeout) throws InterruptedException; + + /** + * Returns taken, not-yet-routed work with its updated execution projection, without silently changing + * earlier-stage state. Identity/source ownership must match the take. Repeated release cannot enqueue + * duplicates. Preserving runtime mutations does not promise durable retry counters or timing. + */ + void release(Selected selected); + + /** + * Stops router scheduling after the single destination has been recorded. This is idempotent and + * retains selected state through terminal processing, including for a future durable router + * combined with memory-backed destinations. It does not complete the pending source. + */ + void markRouted(SelectionId selection); + + /** + * Removes selected ownership after terminal processing and pending completion. Idempotent; + * never use this to signal router dequeue or initial destination dispatch. + */ + void complete(SelectionId selection); +} diff --git a/sendium-core/src/test/java/gr/cytech/sendium/core/outbound/OutboundWorkTest.java b/sendium-core/src/test/java/gr/cytech/sendium/core/outbound/OutboundWorkTest.java new file mode 100644 index 00000000..49fbee42 --- /dev/null +++ b/sendium-core/src/test/java/gr/cytech/sendium/core/outbound/OutboundWorkTest.java @@ -0,0 +1,80 @@ +package gr.cytech.sendium.core.outbound; + +import gr.cytech.sendium.core.message.StandardMessage; +import gr.cytech.sendium.core.outbound.OutboundWork.Assignment; +import gr.cytech.sendium.core.outbound.OutboundWork.Completed; +import gr.cytech.sendium.core.outbound.OutboundWork.Destination; +import gr.cytech.sendium.core.outbound.OutboundWork.Routed; +import gr.cytech.sendium.core.outbound.OutboundWork.Selected; +import gr.cytech.sendium.core.outbound.OutboundWork.SelectionId; +import gr.cytech.sendium.core.outbound.OutboundWork.SourceId; +import gr.cytech.sendium.core.outbound.OutboundWork.WorkId; +import org.junit.jupiter.api.Test; + +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import java.util.UUID; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +class OutboundWorkTest { + @Test + void assignmentAndCompletionKeepEverySourceDespiteCallerCollectionChanges() { + SourceId first = new SourceId(UUID.randomUUID()); + SourceId second = new SourceId(UUID.randomUUID()); + Set sources = new HashSet<>(Set.of(first, second)); + SelectionId selection = new SelectionId(UUID.randomUUID()); + var message = new StandardMessage(); + var selected = new Selected<>(selection, sources, message); + var destination = new Destination<>("provider", message); + var assignment = new Assignment<>(selection, sources, destination); + var completed = new Completed(selection, sources); + + sources.clear(); + + assertThat(selected.sources()).containsExactlyInAnyOrder(first, second); + assertThat(assignment.sources()).containsExactlyInAnyOrder(first, second); + assertThat(completed.sources()).containsExactlyInAnyOrder(first, second); + assertThat(assignment.destination()).isSameAs(destination); + assertThatThrownBy(() -> assignment.sources().clear()).isInstanceOf(UnsupportedOperationException.class); + assertThatThrownBy(() -> completed.sources().clear()).isInstanceOf(UnsupportedOperationException.class); + } + + @Test + void assignmentRequiresSourceOwnershipAndOneNamedDestination() { + var selection = new SelectionId(UUID.randomUUID()); + var sources = Set.of(new SourceId(UUID.randomUUID())); + var destination = new Destination<>("provider", new StandardMessage()); + + assertThatThrownBy(() -> new Assignment<>(selection, sources, null)) + .isInstanceOf(NullPointerException.class); + assertThatThrownBy(() -> new Assignment<>(selection, Set.of(), destination)) + .isInstanceOf(IllegalArgumentException.class); + assertThatThrownBy(() -> new Destination<>(" ", new StandardMessage())) + .isInstanceOf(IllegalArgumentException.class); + } + + @Test + void messageBearingValuesNeverInvokeMessageToString() { + StandardMessage message = new StandardMessage() { + @Override + public String toString() { + throw new AssertionError("Message contents must not be rendered by storage values"); + } + }; + var selection = new SelectionId(UUID.randomUUID()); + var sources = Set.of(new SourceId(UUID.randomUUID())); + var destination = new Destination<>("private-endpoint", message); + var values = List.of( + new Selected<>(selection, sources, message), + destination, + new Assignment<>(selection, sources, destination), + new Routed<>(new WorkId(UUID.randomUUID()), selection, destination.name(), message)); + + for (Object value : values) { + assertThat(value.toString()).doesNotContain("private-endpoint"); + } + } +} diff --git a/sendium-core/src/test/java/gr/cytech/sendium/core/outbound/consumer/OutboundCompositionTest.java b/sendium-core/src/test/java/gr/cytech/sendium/core/outbound/consumer/OutboundCompositionTest.java new file mode 100644 index 00000000..e5227774 --- /dev/null +++ b/sendium-core/src/test/java/gr/cytech/sendium/core/outbound/consumer/OutboundCompositionTest.java @@ -0,0 +1,53 @@ +package gr.cytech.sendium.core.outbound.consumer; + +import gr.cytech.sendium.core.message.StandardMessage; +import gr.cytech.sendium.core.outbound.OutboundWork; +import gr.cytech.sendium.core.storage.PendingMessageStore; +import gr.cytech.sendium.core.storage.RoutedWorkStore; +import gr.cytech.sendium.core.storage.SelectedRouterStore; +import org.junit.jupiter.api.Test; + +import java.time.Duration; +import java.util.Optional; +import java.util.Set; +import java.util.UUID; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +class OutboundCompositionTest { + @Test + @SuppressWarnings("unchecked") + void externalConsumerUsesCustomMessageTypesThroughTheThreeStagePortsDirectly() throws Exception { + PendingMessageStore pending = mock(PendingMessageStore.class); + SelectedRouterStore router = mock(SelectedRouterStore.class); + RoutedWorkStore routed = mock(RoutedWorkStore.class); + + var message = new CampaignMessage(); + message.campaignId = "campaign-7"; + var selection = new OutboundWork.SelectionId(UUID.randomUUID()); + var source = new OutboundWork.SourceId(UUID.randomUUID()); + var sources = Set.of(source); + var selected = new OutboundWork.Selected<>(selection, sources, message); + var destination = new OutboundWork.Destination<>("custom-provider", selected.message()); + var assignment = new OutboundWork.Assignment<>(selection, sources, destination); + var work = new OutboundWork.Routed<>(new OutboundWork.WorkId(UUID.randomUUID()), + selection, destination.name(), destination.message()); + when(router.take(Duration.ZERO)).thenReturn(Optional.of(selected)); + when(routed.record(assignment)).thenReturn(work); + + pending.admit(source, message); + CampaignMessage selectedMessage = router.take(Duration.ZERO).orElseThrow().message(); + CampaignMessage routedMessage = routed.record(assignment).message(); + + assertThat(selectedMessage.campaignId).isEqualTo("campaign-7"); + assertThat(routedMessage.campaignId).isEqualTo("campaign-7"); + verify(pending).admit(source, message); + } + + private static final class CampaignMessage extends StandardMessage { + private String campaignId; + } +} From a90b7e410faacd628ca5819662609f54fc866449 Mon Sep 17 00:00:00 2001 From: pavlos Date: Mon, 28 Sep 2026 14:10:38 +0300 Subject: [PATCH 02/20] feat(storage): validate standalone SMS backend profiles Task 1.2 - Add standalone backend selection, startup validation, and profile logging. Reject unavailable profiles before worker startup, preserve core embedding independence, and verify configuration and startup behavior. Refs #338 Signed-off-by: pavlos --- docs/09-configuration-reference.md | 30 ++++++ docs/14-outbound-storage.md | 44 ++++++--- sendium-app/pom.xml | 50 ++++++++++ .../app/storage/SmsStorageProfile.java | 22 +++++ .../app/storage/StandaloneSmsStorage.java | 37 ++++++++ .../src/main/resources/application.properties | 5 + .../app/storage/SmsAdmissionProbe.java | 26 ++++++ .../storage/SmsStorageFileStartupTest.java | 17 ++++ .../storage/SmsStorageMemoryStartupTest.java | 33 +++++++ .../SmsStoragePartialFileStartupTest.java | 17 ++++ .../SmsStoragePostgresqlStartupTest.java | 17 ++++ .../app/storage/SmsStorageProfileTest.java | 91 +++++++++++++++++++ .../app/storage/SmsStorageStartupSupport.java | 45 +++++++++ 13 files changed, 422 insertions(+), 12 deletions(-) create mode 100644 sendium-app/src/main/java/gr/cytech/sendium/app/storage/SmsStorageProfile.java create mode 100644 sendium-app/src/main/java/gr/cytech/sendium/app/storage/StandaloneSmsStorage.java create mode 100644 sendium-app/src/test/java/gr/cytech/sendium/app/storage/SmsAdmissionProbe.java create mode 100644 sendium-app/src/test/java/gr/cytech/sendium/app/storage/SmsStorageFileStartupTest.java create mode 100644 sendium-app/src/test/java/gr/cytech/sendium/app/storage/SmsStorageMemoryStartupTest.java create mode 100644 sendium-app/src/test/java/gr/cytech/sendium/app/storage/SmsStoragePartialFileStartupTest.java create mode 100644 sendium-app/src/test/java/gr/cytech/sendium/app/storage/SmsStoragePostgresqlStartupTest.java create mode 100644 sendium-app/src/test/java/gr/cytech/sendium/app/storage/SmsStorageProfileTest.java create mode 100644 sendium-app/src/test/java/gr/cytech/sendium/app/storage/SmsStorageStartupSupport.java diff --git a/docs/09-configuration-reference.md b/docs/09-configuration-reference.md index 5337673d..927bda53 100644 --- a/docs/09-configuration-reference.md +++ b/docs/09-configuration-reference.md @@ -43,6 +43,35 @@ In the Docker image, the working directory is `/work`, so the default configurat | `QUARKUS_HTTP_ACCESS_LOG_ENABLE` | `true` | Enables HTTP access logging. | | `QUARKUS_HTTP_ACCESS_LOG_DIRECTORY` | `/work/logs` | HTTP access log directory. | +## Outbound SMS Storage Profile + +The standalone application reads these global, startup-only settings from Quarkus configuration +(`application.properties`, JVM `-D` properties, or the environment). They are not hot-reloaded worker +settings in `smsg.properties`. + +| Property | Environment variable | Default | +| :--- | :--- | :--- | +| `sendium.sms.pending.backend` | `SENDIUM_SMS_PENDING_BACKEND` | `memory` | +| `sendium.sms.router-queue.backend` | `SENDIUM_SMS_ROUTER_QUEUE_BACKEND` | `memory` | +| `sendium.sms.routed-work.backend` | `SENDIUM_SMS_ROUTED_WORK_BACKEND` | `memory` | + +Only **`memory/memory/memory`** is currently selectable. Missing settings default to `memory`; +explicitly empty, unknown, or unsupported values fail startup. Values are case-sensitive. Requests +such as `file/file/memory`, partial-file combinations, or any PostgreSQL outbound combination fail +with an error naming the requested profile, supported profile, and configuration keys. There is no +fallback to a less durable backend. + +Validation runs before Sendium's file-watcher and router/worker startup observers. The validated +profile is retained for the process lifetime. Startup logs the effective profile and a prominent +`NON-DURABLE` warning: accepted messages and in-flight work can be lost on restart. The current +runtime pipeline is still memory-backed; the new stage implementations and end-to-end coordinator +wiring are subsequent work in #338. See [Outbound Storage Contracts](14-outbound-storage.md). + +The producer and startup observer live only in `sendium-app`. Embedding `sendium-core` does not +activate this validation, configure an outbound coordinator, or enforce the standalone profile list. +Embedding applications own their assembly. Outbound SMS selectors do not change the independently +configured DLR subsystem below. + ## DLR Storage Environment Variables | Variable | Default | Description | @@ -98,3 +127,4 @@ When the HTTP server is running, Sendium exposes: - [SMPP Configuration](04-smpp-configuration.md) - [Routing Engine](05-routing-engine.md) - [DLR Persistence](13-dlr-persistence.md) +- [Outbound Storage Contracts](14-outbound-storage.md) diff --git a/docs/14-outbound-storage.md b/docs/14-outbound-storage.md index 3a709927..7d1ff565 100644 --- a/docs/14-outbound-storage.md +++ b/docs/14-outbound-storage.md @@ -1,9 +1,10 @@ # Outbound storage contracts This describes the initial contracts for [#338](https://github.com/cytechmobile/sendium/issues/338), -under [#337](https://github.com/cytechmobile/sendium/issues/337). At this step these are library APIs: -the coordinator, memory stage implementations, configuration selector, and production wiring are -subsequent tasks. Existing ingress and queue behavior has not changed. +under [#337](https://github.com/cytechmobile/sendium/issues/337). The stage contracts are library APIs, +and standalone profile selection, early validation, and startup logging are implemented. The coordinator, +memory stage implementations, and end-to-end production wiring are subsequent tasks. Existing ingress +and queue behavior remains non-durable. ## Ownership model @@ -42,7 +43,7 @@ must reject a copied-route request as `UNSUPPORTED` before assignment or dispatc lookup happens to produce only one destination. It must not silently choose one result, strip the copy flag, or emulate fan-out using several assignments or forwarding calls. -This guard belongs to the upcoming routing integration task; this contract-only step does not change +This guard belongs to the upcoming routing integration task; this profile-selection step does not change existing routing behavior. The single-destination scope is the owner's revision to the copied-route requirements previously described in #337/#338. Those issue bodies have not been edited here. @@ -62,8 +63,15 @@ storage/ OutboundStorageException.java ``` -Memory implementations will live under `storage.memory` when introduced. No backend implementation -package is needed for this contract-only step. +Memory implementations will live under `storage.memory` when introduced. Standalone configuration +assembly lives separately in `sendium-app`, under `gr.cytech.sendium.app.storage`: + +- `SmsStorageProfile` is an immutable validated configuration value. Its constructor is the single + supported-profile validator; it accepts only `memory/memory/memory`. +- `StandaloneSmsStorage` produces that profile as a CDI singleton from runtime configuration and + requires it in an early startup observer. The observer logs the effective profile and non-durable warning. + +These classes are absent from the `sendium-core` artifact; they do not activate inside an embedding application. | Contract | Responsibility | |---|---| @@ -163,8 +171,8 @@ An application assembles compatible `PendingMessageStore`, `SelectedRouterSto `RoutedWorkStore` instances and passes them directly as three constructor dependencies to its coordinator implementation. There is no dependency-bundle type or public copier contract. The coordinator remains an interface at this step; the concrete constructor and wiring arrive with its -implementation. Constructors must have no activation side effects. Standalone CDI assembly is a -separate integration task; these APIs require no CDI annotations or container. +implementation. Constructors must have no activation side effects. The standalone profile is assembled +through CDI in `sendium-app`; the reusable lifecycle/storage APIs require no CDI annotations or container. For example, mCore's `Message extends StandardMessage`, billing-aware batch preparation, asynchronous committed tracking, and own lifecycle orchestration can be adapted without importing @@ -179,9 +187,9 @@ quiescence. Unrouted work returns to the router; already-routed work retains its worker removal is a separate policy, not an alias for application shutdown. Closure is idempotent, cannot mark unfinished work terminal, and does not imply that memory survives a restart. -## Planned standalone profile support +## Standalone profile selection -The following selectors will be introduced by the standalone wiring task: +The standalone application reads these selectors once during startup: ```properties sendium.sms.pending.backend=memory @@ -191,17 +199,29 @@ sendium.sms.routed-work.backend=memory | Pending / router / routed | Availability | Restart behavior | |---|---|---| -| `memory/memory/memory` | Target of #338; not yet wired at this contract-only step | Non-durable. | +| `memory/memory/memory` | Only selectable profile; stage implementations/coordinator wiring follow in #338 | Non-durable. | | `file/memory/memory` | #339 | Recover accepted work; reselect and reroute. | | `file/file/memory` | #345 | Recover selected work; reroute. | | `file/memory/file` | #345 | Reselect not-yet-routed work; restore recorded destinations. | | `file/file/file` | #345 | Restore selected work and recorded destinations. | | PostgreSQL profiles | #333 | Only explicitly implemented combinations will be supported. | -Standalone validation will accept only the implemented profile and reject unsupported requests before +Standalone validation accepts only this profile and rejects unsupported requests before admission, without falling back to memory. A custom backend identity in the library does not register a supported standalone profile. DLR storage remains independent of outbound stage selection. Future durable guarantees require surviving storage and remain at least once. +Missing selectors default to `memory`. Explicit blanks, unknown values, and every unimplemented +combination fail startup with the requested profile and supported choice. The values are case-sensitive +and are read from Quarkus runtime configuration, not hot-reloaded `smsg.properties` worker settings. +Use the corresponding environment variables `SENDIUM_SMS_PENDING_BACKEND`, +`SENDIUM_SMS_ROUTER_QUEUE_BACKEND`, and `SENDIUM_SMS_ROUTED_WORK_BACKEND`, or JVM `-D` +properties. Changes require a restart; they do not require rebuilding the application. + +The startup observer uses `Interceptor.Priority.PLATFORM_BEFORE`, ahead of the existing file watchers +(`LIBRARY_BEFORE`) and router/worker observers (`APPLICATION`). Injecting the singleton profile into +that observer forces validation before those later observers run. Successful startup logs the profile +and a `NON-DURABLE` warning. The concrete stage stores are not constructed at this intermediate step. + This milestone includes actionable errors and startup profile/non-durable logging. Metrics, readiness endpoints, and broader observability are deferred. Contract-level status is not a health endpoint. diff --git a/sendium-app/pom.xml b/sendium-app/pom.xml index 9f0b6ece..3de8c9a0 100644 --- a/sendium-app/pom.xml +++ b/sendium-app/pom.xml @@ -22,6 +22,21 @@ io.quarkus quarkus-rest + + io.quarkus + quarkus-junit-internal + test + + + io.quarkus + quarkus-arc-deployment + test + + + org.assertj + assertj-core + test + @@ -30,6 +45,41 @@ + + maven-surefire-plugin + ${surefire-plugin.version} + + + org.jboss.logmanager.LogManager + ${maven.home} + + + + + maven-checkstyle-plugin + ${checkstyle-plugin.version} + + + com.puppycrawl.tools + checkstyle + ${checkstyle-tools.version} + + + + ${skipTests} + ${project.basedir}/../sendium-core/checkstyle.xml + true + true + warning + + + + validate + validate + check + + + ${quarkus.platform.group-id} quarkus-maven-plugin diff --git a/sendium-app/src/main/java/gr/cytech/sendium/app/storage/SmsStorageProfile.java b/sendium-app/src/main/java/gr/cytech/sendium/app/storage/SmsStorageProfile.java new file mode 100644 index 00000000..5eca61d5 --- /dev/null +++ b/sendium-app/src/main/java/gr/cytech/sendium/app/storage/SmsStorageProfile.java @@ -0,0 +1,22 @@ +package gr.cytech.sendium.app.storage; + +public record SmsStorageProfile(String pending, String routerQueue, String routedWork) { + public static final String PENDING_BACKEND = "sendium.sms.pending.backend"; + public static final String ROUTER_BACKEND = "sendium.sms.router-queue.backend"; + public static final String ROUTED_BACKEND = "sendium.sms.routed-work.backend"; + + public SmsStorageProfile { + if (!"memory".equals(pending) || !"memory".equals(routerQueue) || !"memory".equals(routedWork)) { + throw new IllegalArgumentException( + "Unsupported outbound SMS storage profile '" + pending + "/" + routerQueue + "/" + routedWork + + "' (pending/router-queue/routed-work). Supported profiles: memory/memory/memory. " + + "Configure " + PENDING_BACKEND + ", " + ROUTER_BACKEND + ", and " + ROUTED_BACKEND + + "; file and postgresql outbound backends are not implemented. No fallback is available."); + } + } + + @Override + public String toString() { + return pending + "/" + routerQueue + "/" + routedWork; + } +} diff --git a/sendium-app/src/main/java/gr/cytech/sendium/app/storage/StandaloneSmsStorage.java b/sendium-app/src/main/java/gr/cytech/sendium/app/storage/StandaloneSmsStorage.java new file mode 100644 index 00000000..4cd8fc41 --- /dev/null +++ b/sendium-app/src/main/java/gr/cytech/sendium/app/storage/StandaloneSmsStorage.java @@ -0,0 +1,37 @@ +package gr.cytech.sendium.app.storage; + +import io.quarkus.runtime.StartupEvent; +import jakarta.annotation.Priority; +import jakarta.enterprise.context.ApplicationScoped; +import jakarta.enterprise.event.Observes; +import jakarta.enterprise.inject.Produces; +import jakarta.inject.Singleton; +import jakarta.interceptor.Interceptor; +import org.eclipse.microprofile.config.Config; +import org.eclipse.microprofile.config.ConfigValue; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +@ApplicationScoped +public class StandaloneSmsStorage { + private static final Logger logger = LoggerFactory.getLogger(StandaloneSmsStorage.class); + + @Produces + @Singleton + SmsStorageProfile profile(Config config) { + return new SmsStorageProfile(backend(config, SmsStorageProfile.PENDING_BACKEND), + backend(config, SmsStorageProfile.ROUTER_BACKEND), + backend(config, SmsStorageProfile.ROUTED_BACKEND)); + } + + void start(@Observes @Priority(Interceptor.Priority.PLATFORM_BEFORE) StartupEvent event, SmsStorageProfile profile) { + logger.info("Outbound SMS storage profile (pending/router-queue/routed-work): {}", profile); + logger.warn("NON-DURABLE outbound SMS storage: accepted messages and in-flight work can be lost on process restart."); + } + + private static String backend(Config config, String name) { + ConfigValue property = config.getConfigValue(name); + // An absent setting uses the default; an explicitly empty setting must fail validation. + return property.getRawValue() == null ? "memory" : property.getValue(); + } +} diff --git a/sendium-app/src/main/resources/application.properties b/sendium-app/src/main/resources/application.properties index 171c35b5..61c5d32c 100644 --- a/sendium-app/src/main/resources/application.properties +++ b/sendium-app/src/main/resources/application.properties @@ -2,6 +2,11 @@ smsg.routing.file.path=conf/routingTable.conf smsg.properties.file.path=conf/smsg.properties smsg.credentials.file.path=conf/credentials.yml +# Global, startup-only outbound storage profile. Only memory/memory/memory is implemented. +sendium.sms.pending.backend=memory +sendium.sms.router-queue.backend=memory +sendium.sms.routed-work.backend=memory + # DLR persistence. PostgreSQL requires valid named datasource connection settings. sendium.dlr.persistence.enabled=true sendium.dlr.delivery-claim-duration=${SENDIUM_DLR_DELIVERY_CLAIM_DURATION:5M} diff --git a/sendium-app/src/test/java/gr/cytech/sendium/app/storage/SmsAdmissionProbe.java b/sendium-app/src/test/java/gr/cytech/sendium/app/storage/SmsAdmissionProbe.java new file mode 100644 index 00000000..e19f29d2 --- /dev/null +++ b/sendium-app/src/test/java/gr/cytech/sendium/app/storage/SmsAdmissionProbe.java @@ -0,0 +1,26 @@ +package gr.cytech.sendium.app.storage; + +import io.quarkus.runtime.StartupEvent; +import jakarta.annotation.Priority; +import jakarta.enterprise.context.ApplicationScoped; +import jakarta.enterprise.event.Observes; +import jakarta.inject.Inject; +import jakarta.interceptor.Interceptor; +import org.slf4j.LoggerFactory; + +@ApplicationScoped +public class SmsAdmissionProbe { + @Inject + SmsStorageProfile profile; + + private SmsStorageProfile profileAtStartup; + + void start(@Observes @Priority(Interceptor.Priority.APPLICATION) StartupEvent event) { + profileAtStartup = profile; + LoggerFactory.getLogger(SmsAdmissionProbe.class).info("test admission startup reached"); + } + + public SmsStorageProfile profileAtStartup() { + return profileAtStartup; + } +} diff --git a/sendium-app/src/test/java/gr/cytech/sendium/app/storage/SmsStorageFileStartupTest.java b/sendium-app/src/test/java/gr/cytech/sendium/app/storage/SmsStorageFileStartupTest.java new file mode 100644 index 00000000..5e3e99ff --- /dev/null +++ b/sendium-app/src/test/java/gr/cytech/sendium/app/storage/SmsStorageFileStartupTest.java @@ -0,0 +1,17 @@ +package gr.cytech.sendium.app.storage; + +import io.quarkus.test.QuarkusExtensionTest; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.RegisterExtension; + +import static org.junit.jupiter.api.Assertions.fail; + +class SmsStorageFileStartupTest { + @RegisterExtension + static final QuarkusExtensionTest APP = SmsStorageStartupSupport.rejected("file", "file", "memory"); + + @Test + void startupMustFailBeforeAdmission() { + fail("Unsupported profile started"); + } +} diff --git a/sendium-app/src/test/java/gr/cytech/sendium/app/storage/SmsStorageMemoryStartupTest.java b/sendium-app/src/test/java/gr/cytech/sendium/app/storage/SmsStorageMemoryStartupTest.java new file mode 100644 index 00000000..40682c47 --- /dev/null +++ b/sendium-app/src/test/java/gr/cytech/sendium/app/storage/SmsStorageMemoryStartupTest.java @@ -0,0 +1,33 @@ +package gr.cytech.sendium.app.storage; + +import io.quarkus.test.QuarkusExtensionTest; +import jakarta.inject.Inject; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.RegisterExtension; + +import static org.assertj.core.api.Assertions.assertThat; + +class SmsStorageMemoryStartupTest { + @RegisterExtension + static final QuarkusExtensionTest APP = SmsStorageStartupSupport.application() + .overrideRuntimeConfigKey(SmsStorageProfile.PENDING_BACKEND, "memory") + .overrideRuntimeConfigKey(SmsStorageProfile.ROUTER_BACKEND, "memory") + .overrideRuntimeConfigKey(SmsStorageProfile.ROUTED_BACKEND, "memory") + .assertLogRecords(records -> assertThat(records).extracting(SmsStorageStartupSupport::message) + .containsExactly( + "Outbound SMS storage profile (pending/router-queue/routed-work): memory/memory/memory", + "NON-DURABLE outbound SMS storage: accepted messages and in-flight work can be lost on process restart.", + SmsStorageStartupSupport.ADMISSION_STARTED)); + + @Inject + SmsStorageProfile profile; + + @Inject + SmsAdmissionProbe admission; + + @Test + void validatedProfileIsAvailableBeforeAdmissionStartup() { + assertThat(profile).isEqualTo(new SmsStorageProfile("memory", "memory", "memory")); + assertThat(admission.profileAtStartup()).isSameAs(profile); + } +} diff --git a/sendium-app/src/test/java/gr/cytech/sendium/app/storage/SmsStoragePartialFileStartupTest.java b/sendium-app/src/test/java/gr/cytech/sendium/app/storage/SmsStoragePartialFileStartupTest.java new file mode 100644 index 00000000..9759efac --- /dev/null +++ b/sendium-app/src/test/java/gr/cytech/sendium/app/storage/SmsStoragePartialFileStartupTest.java @@ -0,0 +1,17 @@ +package gr.cytech.sendium.app.storage; + +import io.quarkus.test.QuarkusExtensionTest; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.RegisterExtension; + +import static org.junit.jupiter.api.Assertions.fail; + +class SmsStoragePartialFileStartupTest { + @RegisterExtension + static final QuarkusExtensionTest APP = SmsStorageStartupSupport.rejected("memory", "memory", "file"); + + @Test + void startupMustFailBeforeAdmission() { + fail("Unsupported profile started"); + } +} diff --git a/sendium-app/src/test/java/gr/cytech/sendium/app/storage/SmsStoragePostgresqlStartupTest.java b/sendium-app/src/test/java/gr/cytech/sendium/app/storage/SmsStoragePostgresqlStartupTest.java new file mode 100644 index 00000000..2f5c045e --- /dev/null +++ b/sendium-app/src/test/java/gr/cytech/sendium/app/storage/SmsStoragePostgresqlStartupTest.java @@ -0,0 +1,17 @@ +package gr.cytech.sendium.app.storage; + +import io.quarkus.test.QuarkusExtensionTest; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.RegisterExtension; + +import static org.junit.jupiter.api.Assertions.fail; + +class SmsStoragePostgresqlStartupTest { + @RegisterExtension + static final QuarkusExtensionTest APP = SmsStorageStartupSupport.rejected("postgresql", "postgresql", "postgresql"); + + @Test + void startupMustFailBeforeAdmission() { + fail("Unsupported profile started"); + } +} diff --git a/sendium-app/src/test/java/gr/cytech/sendium/app/storage/SmsStorageProfileTest.java b/sendium-app/src/test/java/gr/cytech/sendium/app/storage/SmsStorageProfileTest.java new file mode 100644 index 00000000..0c8bd970 --- /dev/null +++ b/sendium-app/src/test/java/gr/cytech/sendium/app/storage/SmsStorageProfileTest.java @@ -0,0 +1,91 @@ +package gr.cytech.sendium.app.storage; + +import io.smallrye.config.SmallRyeConfigBuilder; +import org.eclipse.microprofile.config.spi.ConfigSource; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; +import org.junit.jupiter.params.provider.ValueSource; + +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.stream.Stream; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +class SmsStorageProfileTest { + @Test + void missingSelectorsUseMemoryDefaults() { + assertThat(profile(Map.of()).toString()).isEqualTo("memory/memory/memory"); + } + + @Test + void explicitMemoryIsAccepted() { + assertThat(profile(Map.of( + SmsStorageProfile.PENDING_BACKEND, "memory", + SmsStorageProfile.ROUTER_BACKEND, "memory", + SmsStorageProfile.ROUTED_BACKEND, "memory"))) + .isEqualTo(new SmsStorageProfile("memory", "memory", "memory")); + } + + @ParameterizedTest + @MethodSource("unsupportedProfiles") + void rejectsEveryUnimplementedCombination(String pending, String router, String routed) { + assertThatThrownBy(() -> profile(Map.of( + SmsStorageProfile.PENDING_BACKEND, pending, + SmsStorageProfile.ROUTER_BACKEND, router, + SmsStorageProfile.ROUTED_BACKEND, routed))) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("'" + pending + "/" + router + "/" + routed + "'") + .hasMessageContaining("Supported profiles: memory/memory/memory") + .hasMessageContaining(SmsStorageProfile.PENDING_BACKEND) + .hasMessageContaining(SmsStorageProfile.ROUTER_BACKEND) + .hasMessageContaining(SmsStorageProfile.ROUTED_BACKEND); + } + + @ParameterizedTest + @ValueSource(strings = {"", " ", "disk", "MEMORY"}) + void explicitInvalidValuesNeverUseTheDefault(String value) { + for (String key : List.of(SmsStorageProfile.PENDING_BACKEND, + SmsStorageProfile.ROUTER_BACKEND, SmsStorageProfile.ROUTED_BACKEND)) { + assertThatThrownBy(() -> profile(Map.of(key, value))) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("Unsupported outbound SMS storage profile"); + } + } + + private static Stream unsupportedProfiles() { + List backends = List.of("memory", "file", "postgresql"); + return backends.stream().flatMap(pending -> backends.stream().flatMap(router -> backends.stream() + .filter(routed -> !List.of(pending, router, routed).equals(List.of("memory", "memory", "memory"))) + .map(routed -> Arguments.of(pending, router, routed)))); + } + + private static SmsStorageProfile profile(Map settings) { + var config = new SmallRyeConfigBuilder().withSources(new ConfigSource() { + @Override + public Map getProperties() { + return settings; + } + + @Override + public Set getPropertyNames() { + return settings.keySet(); + } + + @Override + public String getValue(String propertyName) { + return settings.get(propertyName); + } + + @Override + public String getName() { + return "profile-test"; + } + }).build(); + return new StandaloneSmsStorage().profile(config); + } +} diff --git a/sendium-app/src/test/java/gr/cytech/sendium/app/storage/SmsStorageStartupSupport.java b/sendium-app/src/test/java/gr/cytech/sendium/app/storage/SmsStorageStartupSupport.java new file mode 100644 index 00000000..5b567c7c --- /dev/null +++ b/sendium-app/src/test/java/gr/cytech/sendium/app/storage/SmsStorageStartupSupport.java @@ -0,0 +1,45 @@ +package gr.cytech.sendium.app.storage; + +import io.quarkus.test.QuarkusExtensionTest; +import org.jboss.logmanager.ExtLogRecord; + +import java.util.logging.LogRecord; + +import static org.assertj.core.api.Assertions.assertThat; + +final class SmsStorageStartupSupport { + static final String ADMISSION_STARTED = "test admission startup reached"; + + private SmsStorageStartupSupport() { + } + + static QuarkusExtensionTest application() { + return new QuarkusExtensionTest() + .withApplicationRoot(archive -> archive.addClasses( + StandaloneSmsStorage.class, SmsStorageProfile.class, SmsAdmissionProbe.class, + SmsStorageStartupSupport.class)) + // Exercise the standalone assembly independently of protocol worker beans. + .overrideConfigKey("quarkus.arc.exclude-dependency.core.group-id", "gr.cytech") + .overrideConfigKey("quarkus.arc.exclude-dependency.core.artifact-id", "sendium-core") + .overrideConfigKey("sendium.dlr.persistence.enabled", "false") + .overrideConfigKey("quarkus.http.test-port", "0") + .setLogRecordPredicate(record -> StandaloneSmsStorage.class.getName().equals(record.getLoggerName()) || + SmsAdmissionProbe.class.getName().equals(record.getLoggerName())); + } + + static QuarkusExtensionTest rejected(String pending, String router, String routed) { + return application() + .overrideRuntimeConfigKey(SmsStorageProfile.PENDING_BACKEND, pending) + .overrideRuntimeConfigKey(SmsStorageProfile.ROUTER_BACKEND, router) + .overrideRuntimeConfigKey(SmsStorageProfile.ROUTED_BACKEND, routed) + .assertException(failure -> assertThat(failure) + .hasStackTraceContaining("Unsupported outbound SMS storage profile '" + + pending + "/" + router + "/" + routed + "'") + .hasStackTraceContaining("Supported profiles: memory/memory/memory")) + .assertLogRecords(records -> assertThat(records).isEmpty()); + } + + static String message(LogRecord record) { + return record instanceof ExtLogRecord extended ? extended.getFormattedMessage() : record.getMessage(); + } +} From 6b1653f8fd183c1bdde9468fdd2bdf7bc4872fd4 Mon Sep 17 00:00:00 2001 From: pavlos Date: Mon, 28 Sep 2026 14:40:46 +0300 Subject: [PATCH 03/20] feat(storage): add memory pending and router stores Task 2.1 - Implement memory pending ownership and bounded router selection. Retain accepted sources through routing, select indexed eligible work within bounded queued and in-flight capacity, and preserve retry ownership through failures. Refs #338 Signed-off-by: pavlos --- docs/09-configuration-reference.md | 6 +- docs/14-outbound-storage.md | 73 ++- .../core/storage/SelectedRouterStore.java | 2 + .../memory/MemoryPendingMessageStore.java | 226 +++++++++ .../memory/MemorySelectedRouterStore.java | 267 ++++++++++ .../storage/memory/MemoryStorageTest.java | 475 ++++++++++++++++++ 6 files changed, 1042 insertions(+), 7 deletions(-) create mode 100644 sendium-core/src/main/java/gr/cytech/sendium/core/storage/memory/MemoryPendingMessageStore.java create mode 100644 sendium-core/src/main/java/gr/cytech/sendium/core/storage/memory/MemorySelectedRouterStore.java create mode 100644 sendium-core/src/test/java/gr/cytech/sendium/core/storage/memory/MemoryStorageTest.java diff --git a/docs/09-configuration-reference.md b/docs/09-configuration-reference.md index 927bda53..53d13d41 100644 --- a/docs/09-configuration-reference.md +++ b/docs/09-configuration-reference.md @@ -64,8 +64,10 @@ fallback to a less durable backend. Validation runs before Sendium's file-watcher and router/worker startup observers. The validated profile is retained for the process lifetime. Startup logs the effective profile and a prominent `NON-DURABLE` warning: accepted messages and in-flight work can be lost on restart. The current -runtime pipeline is still memory-backed; the new stage implementations and end-to-end coordinator -wiring are subsequent work in #338. See [Outbound Storage Contracts](14-outbound-storage.md). +runtime pipeline is still memory-backed. Pending and selected-router memory stores exist as library +components; end-to-end coordinator and ingress/worker wiring are subsequent work in #338. Their +capacity bounds are constructor arguments at this stage, not additional operator settings. +See [Outbound Storage Contracts](14-outbound-storage.md). The producer and startup observer live only in `sendium-app`. Embedding `sendium-core` does not activate this validation, configure an outbound coordinator, or enforce the standalone profile list. diff --git a/docs/14-outbound-storage.md b/docs/14-outbound-storage.md index 7d1ff565..4acfb36a 100644 --- a/docs/14-outbound-storage.md +++ b/docs/14-outbound-storage.md @@ -2,9 +2,10 @@ This describes the initial contracts for [#338](https://github.com/cytechmobile/sendium/issues/338), under [#337](https://github.com/cytechmobile/sendium/issues/337). The stage contracts are library APIs, -and standalone profile selection, early validation, and startup logging are implemented. The coordinator, -memory stage implementations, and end-to-end production wiring are subsequent tasks. Existing ingress -and queue behavior remains non-durable. +and standalone profile selection, early validation, and startup logging are implemented. Memory pending +and selected-router stores are available as explicitly constructed library components. The coordinator, +routed-work store, and end-to-end production wiring are subsequent tasks. Existing ingress and queue +behavior remains non-durable. ## Ownership model @@ -61,9 +62,12 @@ storage/ SelectedRouterStore.java RoutedWorkStore.java OutboundStorageException.java + memory/ + MemoryPendingMessageStore.java + MemorySelectedRouterStore.java ``` -Memory implementations will live under `storage.memory` when introduced. Standalone configuration +Memory implementations live under `storage.memory`. Standalone configuration assembly lives separately in `sendium-app`, under `gr.cytech.sendium.app.storage`: - `SmsStorageProfile` is an immutable validated configuration value. Its constructor is the single @@ -128,6 +132,65 @@ provider-outcome checkpoints and exactly-once submission remain outside the guar These are requirements for the later durable implementations, not a restart guarantee for memory. +## Memory pending and selected-router implementation + +These two classes are plain Java components with no CDI activation or background refill threads: + +- `MemoryPendingMessageStore` takes a positive maximum source count and a + `UnaryOperator` snapshot function. An optional `Function` supplies absolute + eligibility time at admission; the default makes every admitted source immediately eligible. No new + interpretation of HTTP deferred-delivery or validity fields is introduced by this library step. +- `MemorySelectedRouterStore` takes that pending store, a positive routing capacity, and optionally + a `Clock` (UTC by default). Open pending first, then the selected store. A pending instance accepts + only one selected-router owner for its lifetime, including after that router closes. + +The snapshot function is a concrete memory-backend constructor dependency, not a new public copier +interface. It must preserve the concrete subtype and all fields, detach mutable state, and not retain +references that a caller could later mutate. The backend rejects null, identity, and subtype-flattening +results. It cannot generically prove deep isolation of arbitrary application-specific objects; supplying +a correct snapshot function belongs to assembly. Functions run under the store lock and must be +side-effect-free, nonblocking, and must not reenter these stores. The standalone message mapper is +supplied when ingress is wired; this step does not serialize messages or define a durable codec. + +Pending admission is idempotent by source ID while that source exists, even at capacity. New source +admission fails with `CAPACITY_EXCEEDED` when the bound is reached. Admission or snapshot failure +does not reserve a source slot. A single-source `find` returns an isolated value for backend consumers; +there is no all-backlog loading API. Pending completion remains the caller's terminal-processing +decision, not a side effect of selection, take, or routing. + +Selection uses priority-indexed ordered sets maintained at admission. It inspects due candidates rather +than copying or sorting the whole backlog on each refill. Higher numeric priorities select first; within +a priority, earlier eligibility comes first, then admission order. Future work in a higher-priority group +does not block eligible work at a lower priority. A batch captures one clock instant. This is an +in-process scheduling policy, not a durable global-order guarantee. + +`selectAndStage(limit)` publishes at most the smaller of the requested limit and free routing slots. +Each selected source is removed from the eligibility index but retained in pending storage. Shared +locking makes publication and removal from eligibility indivisible per source. If snapshotting a later +item fails, the published prefix remains available and the failed item remains eligible; retry neither +duplicates selected sources nor loses accepted work. + +The selected backlog schedules in publication order. Routing capacity counts both queued and taken +items, so returning taken work never requires another slot. `markRouted` releases the routing slot but +retains selected ownership; only after pending completion may selected `complete` remove the record. +Routed sources are not eligible for another batch. The caller invokes bounded refill as slots become +available; taking work does not implicitly select a new batch. + +Take returns an isolated execution projection. Return the actual `Selected` value from that take after +updating its message; it serves as the local attempt handle, while its stable selection ID remains the +stage identity. Return snapshots the updated execution payload without changing accepted content. +Duplicate returns do not enqueue duplicates, and an older attempt cannot replace an active newer +take. Reconstructed or unrelated projections are rejected. This local handle is not a persisted dequeue +stage or a multi-process lease. + +Blocking takes support zero-time polling, timeout, interruption, and wake-up when work is published +or either store closes. Close selected before pending after stopping execution. Closing is idempotent; +neither store can reopen after close. Closing a selected store does not complete its pending sources +or make its claimed sources eligible again; shutdown finishes by closing the pending store as well. +All state is instance-local. New instances start empty and provide no restart recovery. Counts bound +admission and active routing, not message byte size; required terminal bookkeeping is removed by +coordinated cleanup in the later integration tasks. + ## Completion and retry boundaries - Admission retries reuse the same source ID and do not overwrite the accepted state while the @@ -221,7 +284,7 @@ properties. Changes require a restart; they do not require rebuilding the applic The startup observer uses `Interceptor.Priority.PLATFORM_BEFORE`, ahead of the existing file watchers (`LIBRARY_BEFORE`) and router/worker observers (`APPLICATION`). Injecting the singleton profile into that observer forces validation before those later observers run. Successful startup logs the profile -and a `NON-DURABLE` warning. The concrete stage stores are not constructed at this intermediate step. +and a `NON-DURABLE` warning. The library memory stores are not yet constructed by this startup observer. This milestone includes actionable errors and startup profile/non-durable logging. Metrics, readiness endpoints, and broader observability are deferred. Contract-level status is not a health endpoint. diff --git a/sendium-core/src/main/java/gr/cytech/sendium/core/storage/SelectedRouterStore.java b/sendium-core/src/main/java/gr/cytech/sendium/core/storage/SelectedRouterStore.java index e6cc627a..a5a817b5 100644 --- a/sendium-core/src/main/java/gr/cytech/sendium/core/storage/SelectedRouterStore.java +++ b/sendium-core/src/main/java/gr/cytech/sendium/core/storage/SelectedRouterStore.java @@ -25,6 +25,7 @@ public interface SelectedRouterStore extends Outbound * Takes an execution projection without completing selected or pending ownership. * Execution mutations must not silently modify retained earlier-stage state. * Zero timeout polls; negative timeouts are invalid. An item has at most one active taker. + * Retain the returned Selected value as the local attempt handle when returning unfinished work. */ Optional> take(Duration timeout) throws InterruptedException; @@ -32,6 +33,7 @@ public interface SelectedRouterStore extends Outbound * Returns taken, not-yet-routed work with its updated execution projection, without silently changing * earlier-stage state. Identity/source ownership must match the take. Repeated release cannot enqueue * duplicates. Preserving runtime mutations does not promise durable retry counters or timing. + * Pass the take result itself with its message updated, not a reconstructed or unrelated projection. */ void release(Selected selected); diff --git a/sendium-core/src/main/java/gr/cytech/sendium/core/storage/memory/MemoryPendingMessageStore.java b/sendium-core/src/main/java/gr/cytech/sendium/core/storage/memory/MemoryPendingMessageStore.java new file mode 100644 index 00000000..f2501889 --- /dev/null +++ b/sendium-core/src/main/java/gr/cytech/sendium/core/storage/memory/MemoryPendingMessageStore.java @@ -0,0 +1,226 @@ +package gr.cytech.sendium.core.storage.memory; + +import gr.cytech.sendium.core.message.StandardMessage; +import gr.cytech.sendium.core.outbound.OutboundWork.SourceId; +import gr.cytech.sendium.core.storage.OutboundStorageException; +import gr.cytech.sendium.core.storage.PendingMessageStore; + +import java.time.Instant; +import java.util.Comparator; +import java.util.HashMap; +import java.util.Map; +import java.util.NavigableMap; +import java.util.NavigableSet; +import java.util.Objects; +import java.util.Optional; +import java.util.Set; +import java.util.TreeMap; +import java.util.TreeSet; +import java.util.concurrent.locks.Condition; +import java.util.concurrent.locks.ReentrantLock; +import java.util.function.Function; +import java.util.function.UnaryOperator; + +import static gr.cytech.sendium.core.storage.OutboundStorageException.Reason.CAPACITY_EXCEEDED; +import static gr.cytech.sendium.core.storage.OutboundStorageException.Reason.INVALID_TRANSITION; +import static gr.cytech.sendium.core.storage.OutboundStorageException.Reason.OWNERSHIP_CONFLICT; +import static gr.cytech.sendium.core.storage.OutboundStorageException.Reason.UNAVAILABLE; +import static gr.cytech.sendium.core.storage.OutboundStorageException.Reason.UNSUPPORTED; + +/** + * Instance-local accepted state with a bounded source count. Snapshot functions must preserve the + * concrete message type and detach all mutable fields. Snapshot and eligibility functions run under + * the store lock and must be side-effect-free, nonblocking, and must not call back into these stores. + */ +public final class MemoryPendingMessageStore implements PendingMessageStore { + final ReentrantLock lock = new ReentrantLock(); + final Condition changed = lock.newCondition(); + + private final int capacity; + private final UnaryOperator snapshots; + private final Function eligibleAt; + private final Map> records = new HashMap<>(); + private final NavigableMap>> waiting = new TreeMap<>(Comparator.reverseOrder()); + private State state = State.NEW; + private Object selectorOwner; + private long sequence; + + public MemoryPendingMessageStore(int capacity, UnaryOperator snapshots) { + this(capacity, snapshots, message -> Instant.MIN); + } + + /** Eligibility is evaluated once at admission; subsequent message mutations cannot change it. */ + public MemoryPendingMessageStore(int capacity, UnaryOperator snapshots, Function eligibleAt) { + if (capacity <= 0) { + throw new IllegalArgumentException("Pending capacity must be positive"); + } + this.capacity = capacity; + this.snapshots = Objects.requireNonNull(snapshots, "snapshots"); + this.eligibleAt = Objects.requireNonNull(eligibleAt, "eligibleAt"); + } + + @Override + public void open() { + lock.lock(); + try { + if (state == State.CLOSED) { + throw new OutboundStorageException(Role.PENDING, INVALID_TRANSITION, "Closed memory pending store cannot reopen"); + } + state = State.READY; + } finally { + lock.unlock(); + } + } + + @Override + public Status status() { + lock.lock(); + try { + return new Status("memory", false, state); + } finally { + lock.unlock(); + } + } + + @Override + public void admit(SourceId source, M message) { + Objects.requireNonNull(source, "source"); + Objects.requireNonNull(message, "message"); + lock.lock(); + try { + requireReady(Role.PENDING); + if (records.containsKey(source)) { + return; + } + if (records.size() >= capacity) { + throw new OutboundStorageException(Role.PENDING, CAPACITY_EXCEEDED, "Memory pending capacity reached"); + } + M snapshot = snapshot(message, Role.PENDING); + Instant available; + try { + available = Objects.requireNonNull(eligibleAt.apply(snapshot), "eligibleAt result"); + } catch (RuntimeException failure) { + throw new OutboundStorageException(Role.PENDING, UNSUPPORTED, "Cannot determine source eligibility", failure); + } + Entry entry = new Entry<>(source, snapshot, available, sequence++); + records.put(source, entry); + waiting.computeIfAbsent(entry.priority, ignored -> new TreeSet<>(Comparator + .comparing((Entry item) -> item.available) + .thenComparingLong(item -> item.sequence))).add(entry); + } finally { + lock.unlock(); + } + } + + /** Single-source lookup for memory-backend consumers; returns an isolated execution value. */ + public Optional find(SourceId source) { + Objects.requireNonNull(source, "source"); + lock.lock(); + try { + requireReady(Role.PENDING); + Entry entry = records.get(source); + return entry == null ? Optional.empty() : Optional.of(snapshot(entry.message, Role.PENDING)); + } finally { + lock.unlock(); + } + } + + @Override + public void complete(Set sources) { + Set checked = Set.copyOf(sources); + lock.lock(); + try { + requireReady(Role.PENDING); + for (SourceId source : checked) { + Entry entry = records.remove(source); + if (entry != null) { + removeWaiting(entry); + } + } + } finally { + lock.unlock(); + } + } + + @Override + public void close() { + lock.lock(); + try { + state = State.CLOSED; + records.clear(); + waiting.clear(); + changed.signalAll(); + } finally { + lock.unlock(); + } + } + + void requireReady(Role operation) { + if (state != State.READY) { + throw new OutboundStorageException(operation, UNAVAILABLE, "Memory pending store is not ready"); + } + } + + void attachSelector(Object owner) { + requireReady(Role.ROUTER_QUEUE); + if (selectorOwner != null && selectorOwner != owner) { + throw new OutboundStorageException(Role.ROUTER_QUEUE, OWNERSHIP_CONFLICT, + "Memory pending store already has a selected-router owner"); + } + selectorOwner = owner; + } + + Entry nextEligible(Instant now) { + for (NavigableSet> priority : waiting.values()) { + Entry candidate = priority.first(); + if (!candidate.available.isAfter(now)) { + return candidate; + } + } + return null; + } + + void removeWaiting(Entry entry) { + NavigableSet> priority = waiting.get(entry.priority); + if (priority != null) { + priority.remove(entry); + if (priority.isEmpty()) { + waiting.remove(entry.priority); + } + } + } + + boolean contains(SourceId source) { + return records.containsKey(source); + } + + M snapshot(M message, Role operation) { + M copy; + try { + copy = snapshots.apply(message); + } catch (RuntimeException failure) { + throw new OutboundStorageException(operation, UNAVAILABLE, "Cannot snapshot memory work", failure); + } + if (copy == null || copy == message || copy.getClass() != message.getClass()) { + throw new OutboundStorageException(operation, UNSUPPORTED, + "Memory snapshots must be independent and preserve the concrete message type"); + } + return copy; + } + + static final class Entry { + final SourceId source; + final M message; + final int priority; + final Instant available; + final long sequence; + + Entry(SourceId source, M message, Instant available, long sequence) { + this.source = source; + this.message = message; + this.priority = message.priority; + this.available = available; + this.sequence = sequence; + } + } +} diff --git a/sendium-core/src/main/java/gr/cytech/sendium/core/storage/memory/MemorySelectedRouterStore.java b/sendium-core/src/main/java/gr/cytech/sendium/core/storage/memory/MemorySelectedRouterStore.java new file mode 100644 index 00000000..30e275af --- /dev/null +++ b/sendium-core/src/main/java/gr/cytech/sendium/core/storage/memory/MemorySelectedRouterStore.java @@ -0,0 +1,267 @@ +package gr.cytech.sendium.core.storage.memory; + +import gr.cytech.sendium.core.message.StandardMessage; +import gr.cytech.sendium.core.outbound.OutboundWork.Selected; +import gr.cytech.sendium.core.outbound.OutboundWork.SelectionId; +import gr.cytech.sendium.core.outbound.OutboundWork.SourceId; +import gr.cytech.sendium.core.storage.OutboundStorageException; +import gr.cytech.sendium.core.storage.SelectedRouterStore; + +import java.time.Clock; +import java.time.Duration; +import java.time.Instant; +import java.util.ArrayDeque; +import java.util.HashMap; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import java.util.Set; +import java.util.UUID; + +import static gr.cytech.sendium.core.storage.OutboundStorageException.Reason.INVALID_TRANSITION; +import static gr.cytech.sendium.core.storage.OutboundStorageException.Reason.UNAVAILABLE; + +/** + * One selected-router owner per pending-store lifetime. A shared lock makes memory selection and + * publication indivisible per source to other operations. Capacity includes queued and taken routing work; + * routed records remain retained until source cleanup but no longer occupy a routing slot. + */ +public final class MemorySelectedRouterStore implements SelectedRouterStore { + private final MemoryPendingMessageStore pending; + private final int capacity; + private final Clock clock; + private final Map> records = new HashMap<>(); + private final ArrayDeque> ready = new ArrayDeque<>(); + private State state = State.NEW; + private int routingSlots; + + public MemorySelectedRouterStore(MemoryPendingMessageStore pending, int capacity) { + this(pending, capacity, Clock.systemUTC()); + } + + public MemorySelectedRouterStore(MemoryPendingMessageStore pending, int capacity, Clock clock) { + if (capacity <= 0) { + throw new IllegalArgumentException("Router capacity must be positive"); + } + this.pending = Objects.requireNonNull(pending, "pending"); + this.capacity = capacity; + this.clock = Objects.requireNonNull(clock, "clock"); + } + + @Override + public void open() { + pending.lock.lock(); + try { + if (state == State.CLOSED) { + throw invalid("Closed memory selected-router store cannot reopen"); + } + pending.attachSelector(this); + state = State.READY; + } finally { + pending.lock.unlock(); + } + } + + @Override + public Status status() { + pending.lock.lock(); + try { + State effective = state == State.READY && pending.status().state() != State.READY ? State.FAILED : state; + return new Status("memory", false, effective); + } finally { + pending.lock.unlock(); + } + } + + @Override + public int selectAndStage(int limit) { + if (limit <= 0) { + throw new IllegalArgumentException("Selection limit must be positive"); + } + pending.lock.lock(); + try { + requireReady(); + int budget = Math.min(limit, capacity - routingSlots); + int staged = 0; + Instant now = clock.instant(); + while (staged < budget) { + MemoryPendingMessageStore.Entry source = pending.nextEligible(now); + if (source == null) { + break; + } + M message = pending.snapshot(source.message, Role.ROUTER_QUEUE); + Entry entry = new Entry<>(new SelectionId(UUID.randomUUID()), Set.of(source.source), message); + records.put(entry.id, entry); + ready.addLast(entry); + pending.removeWaiting(source); + routingSlots++; + staged++; + pending.changed.signalAll(); + } + return staged; + } finally { + pending.lock.unlock(); + } + } + + @Override + public Optional> take(Duration timeout) throws InterruptedException { + long remaining = timeoutNanos(timeout); + pending.lock.lockInterruptibly(); + try { + while (true) { + requireReady(); + Entry entry = ready.peekFirst(); + if (entry != null) { + M message = pending.snapshot(entry.message, Role.ROUTER_QUEUE); + Selected projection = new Selected<>(entry.id, entry.sources, message); + ready.removeFirst(); + entry.phase = Phase.TAKEN; + entry.inFlight = projection; + return Optional.of(projection); + } + if (remaining <= 0) { + return Optional.empty(); + } + remaining = pending.changed.awaitNanos(remaining); + } + } finally { + pending.lock.unlock(); + } + } + + /** Return the take result itself; it identifies the current in-process routing attempt. */ + @Override + public void release(Selected selected) { + Objects.requireNonNull(selected, "selected"); + pending.lock.lock(); + try { + requireReady(); + Entry entry = requireEntry(selected.id()); + if (entry.lastReturned == selected) { + return; + } + if (entry.phase != Phase.TAKEN || entry.inFlight != selected || !entry.sources.equals(selected.sources())) { + throw invalid("Only the current routing take can be returned"); + } + M snapshot = pending.snapshot(selected.message(), Role.ROUTER_QUEUE); + entry.message = snapshot; + entry.phase = Phase.QUEUED; + entry.inFlight = null; + entry.lastReturned = selected; + ready.addLast(entry); + pending.changed.signalAll(); + } finally { + pending.lock.unlock(); + } + } + + @Override + public void markRouted(SelectionId selection) { + Objects.requireNonNull(selection, "selection"); + pending.lock.lock(); + try { + requireReady(); + Entry entry = requireEntry(selection); + if (entry.phase == Phase.ROUTED) { + return; + } + if (entry.phase != Phase.TAKEN) { + throw invalid("Only taken routing work can be marked routed"); + } + entry.phase = Phase.ROUTED; + entry.inFlight = null; + entry.lastReturned = null; + routingSlots--; + } finally { + pending.lock.unlock(); + } + } + + @Override + public void complete(SelectionId selection) { + Objects.requireNonNull(selection, "selection"); + pending.lock.lock(); + try { + requireReady(); + Entry entry = records.get(selection); + if (entry == null) { + return; + } + if (entry.sources.stream().anyMatch(pending::contains)) { + throw invalid("Complete pending sources before removing selected ownership"); + } + records.remove(selection); + ready.remove(entry); + if (entry.phase != Phase.ROUTED) { + routingSlots--; + } + } finally { + pending.lock.unlock(); + } + } + + @Override + public void close() { + pending.lock.lock(); + try { + state = State.CLOSED; + records.clear(); + ready.clear(); + routingSlots = 0; + pending.changed.signalAll(); + } finally { + pending.lock.unlock(); + } + } + + private void requireReady() { + if (state != State.READY) { + throw new OutboundStorageException(Role.ROUTER_QUEUE, UNAVAILABLE, "Memory selected-router store is not ready"); + } + pending.requireReady(Role.ROUTER_QUEUE); + } + + private Entry requireEntry(SelectionId selection) { + Entry entry = records.get(selection); + if (entry == null) { + throw invalid("Unknown selection"); + } + return entry; + } + + private static OutboundStorageException invalid(String message) { + return new OutboundStorageException(Role.ROUTER_QUEUE, INVALID_TRANSITION, message); + } + + private static long timeoutNanos(Duration timeout) { + Objects.requireNonNull(timeout, "timeout"); + if (timeout.isNegative()) { + throw new IllegalArgumentException("Timeout must not be negative"); + } + try { + return timeout.toNanos(); + } catch (ArithmeticException overflow) { + return Long.MAX_VALUE; + } + } + + private enum Phase { + QUEUED, TAKEN, ROUTED + } + + private static final class Entry { + private final SelectionId id; + private final Set sources; + private M message; + private Phase phase = Phase.QUEUED; + private Selected inFlight; + private Selected lastReturned; + + private Entry(SelectionId id, Set sources, M message) { + this.id = id; + this.sources = sources; + this.message = message; + } + } +} diff --git a/sendium-core/src/test/java/gr/cytech/sendium/core/storage/memory/MemoryStorageTest.java b/sendium-core/src/test/java/gr/cytech/sendium/core/storage/memory/MemoryStorageTest.java new file mode 100644 index 00000000..71f5a016 --- /dev/null +++ b/sendium-core/src/test/java/gr/cytech/sendium/core/storage/memory/MemoryStorageTest.java @@ -0,0 +1,475 @@ +package gr.cytech.sendium.core.storage.memory; + +import gr.cytech.sendium.core.message.StandardMessage; +import gr.cytech.sendium.core.outbound.OutboundWork.Selected; +import gr.cytech.sendium.core.outbound.OutboundWork.SelectionId; +import gr.cytech.sendium.core.outbound.OutboundWork.SourceId; +import gr.cytech.sendium.core.storage.OutboundStage; +import gr.cytech.sendium.core.storage.OutboundStorageException; +import org.assertj.core.api.ThrowableAssert.ThrowingCallable; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.Timeout; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; + +import java.time.Clock; +import java.time.Duration; +import java.time.Instant; +import java.time.ZoneOffset; +import java.util.ArrayList; +import java.util.List; +import java.util.Set; +import java.util.UUID; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; +import java.util.function.UnaryOperator; + +import static gr.cytech.sendium.core.storage.OutboundStage.Role.PENDING; +import static gr.cytech.sendium.core.storage.OutboundStage.Role.ROUTER_QUEUE; +import static gr.cytech.sendium.core.storage.OutboundStage.State.CLOSED; +import static gr.cytech.sendium.core.storage.OutboundStage.State.NEW; +import static gr.cytech.sendium.core.storage.OutboundStage.State.READY; +import static gr.cytech.sendium.core.storage.OutboundStorageException.Reason.CAPACITY_EXCEEDED; +import static gr.cytech.sendium.core.storage.OutboundStorageException.Reason.INVALID_TRANSITION; +import static gr.cytech.sendium.core.storage.OutboundStorageException.Reason.OWNERSHIP_CONFLICT; +import static gr.cytech.sendium.core.storage.OutboundStorageException.Reason.UNAVAILABLE; +import static gr.cytech.sendium.core.storage.OutboundStorageException.Reason.UNSUPPORTED; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +@Timeout(10) +class MemoryStorageTest { + private static final Instant NOW = Instant.parse("2026-09-28T00:00:00Z"); + private static final Clock CLOCK = Clock.fixed(NOW, ZoneOffset.UTC); + + @Test + void lifecycleIsExplicitAndOnlyOneSelectorCanOwnPendingState() { + var pending = new MemoryPendingMessageStore(2, TestMessage::copy); + var router = new MemorySelectedRouterStore<>(pending, 1, CLOCK); + var other = new MemorySelectedRouterStore<>(pending, 1, CLOCK); + assertThat(pending.status()).isEqualTo(new OutboundStage.Status("memory", false, NEW)); + fails(router::open, ROUTER_QUEUE, UNAVAILABLE); + fails(() -> pending.admit(source(), message("one", 1)), PENDING, UNAVAILABLE); + pending.open(); + pending.open(); + router.open(); + router.open(); + assertThat(router.status()).isEqualTo(new OutboundStage.Status("memory", false, READY)); + fails(other::open, ROUTER_QUEUE, OWNERSHIP_CONFLICT); + router.close(); + router.close(); + fails(other::open, ROUTER_QUEUE, OWNERSHIP_CONFLICT); + fails(router::open, ROUTER_QUEUE, INVALID_TRANSITION); + pending.close(); + pending.close(); + fails(pending::open, PENDING, INVALID_TRANSITION); + fails(() -> pending.find(source()), PENDING, UNAVAILABLE); + fails(() -> router.take(Duration.ZERO), ROUTER_QUEUE, UNAVAILABLE); + assertThat(pending.status().state()).isEqualTo(CLOSED); + assertThat(router.status().state()).isEqualTo(CLOSED); + } + + @Test + void duplicateAdmissionDoesNotOverwriteAcceptedStateOrConsumeCapacity() { + try (var stores = new Stores(1, 1)) { + SourceId source = source(); + TestMessage input = message("accepted", 2); + input.tags.add("original"); + stores.pending.admit(source, input); + input.body = "changed"; + input.tags.clear(); + stores.pending.admit(source, message("duplicate", 3)); + TestMessage firstRead = stores.pending.find(source).orElseThrow(); + assertThat(firstRead.body).isEqualTo("accepted"); + assertThat(firstRead.tags).containsExactly("original"); + firstRead.tags.clear(); + assertThat(stores.pending.find(source).orElseThrow().tags).containsExactly("original"); + fails(() -> stores.pending.admit(source(), input), PENDING, CAPACITY_EXCEEDED); + } + } + + @Test + void selectionHonorsDueTimesAndPriorityWithoutFutureWorkBlockingReadyMessages() throws Exception { + try (var stores = new Stores(5, 5)) { + stores.pending.admit(source(), new TestMessage("future-high", 3, NOW.plusSeconds(10))); + stores.pending.admit(source(), message("low", 1)); + stores.pending.admit(source(), message("high-first", 3)); + stores.pending.admit(source(), message("normal", 2)); + stores.pending.admit(source(), message("high-second", 3)); + assertThat(stores.router.selectAndStage(5)).isEqualTo(4); + List bodies = new ArrayList<>(); + for (int i = 0; i < 4; i++) { + bodies.add(stores.router.take(Duration.ZERO).orElseThrow().message().body); + } + assertThat(bodies).containsExactly("high-first", "high-second", "normal", "low"); + assertThat(stores.router.selectAndStage(5)).isZero(); + } + } + + @Test + void advancingTheClockMakesDeferredWorkEligible() throws Exception { + AtomicReference time = new AtomicReference<>(NOW); + Clock clock = new Clock() { + public java.time.ZoneId getZone() { return ZoneOffset.UTC; } + public Clock withZone(java.time.ZoneId zone) { return this; } + public Instant instant() { return time.get(); } + }; + try (var pending = new MemoryPendingMessageStore(1, TestMessage::copy, msg -> msg.available); + var router = new MemorySelectedRouterStore<>(pending, 1, clock)) { + pending.open(); + router.open(); + TestMessage delayed = new TestMessage("delayed", 2, NOW.plusSeconds(1)); + pending.admit(source(), delayed); + delayed.available = NOW; + assertThat(router.selectAndStage(1)).isZero(); + time.set(NOW.plusSeconds(1)); + assertThat(router.selectAndStage(1)).isEqualTo(1); + assertThat(router.take(Duration.ZERO).orElseThrow().message().body).isEqualTo("delayed"); + } + } + + @Test + void refillCopiesOnlyItsBudgetAndReservesCapacityForTakenWork() throws Exception { + AtomicInteger copies = new AtomicInteger(); + try (var stores = new Stores(100, 2, msg -> { + copies.incrementAndGet(); + return msg.copy(); + })) { + for (int i = 0; i < 100; i++) { + stores.pending.admit(source(), message("item-" + i, 2)); + } + copies.set(0); + assertThat(stores.router.selectAndStage(1)).isEqualTo(1); + assertThat(copies).hasValue(1); + assertThat(stores.router.selectAndStage(50)).isEqualTo(1); + assertThat(copies).hasValue(2); + Selected first = stores.router.take(Duration.ZERO).orElseThrow(); + Selected second = stores.router.take(Duration.ZERO).orElseThrow(); + assertThat(stores.router.selectAndStage(50)).isZero(); + assertThat(copies).hasValue(4); + stores.router.release(first); + stores.router.release(first); + assertThat(stores.router.take(Duration.ZERO).orElseThrow().id()).isEqualTo(first.id()); + assertThat(stores.router.take(Duration.ZERO)).isEmpty(); + stores.router.markRouted(second.id()); + assertThat(stores.router.selectAndStage(50)).isEqualTo(1); + } + } + + @Test + void releasePreservesExecutionChangesWithoutMutatingPendingOrReactivatingAStaleTake() throws Exception { + try (var stores = new Stores(1, 1)) { + SourceId source = source(); + stores.pending.admit(source, message("accepted", 2)); + stores.router.selectAndStage(1); + Selected first = stores.router.take(Duration.ZERO).orElseThrow(); + first.message().body = "filtered"; + first.message().tags.add("filter-tag"); + stores.router.release(first); + first.message().body = "late mutation"; + first.message().tags.clear(); + Selected second = stores.router.take(Duration.ZERO).orElseThrow(); + assertThat(second.id()).isEqualTo(first.id()); + assertThat(second.message().body).isEqualTo("filtered"); + assertThat(second.message().tags).containsExactly("filter-tag"); + assertThat(stores.pending.find(source).orElseThrow().body).isEqualTo("accepted"); + assertThat(stores.pending.find(source).orElseThrow().tags).isEmpty(); + stores.router.release(first); + assertThat(stores.router.take(Duration.ZERO)).isEmpty(); + var forged = new Selected<>(second.id(), Set.of(source()), second.message()); + fails(() -> stores.router.release(forged), ROUTER_QUEUE, INVALID_TRANSITION); + stores.router.markRouted(second.id()); + fails(() -> stores.router.release(second), ROUTER_QUEUE, INVALID_TRANSITION); + } + } + + @Test + void routingDoesNotCompleteTheSourceAndCleanupIsRetrySafe() throws Exception { + try (var stores = new Stores(1, 1)) { + SourceId source = source(); + stores.pending.admit(source, message("one", 2)); + stores.router.selectAndStage(1); + Selected selected = stores.router.take(Duration.ZERO).orElseThrow(); + fails(() -> stores.router.complete(selected.id()), ROUTER_QUEUE, INVALID_TRANSITION); + stores.router.markRouted(selected.id()); + stores.router.markRouted(selected.id()); + assertThat(stores.router.selectAndStage(1)).isZero(); + assertThat(stores.pending.find(source)).isPresent(); + fails(() -> stores.pending.admit(source(), message("two", 2)), PENDING, CAPACITY_EXCEEDED); + stores.pending.complete(Set.of(source, source())); + stores.pending.complete(Set.of(source)); + stores.router.complete(selected.id()); + stores.router.complete(selected.id()); + assertThat(stores.pending.find(source)).isEmpty(); + stores.pending.admit(source(), message("two", 2)); + assertThat(stores.router.selectAndStage(1)).isEqualTo(1); + } + } + + @Test + void failedAdmissionDoesNotReserveCapacityAndIdentitySnapshotsAreRejected() { + AtomicBoolean fail = new AtomicBoolean(true); + try (var stores = new Stores(1, 1, msg -> { + if (fail.get()) { + throw new IllegalStateException("snapshot failure"); + } + return msg.copy(); + })) { + SourceId source = source(); + fails(() -> stores.pending.admit(source, message("failed", 2)), PENDING, UNAVAILABLE); + assertThat(stores.pending.find(source)).isEmpty(); + fail.set(false); + stores.pending.admit(source(), message("accepted", 2)); + } + try (var pending = new MemoryPendingMessageStore(1, UnaryOperator.identity())) { + pending.open(); + fails(() -> pending.admit(source(), message("unsafe", 2)), PENDING, UNSUPPORTED); + } + } + + @Test + void snapshotsCannotFlattenAnApplicationMessageSubtype() { + try (var pending = new MemoryPendingMessageStore(1, ignored -> new StandardMessage())) { + pending.open(); + SourceId source = source(); + fails(() -> pending.admit(source, message("custom payload", 2)), PENDING, UNSUPPORTED); + assertThat(pending.find(source)).isEmpty(); + pending.admit(source(), new StandardMessage()); + } + } + + @Test + void failureDuringBatchPublicationRetainsPublishedPrefixAndUnselectedSource() throws Exception { + AtomicBoolean fail = new AtomicBoolean(false); + try (var stores = new Stores(2, 2, msg -> { + if (fail.get() && msg.body.equals("second")) { + throw new IllegalStateException("snapshot failure"); + } + return msg.copy(); + })) { + SourceId first = source(); + SourceId second = source(); + stores.pending.admit(first, message("first", 2)); + stores.pending.admit(second, message("second", 2)); + fail.set(true); + fails(() -> stores.router.selectAndStage(2), ROUTER_QUEUE, UNAVAILABLE); + assertThat(stores.router.take(Duration.ZERO).orElseThrow().sources()).containsExactly(first); + fail.set(false); + assertThat(stores.pending.find(first)).isPresent(); + assertThat(stores.pending.find(second)).isPresent(); + assertThat(stores.router.selectAndStage(2)).isEqualTo(1); + assertThat(stores.router.take(Duration.ZERO).orElseThrow().sources()).containsExactly(second); + assertThat(stores.router.take(Duration.ZERO)).isEmpty(); + } + } + + @Test + void snapshotFailuresDuringTakeAndReturnLeaveOwnershipRetryable() throws Exception { + AtomicBoolean fail = new AtomicBoolean(false); + try (var stores = new Stores(1, 1, msg -> { + if (fail.get()) { + throw new IllegalStateException("snapshot failure"); + } + return msg.copy(); + })) { + stores.pending.admit(source(), message("one", 2)); + stores.router.selectAndStage(1); + fail.set(true); + fails(() -> stores.router.take(Duration.ZERO), ROUTER_QUEUE, UNAVAILABLE); + fail.set(false); + Selected selected = stores.router.take(Duration.ZERO).orElseThrow(); + selected.message().body = "updated"; + fail.set(true); + fails(() -> stores.router.release(selected), ROUTER_QUEUE, UNAVAILABLE); + fail.set(false); + stores.router.release(selected); + assertThat(stores.router.take(Duration.ZERO).orElseThrow().message().body).isEqualTo("updated"); + assertThat(stores.router.take(Duration.ZERO)).isEmpty(); + } + } + + @Test + void concurrentSelectorsAndTakersNeverSelectTheSameSourceTwice() throws Exception { + int count = 80; + try (var stores = new Stores(count, 8); var executor = Executors.newVirtualThreadPerTaskExecutor()) { + for (int i = 0; i < count; i++) { + stores.pending.admit(source(), message("item-" + i, i % 4)); + } + Set seen = ConcurrentHashMap.newKeySet(); + Set selections = ConcurrentHashMap.newKeySet(); + var jobs = new ArrayList>(); + for (int worker = 0; worker < 4; worker++) { + jobs.add(executor.submit(() -> { + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(5); + while (seen.size() < count && System.nanoTime() < deadline) { + stores.router.selectAndStage(3); + var selected = stores.router.take(Duration.ZERO); + if (selected.isPresent()) { + var work = selected.orElseThrow(); + assertThat(seen.add(work.sources().iterator().next())).isTrue(); + assertThat(selections.add(work.id())).isTrue(); + stores.router.markRouted(work.id()); + } + } + return null; + })); + } + for (var job : jobs) { + job.get(6, TimeUnit.SECONDS); + } + assertThat(seen).hasSize(count); + assertThat(stores.router.selectAndStage(count)).isZero(); + for (SourceId source : seen) { + assertThat(stores.pending.find(source)).isPresent(); + } + stores.pending.complete(seen); + selections.forEach(stores.router::complete); + } + } + + @Test + void aWaitingTakeWakesWhenWorkIsStaged() throws Exception { + try (var stores = new Stores(1, 1); var executor = Executors.newVirtualThreadPerTaskExecutor()) { + AtomicReference waiter = new AtomicReference<>(); + var result = executor.submit(() -> { + waiter.set(Thread.currentThread()); + return stores.router.take(Duration.ofSeconds(2)); + }); + awaitWaiting(waiter); + stores.pending.admit(source(), message("one", 2)); + stores.router.selectAndStage(1); + assertThat(result.get(1, TimeUnit.SECONDS).orElseThrow().message().body).isEqualTo("one"); + } + } + + @ParameterizedTest + @ValueSource(booleans = {true, false}) + void closingEitherStoreWakesWaitingTakes(boolean closePending) throws Exception { + try (var stores = new Stores(1, 1); var executor = Executors.newVirtualThreadPerTaskExecutor()) { + AtomicReference waiter = new AtomicReference<>(); + var result = executor.submit(() -> { + waiter.set(Thread.currentThread()); + return stores.router.take(Duration.ofSeconds(2)); + }); + awaitWaiting(waiter); + if (closePending) { + stores.pending.close(); + assertThat(stores.router.status().state()).isEqualTo(OutboundStage.State.FAILED); + } else { + stores.router.close(); + } + assertThatThrownBy(() -> result.get(1, TimeUnit.SECONDS)) + .hasCauseInstanceOf(OutboundStorageException.class); + } + } + + @Test + void takeIsInterruptibleAndFreshInstancesRecoverNothing() throws Exception { + SourceId source = source(); + try (var stores = new Stores(1, 1)) { + AtomicReference failure = new AtomicReference<>(); + CountDownLatch started = new CountDownLatch(1); + Thread taker = Thread.ofVirtual().start(() -> { + started.countDown(); + try { + stores.router.take(Duration.ofSeconds(2)); + } catch (Throwable caught) { + failure.set(caught); + } + }); + assertThat(started.await(1, TimeUnit.SECONDS)).isTrue(); + taker.interrupt(); + taker.join(3000); + assertThat(taker.isAlive()).isFalse(); + assertThat(failure.get()).isInstanceOf(InterruptedException.class); + stores.pending.admit(source, message("will be lost", 2)); + stores.router.selectAndStage(1); + } + try (var fresh = new Stores(1, 1)) { + assertThat(fresh.pending.find(source)).isEmpty(); + assertThat(fresh.router.selectAndStage(1)).isZero(); + assertThat(fresh.router.take(Duration.ZERO)).isEmpty(); + } + } + + @Test + void invalidLimitsAndTimeoutsDoNotConsumeWork() throws Exception { + try (var stores = new Stores(1, 1)) { + stores.pending.admit(source(), message("one", 2)); + assertThatThrownBy(() -> stores.router.selectAndStage(0)).isInstanceOf(IllegalArgumentException.class); + assertThatThrownBy(() -> stores.router.take(Duration.ofNanos(-1))).isInstanceOf(IllegalArgumentException.class); + assertThat(stores.router.selectAndStage(1)).isEqualTo(1); + assertThat(stores.router.take(Duration.ofSeconds(Long.MAX_VALUE))).isPresent(); + assertThat(stores.router.take(Duration.ofMillis(1))).isEmpty(); + } + } + + private static void awaitWaiting(AtomicReference waiter) throws InterruptedException { + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(1); + while ((waiter.get() == null || waiter.get().getState() != Thread.State.TIMED_WAITING) && + System.nanoTime() < deadline) { + Thread.sleep(1); + } + assertThat(waiter.get()).isNotNull(); + assertThat(waiter.get().getState()).isEqualTo(Thread.State.TIMED_WAITING); + } + + private static SourceId source() { + return new SourceId(UUID.randomUUID()); + } + + private static TestMessage message(String body, int priority) { + return new TestMessage(body, priority, NOW); + } + + private static void fails(ThrowingCallable operation, OutboundStage.Role role, OutboundStorageException.Reason reason) { + assertThatThrownBy(operation).isInstanceOfSatisfying(OutboundStorageException.class, failure -> { + assertThat(failure.stage()).isEqualTo(role); + assertThat(failure.reason()).isEqualTo(reason); + }); + } + + private static final class Stores implements AutoCloseable { + private final MemoryPendingMessageStore pending; + private final MemorySelectedRouterStore router; + + private Stores(int pendingCapacity, int routerCapacity) { + this(pendingCapacity, routerCapacity, TestMessage::copy); + } + + private Stores(int pendingCapacity, int routerCapacity, UnaryOperator snapshots) { + pending = new MemoryPendingMessageStore<>(pendingCapacity, snapshots, msg -> msg.available); + router = new MemorySelectedRouterStore<>(pending, routerCapacity, CLOCK); + pending.open(); + router.open(); + } + + @Override + public void close() { + router.close(); + pending.close(); + } + } + + private static final class TestMessage extends StandardMessage { + private Instant available; + private final List tags = new ArrayList<>(); + + private TestMessage(String body, int priority, Instant available) { + this.body = body; + this.priority = priority; + this.available = available; + } + + private TestMessage copy() { + var copy = new TestMessage(body, priority, available); + copy.tags.addAll(tags); + return copy; + } + } +} From 29147d7dfb64fdf5ffa413ddd1a57ae5badb6a7d Mon Sep 17 00:00:00 2001 From: pavlos Date: Mon, 28 Sep 2026 15:27:59 +0300 Subject: [PATCH 04/20] feat(storage): add memory routed-work ownership Task 2.1a - Implement single-destination memory routed-work storage. Track destination takes, retry-safe forwarding, and terminal cleanup while retaining source ownership and rejecting conflicting assignments. Refs #338 Signed-off-by: pavlos --- docs/09-configuration-reference.md | 2 +- docs/14-outbound-storage.md | 63 ++- .../sendium/core/storage/RoutedWorkStore.java | 5 +- .../storage/memory/MemoryRoutedWorkStore.java | 370 ++++++++++++++++ .../memory/MemoryRoutedWorkStoreTest.java | 414 ++++++++++++++++++ 5 files changed, 848 insertions(+), 6 deletions(-) create mode 100644 sendium-core/src/main/java/gr/cytech/sendium/core/storage/memory/MemoryRoutedWorkStore.java create mode 100644 sendium-core/src/test/java/gr/cytech/sendium/core/storage/memory/MemoryRoutedWorkStoreTest.java diff --git a/docs/09-configuration-reference.md b/docs/09-configuration-reference.md index 53d13d41..0d34075b 100644 --- a/docs/09-configuration-reference.md +++ b/docs/09-configuration-reference.md @@ -64,7 +64,7 @@ fallback to a less durable backend. Validation runs before Sendium's file-watcher and router/worker startup observers. The validated profile is retained for the process lifetime. Startup logs the effective profile and a prominent `NON-DURABLE` warning: accepted messages and in-flight work can be lost on restart. The current -runtime pipeline is still memory-backed. Pending and selected-router memory stores exist as library +runtime pipeline is still memory-backed. All three memory stage stores exist as library components; end-to-end coordinator and ingress/worker wiring are subsequent work in #338. Their capacity bounds are constructor arguments at this stage, not additional operator settings. See [Outbound Storage Contracts](14-outbound-storage.md). diff --git a/docs/14-outbound-storage.md b/docs/14-outbound-storage.md index 4acfb36a..d8cca4f3 100644 --- a/docs/14-outbound-storage.md +++ b/docs/14-outbound-storage.md @@ -2,10 +2,9 @@ This describes the initial contracts for [#338](https://github.com/cytechmobile/sendium/issues/338), under [#337](https://github.com/cytechmobile/sendium/issues/337). The stage contracts are library APIs, -and standalone profile selection, early validation, and startup logging are implemented. Memory pending -and selected-router stores are available as explicitly constructed library components. The coordinator, -routed-work store, and end-to-end production wiring are subsequent tasks. Existing ingress and queue -behavior remains non-durable. +and standalone profile selection, early validation, and startup logging are implemented. All three +memory stage stores are available as explicitly constructed library components. The coordinator and +end-to-end production wiring are subsequent tasks. Existing ingress and queue behavior remains non-durable. ## Ownership model @@ -65,6 +64,7 @@ storage/ memory/ MemoryPendingMessageStore.java MemorySelectedRouterStore.java + MemoryRoutedWorkStore.java ``` Memory implementations live under `storage.memory`. Standalone configuration @@ -191,6 +191,61 @@ All state is instance-local. New instances start empty and provide no restart re admission and active routing, not message byte size; required terminal bookkeeping is removed by coordinated cleanup in the later integration tasks. +## Memory routed-work implementation + +`MemoryRoutedWorkStore` is independently constructed with a positive selection capacity and a +`UnaryOperator` snapshot function, with the same isolation/subtype requirements as the pending +memory backend. It has its own lock and does not depend on a concrete pending or selected store. +Coordination across stores belongs to the upcoming coordinator. + +```text +record assignment → QUEUED → take → TAKEN + ↑ | + +-- release --+ + | + +--------------------+-------------------+ + | | + transfer complete + | | + previous: TRANSFERRED COMPLETED + successor: QUEUED | + (new ID, same selection) repeatable cleanup result + | + coordinator finishes cleanup + | + forget +``` + +- `record` retains one assignment per selection. Matching retries return its original work ID and + original assignment payload, even after forwarding or completion, without re-enqueueing it. A + changed source set/destination is invalid. A source cannot belong to two retained routed selections. +- `record` and `transfer` results describe recorded work; they do not claim execution. Only `take` + claims a work item. Each destination schedules in publication order. Take and return preserve one + active taker and updated execution state; return the actual take result as its local attempt handle. +- `transfer` requires taken work and atomically moves responsibility to one successor under the same + selection. It works even at full selection capacity because it reuses that selection's slot. Retrying + the predecessor returns its original immediate successor, even if that successor has moved onward. + A conflicting retry destination is invalid. Old work is never scheduled again. +- `complete` requires taken work after the caller has established terminal processing and required + handoff. Repeated completion returns the same source set. Completion for any transferred predecessor + returns empty, including after its successor completes, so late callbacks cannot release sources early. +- `forget` rejects nonterminal selections. After the coordinator completes pending/selected cleanup, + it removes that selection's entire forwarding history and source claims, freeing capacity. Repeated + forget is harmless. Unknown work IDs fail explicitly. Replay guarantees apply while records are + retained; the coordinator must reject stale route/transfer requests and absorb duplicate terminal + callbacks after cleanup rather than recreating old work. + +Capacity counts retained selections, including taken work and terminal selections awaiting cleanup. +Return and forwarding cannot create additional active selections or exhaust a separate return slot. +Forwarding history and original replay payloads are retained once per hop until forget; the selection +count is not a bound on history length or message bytes. The later worker integration owns retry and +forwarding policy. + +All required snapshots, including the returned result, are prepared before record/transfer publication. +If a snapshot fails, no partial assignment or successor is published, and the current work remains +retryable. Takes are interruptible and wake on publication or close. Closure clears instance-local state +without manufacturing completion; closed instances cannot reopen and fresh instances recover nothing. + ## Completion and retry boundaries - Admission retries reuse the same source ID and do not overwrite the accepted state while the diff --git a/sendium-core/src/main/java/gr/cytech/sendium/core/storage/RoutedWorkStore.java b/sendium-core/src/main/java/gr/cytech/sendium/core/storage/RoutedWorkStore.java index cd16e4a7..e501b70e 100644 --- a/sendium-core/src/main/java/gr/cytech/sendium/core/storage/RoutedWorkStore.java +++ b/sendium-core/src/main/java/gr/cytech/sendium/core/storage/RoutedWorkStore.java @@ -15,11 +15,12 @@ public interface RoutedWorkStore extends OutboundStag /** * Records the execution payload and one destination consistently before its work may be taken. * Future recovery resumes this recorded work at its worker, not through routing again. - * Repeating a selection returns its original assignment and work ID without reactivating + * While retained, repeating a selection returns its original assignment and work ID without reactivating * completed/transferred work or overwriting content. * Callers retry with the same assignment; changing an active destination requires transfer, not * another assignment. There is at most one active destination work item per selection. * Conflicting source identities or destinations for an existing selection fail as INVALID_TRANSITION. + * After forget, the coordinator must reject stale recording attempts using its lifecycle ownership. */ Routed record(Assignment assignment); @@ -27,6 +28,7 @@ public interface RoutedWorkStore extends OutboundStag * Takes one work projection without completing it or silently changing earlier-stage records. * Zero timeout polls; negative timeouts are invalid. Destination must be nonblank. * A work item has at most one active taker. + * Retain the returned Routed value as the local attempt handle when returning unfinished work. */ Optional> take(String destination, Duration timeout) throws InterruptedException; @@ -35,6 +37,7 @@ public interface RoutedWorkStore extends OutboundStag * in-flight work. Identity, selection, and destination must match the take. Repeated release cannot * duplicate work or reactivate completed/transferred work. Earlier-stage state cannot silently change; * preserving runtime mutations does not promise durable retry counters or timing. + * Pass the take result itself with its message updated, not a reconstructed projection. */ void release(Routed work); diff --git a/sendium-core/src/main/java/gr/cytech/sendium/core/storage/memory/MemoryRoutedWorkStore.java b/sendium-core/src/main/java/gr/cytech/sendium/core/storage/memory/MemoryRoutedWorkStore.java new file mode 100644 index 00000000..0cbc2955 --- /dev/null +++ b/sendium-core/src/main/java/gr/cytech/sendium/core/storage/memory/MemoryRoutedWorkStore.java @@ -0,0 +1,370 @@ +package gr.cytech.sendium.core.storage.memory; + +import gr.cytech.sendium.core.message.StandardMessage; +import gr.cytech.sendium.core.outbound.OutboundWork.Assignment; +import gr.cytech.sendium.core.outbound.OutboundWork.Completed; +import gr.cytech.sendium.core.outbound.OutboundWork.Destination; +import gr.cytech.sendium.core.outbound.OutboundWork.Routed; +import gr.cytech.sendium.core.outbound.OutboundWork.SelectionId; +import gr.cytech.sendium.core.outbound.OutboundWork.SourceId; +import gr.cytech.sendium.core.outbound.OutboundWork.WorkId; +import gr.cytech.sendium.core.storage.OutboundStorageException; +import gr.cytech.sendium.core.storage.RoutedWorkStore; + +import java.time.Duration; +import java.util.ArrayDeque; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import java.util.Set; +import java.util.UUID; +import java.util.concurrent.locks.Condition; +import java.util.concurrent.locks.ReentrantLock; +import java.util.function.UnaryOperator; + +import static gr.cytech.sendium.core.storage.OutboundStorageException.Reason.CAPACITY_EXCEEDED; +import static gr.cytech.sendium.core.storage.OutboundStorageException.Reason.INVALID_TRANSITION; +import static gr.cytech.sendium.core.storage.OutboundStorageException.Reason.UNAVAILABLE; +import static gr.cytech.sendium.core.storage.OutboundStorageException.Reason.UNSUPPORTED; + +/** + * Single-destination execution with instance-local forwarding history. Capacity counts retained + * selections, including terminal selections awaiting forget. Snapshots must detach mutable fields, + * preserve subtype/data, and be nonblocking, side-effect-free, and non-reentrant. + */ +public final class MemoryRoutedWorkStore implements RoutedWorkStore { + private final ReentrantLock lock = new ReentrantLock(); + private final Condition changed = lock.newCondition(); + private final int capacity; + private final UnaryOperator snapshots; + private final Map> selections = new HashMap<>(); + private final Map sourceOwners = new HashMap<>(); + private final Map> work = new HashMap<>(); + private final Map>> ready = new HashMap<>(); + private State state = State.NEW; + + public MemoryRoutedWorkStore(int capacity, UnaryOperator snapshots) { + if (capacity <= 0) { + throw new IllegalArgumentException("Routed selection capacity must be positive"); + } + this.capacity = capacity; + this.snapshots = Objects.requireNonNull(snapshots, "snapshots"); + } + + @Override + public void open() { + lock.lock(); + try { + if (state == State.CLOSED) { + throw invalid("Closed memory routed-work store cannot reopen"); + } + state = State.READY; + } finally { + lock.unlock(); + } + } + + @Override + public Status status() { + lock.lock(); + try { + return new Status("memory", false, state); + } finally { + lock.unlock(); + } + } + + @Override + public Routed record(Assignment assignment) { + Objects.requireNonNull(assignment, "assignment"); + lock.lock(); + try { + requireReady(); + Selection existing = selections.get(assignment.selection()); + if (existing != null) { + if (!existing.sources.equals(assignment.sources()) || + !existing.initial.destination.equals(assignment.destination().name())) { + throw invalid("An existing selection cannot change sources or its initial destination"); + } + return projection(existing.initial, existing.initial.original); + } + if (assignment.sources().stream().anyMatch(sourceOwners::containsKey)) { + throw invalid("A source already belongs to another routed selection"); + } + if (selections.size() >= capacity) { + throw new OutboundStorageException(Role.ROUTED_WORK, CAPACITY_EXCEEDED, "Memory routed selection capacity reached"); + } + Selection selection = new Selection<>(assignment.selection(), assignment.sources()); + Entry entry = entry(selection, assignment.destination()); + final Routed result = projection(entry, entry.original); + selection.initial = entry; + selections.put(selection.id, selection); + selection.sources.forEach(source -> sourceOwners.put(source, selection.id)); + publish(entry); + return result; + } finally { + lock.unlock(); + } + } + + @Override + public Optional> take(String destination, Duration timeout) throws InterruptedException { + requireDestination(destination); + long remaining = timeoutNanos(timeout); + lock.lockInterruptibly(); + try { + while (true) { + requireReady(); + ArrayDeque> queue = ready.get(destination); + if (queue != null) { + Entry entry = queue.peekFirst(); + final Routed result = projection(entry, entry.message); + queue.removeFirst(); + if (queue.isEmpty()) { + ready.remove(destination); + } + entry.phase = Phase.TAKEN; + entry.inFlight = result; + return Optional.of(result); + } + if (remaining <= 0) { + return Optional.empty(); + } + remaining = changed.awaitNanos(remaining); + } + } finally { + lock.unlock(); + } + } + + @Override + public void release(Routed taken) { + Objects.requireNonNull(taken, "taken"); + lock.lock(); + try { + requireReady(); + Entry entry = requireEntry(taken.id()); + if (entry.lastReturned == taken) { + return; + } + if (entry.phase != Phase.TAKEN || entry.inFlight != taken) { + throw invalid("Only the current destination take can be returned"); + } + M message = snapshot(taken.message()); + entry.message = message; + entry.phase = Phase.QUEUED; + entry.inFlight = null; + entry.lastReturned = taken; + enqueue(entry); + } finally { + lock.unlock(); + } + } + + @Override + public Routed transfer(WorkId previous, Destination destination) { + Objects.requireNonNull(previous, "previous"); + Objects.requireNonNull(destination, "destination"); + lock.lock(); + try { + requireReady(); + Entry entry = requireEntry(previous); + if (entry.phase == Phase.TRANSFERRED) { + Entry successor = requireEntry(entry.successor); + if (!successor.destination.equals(destination.name())) { + throw invalid("A transfer retry cannot change the successor destination"); + } + return projection(successor, successor.original); + } + if (entry.phase != Phase.TAKEN) { + throw invalid("Only taken destination work can be transferred"); + } + Entry successor = entry(entry.selection, destination); + final Routed result = projection(successor, successor.original); + entry.phase = Phase.TRANSFERRED; + entry.successor = successor.id; + clearExecution(entry); + publish(successor); + return result; + } finally { + lock.unlock(); + } + } + + @Override + public Optional complete(WorkId id) { + Objects.requireNonNull(id, "id"); + lock.lock(); + try { + requireReady(); + Entry entry = requireEntry(id); + if (entry.phase == Phase.TRANSFERRED) { + return Optional.empty(); + } + if (entry.phase == Phase.COMPLETED) { + return Optional.of(entry.selection.completed); + } + if (entry.phase != Phase.TAKEN) { + throw invalid("Only taken destination work can complete"); + } + entry.selection.completed = new Completed(entry.selection.id, entry.selection.sources); + entry.phase = Phase.COMPLETED; + clearExecution(entry); + return Optional.of(entry.selection.completed); + } finally { + lock.unlock(); + } + } + + @Override + public void forget(SelectionId id) { + Objects.requireNonNull(id, "id"); + lock.lock(); + try { + requireReady(); + Selection selection = selections.get(id); + if (selection == null) { + return; + } + if (selection.completed == null) { + throw invalid("Nonterminal routed work cannot be forgotten"); + } + selection.history.forEach(work::remove); + selection.sources.forEach(sourceOwners::remove); + selections.remove(id); + } finally { + lock.unlock(); + } + } + + @Override + public void close() { + lock.lock(); + try { + state = State.CLOSED; + selections.clear(); + sourceOwners.clear(); + work.clear(); + ready.clear(); + changed.signalAll(); + } finally { + lock.unlock(); + } + } + + private Entry entry(Selection selection, Destination destination) { + return new Entry<>(new WorkId(UUID.randomUUID()), selection, destination.name(), snapshot(destination.message())); + } + + private void publish(Entry entry) { + work.put(entry.id, entry); + entry.selection.history.add(entry.id); + enqueue(entry); + } + + private void enqueue(Entry entry) { + ready.computeIfAbsent(entry.destination, ignored -> new ArrayDeque<>()).addLast(entry); + changed.signalAll(); + } + + private Routed projection(Entry entry, M message) { + return new Routed<>(entry.id, entry.selection.id, entry.destination, snapshot(message)); + } + + private M snapshot(M message) { + M copy; + try { + copy = snapshots.apply(message); + } catch (RuntimeException failure) { + throw new OutboundStorageException(Role.ROUTED_WORK, UNAVAILABLE, "Cannot snapshot routed memory work", failure); + } + if (copy == null || copy == message || copy.getClass() != message.getClass()) { + throw new OutboundStorageException(Role.ROUTED_WORK, UNSUPPORTED, + "Memory snapshots must be independent and preserve the concrete message type"); + } + return copy; + } + + private void clearExecution(Entry entry) { + entry.message = null; + entry.inFlight = null; + entry.lastReturned = null; + } + + private void requireReady() { + if (state != State.READY) { + throw new OutboundStorageException(Role.ROUTED_WORK, UNAVAILABLE, "Memory routed-work store is not ready"); + } + } + + private Entry requireEntry(WorkId id) { + Entry entry = work.get(id); + if (entry == null) { + throw invalid("Unknown routed work"); + } + return entry; + } + + private static OutboundStorageException invalid(String message) { + return new OutboundStorageException(Role.ROUTED_WORK, INVALID_TRANSITION, message); + } + + private static void requireDestination(String destination) { + Objects.requireNonNull(destination, "destination"); + if (destination.isBlank()) { + throw new IllegalArgumentException("Destination must not be blank"); + } + } + + private static long timeoutNanos(Duration timeout) { + Objects.requireNonNull(timeout, "timeout"); + if (timeout.isNegative()) { + throw new IllegalArgumentException("Timeout must not be negative"); + } + try { + return timeout.toNanos(); + } catch (ArithmeticException overflow) { + return Long.MAX_VALUE; + } + } + + private enum Phase { + QUEUED, TAKEN, TRANSFERRED, COMPLETED + } + + private static final class Selection { + private final SelectionId id; + private final Set sources; + private final List history = new ArrayList<>(); + private Entry initial; + private Completed completed; + + private Selection(SelectionId id, Set sources) { + this.id = id; + this.sources = sources; + } + } + + private static final class Entry { + private final WorkId id; + private final Selection selection; + private final String destination; + private final M original; + private M message; + private Phase phase = Phase.QUEUED; + private WorkId successor; + private Routed inFlight; + private Routed lastReturned; + + private Entry(WorkId id, Selection selection, String destination, M original) { + this.id = id; + this.selection = selection; + this.destination = destination; + this.original = original; + this.message = original; + } + } +} diff --git a/sendium-core/src/test/java/gr/cytech/sendium/core/storage/memory/MemoryRoutedWorkStoreTest.java b/sendium-core/src/test/java/gr/cytech/sendium/core/storage/memory/MemoryRoutedWorkStoreTest.java new file mode 100644 index 00000000..187f8147 --- /dev/null +++ b/sendium-core/src/test/java/gr/cytech/sendium/core/storage/memory/MemoryRoutedWorkStoreTest.java @@ -0,0 +1,414 @@ +package gr.cytech.sendium.core.storage.memory; + +import gr.cytech.sendium.core.message.StandardMessage; +import gr.cytech.sendium.core.outbound.OutboundWork.Assignment; +import gr.cytech.sendium.core.outbound.OutboundWork.Destination; +import gr.cytech.sendium.core.outbound.OutboundWork.Routed; +import gr.cytech.sendium.core.outbound.OutboundWork.SelectionId; +import gr.cytech.sendium.core.outbound.OutboundWork.SourceId; +import gr.cytech.sendium.core.outbound.OutboundWork.WorkId; +import gr.cytech.sendium.core.storage.OutboundStage; +import gr.cytech.sendium.core.storage.OutboundStorageException; +import org.assertj.core.api.ThrowableAssert.ThrowingCallable; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.Timeout; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; + +import java.time.Duration; +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; +import java.util.Set; +import java.util.UUID; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; +import java.util.function.UnaryOperator; + +import static gr.cytech.sendium.core.storage.OutboundStorageException.Reason.CAPACITY_EXCEEDED; +import static gr.cytech.sendium.core.storage.OutboundStorageException.Reason.INVALID_TRANSITION; +import static gr.cytech.sendium.core.storage.OutboundStorageException.Reason.UNAVAILABLE; +import static gr.cytech.sendium.core.storage.OutboundStorageException.Reason.UNSUPPORTED; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +@Timeout(10) +class MemoryRoutedWorkStoreTest { + @Test + void recordRetriesKeepOriginalPayloadAndOneScheduledWorkItem() throws Exception { + try (var store = store(1)) { + var assignment = assignment("A", "original"); + assignment.destination().message().tags.add("retained"); + Routed recorded = store.record(assignment); + assignment.destination().message().body = "changed input"; + assignment.destination().message().tags.clear(); + recorded.message().body = "changed result"; + var replay = store.record(new Assignment<>(assignment.selection(), assignment.sources(), destination("A", "retry"))); + assertThat(replay.id()).isEqualTo(recorded.id()); + assertThat(replay.message().body).isEqualTo("original"); + assertThat(replay.message().tags).containsExactly("retained"); + replay.message().tags.clear(); + fails(() -> store.release(recorded), INVALID_TRANSITION); + var taken = store.take("A", Duration.ZERO).orElseThrow(); + assertThat(taken.message().tags).containsExactly("retained"); + assertThat(store.take("A", Duration.ZERO)).isEmpty(); + fails(() -> store.record(assignment("B", "full")), CAPACITY_EXCEEDED); + fails(() -> store.forget(assignment.selection()), INVALID_TRANSITION); + } + } + + @Test + void conflictingAssignmentsCannotCreateCopiedDestinationsOrOverlapSources() throws Exception { + try (var store = store(3)) { + var assignment = assignment("A", "original"); + store.record(assignment); + fails(() -> store.record(new Assignment<>(assignment.selection(), assignment.sources(), destination("B", "copy"))), + INVALID_TRANSITION); + fails(() -> store.record(new Assignment<>(assignment.selection(), Set.of(source()), destination("A", "replacement"))), + INVALID_TRANSITION); + fails(() -> store.record(new Assignment<>(selection(), assignment.sources(), destination("B", "copy"))), + INVALID_TRANSITION); + assertThat(store.take("B", Duration.ZERO)).isEmpty(); + assertThat(store.take("A", Duration.ZERO)).isPresent(); + assertThat(store.take("A", Duration.ZERO)).isEmpty(); + } + } + + @Test + void returnPreservesExecutionStateWithoutChangingAssignmentOrOverwritingANewerTake() throws Exception { + try (var store = store(1)) { + var assignment = assignment("A", "original"); + store.record(assignment); + var first = store.take("A", Duration.ZERO).orElseThrow(); + first.message().body = "retry state"; + first.message().tags.add("retry tag"); + store.release(first); + store.release(first); + first.message().tags.clear(); + var second = store.take("A", Duration.ZERO).orElseThrow(); + assertThat(second.id()).isEqualTo(first.id()); + assertThat(second.message().body).isEqualTo("retry state"); + assertThat(second.message().tags).containsExactly("retry tag"); + assertThat(store.record(assignment).message().body).isEqualTo("original"); + store.release(first); + assertThat(store.take("A", Duration.ZERO)).isEmpty(); + fails(() -> store.release(new Routed<>(second.id(), second.selection(), "B", second.message())), INVALID_TRANSITION); + store.complete(second.id()); + fails(() -> store.release(second), INVALID_TRANSITION); + } + } + + @Test + void forwardingTransfersTheSlotAndLateCallbacksCannotCompleteTheSuccessor() throws Exception { + try (var store = store(1)) { + var assignment = assignment("A", "original"); + var original = store.record(assignment); + fails(() -> store.transfer(original.id(), destination("B", "premature")), INVALID_TRANSITION); + fails(() -> store.complete(original.id()), INVALID_TRANSITION); + var previous = store.take("A", Duration.ZERO).orElseThrow(); + var next = store.transfer(previous.id(), destination("B", "forwarded")); + assertThat(next.id()).isNotEqualTo(previous.id()); + assertThat(next.selection()).isEqualTo(previous.selection()); + assertThat(store.complete(previous.id())).isEmpty(); + fails(() -> store.release(previous), INVALID_TRANSITION); + fails(() -> store.forget(assignment.selection()), INVALID_TRANSITION); + var replay = store.transfer(previous.id(), destination("B", "different retry payload")); + assertThat(replay.id()).isEqualTo(next.id()); + assertThat(replay.message().body).isEqualTo("forwarded"); + fails(() -> store.transfer(previous.id(), destination("C", "conflict")), INVALID_TRANSITION); + assertThat(store.record(assignment).id()).isEqualTo(original.id()); + assertThat(store.take("A", Duration.ZERO)).isEmpty(); + assertThat(store.take("B", Duration.ZERO).orElseThrow().id()).isEqualTo(next.id()); + assertThat(store.take("B", Duration.ZERO)).isEmpty(); + var completed = store.complete(next.id()).orElseThrow(); + assertThat(completed.sources()).isEqualTo(assignment.sources()); + assertThat(store.complete(previous.id())).isEmpty(); + assertThat(store.complete(next.id())).contains(completed); + store.record(assignment); + store.transfer(previous.id(), destination("B", "late retry")); + assertThat(store.take("A", Duration.ZERO)).isEmpty(); + assertThat(store.take("B", Duration.ZERO)).isEmpty(); + } + } + + @Test + void cleanupRetainsAllSourceIdsAndHistoryUntilForget() throws Exception { + try (var store = store(1)) { + var sources = Set.of(source(), source()); + var assignment = new Assignment<>(selection(), sources, destination("A", "multipart aggregate")); + var first = store.record(assignment); + store.take("A", Duration.ZERO).orElseThrow(); + var second = store.transfer(first.id(), destination("B", "second")); + store.take("B", Duration.ZERO).orElseThrow(); + var third = store.transfer(second.id(), destination("A", "third")); + assertThat(store.transfer(first.id(), destination("B", "retry")).id()).isEqualTo(second.id()); + assertThat(store.take("B", Duration.ZERO)).isEmpty(); + assertThat(store.take("A", Duration.ZERO).orElseThrow().id()).isEqualTo(third.id()); + var completed = store.complete(third.id()).orElseThrow(); + assertThat(completed.sources()).isEqualTo(sources); + assertThat(store.complete(first.id())).isEmpty(); + assertThat(store.complete(second.id())).isEmpty(); + fails(() -> store.record(assignment("B", "capacity still held")), CAPACITY_EXCEEDED); + assertThat(store.complete(third.id())).contains(completed); + store.forget(assignment.selection()); + store.forget(assignment.selection()); + fails(() -> store.complete(first.id()), INVALID_TRANSITION); + fails(() -> store.complete(third.id()), INVALID_TRANSITION); + var replacement = new Assignment<>(selection(), sources, destination("A", "new selection")); + store.record(replacement); + assertThat(store.take("A", Duration.ZERO).orElseThrow().selection()).isEqualTo(replacement.selection()); + } + } + + @Test + void queuesAreDestinationSpecificAndPreservePublicationOrder() throws Exception { + try (var store = store(3)) { + store.record(assignment("A", "first")); + store.record(assignment("B", "other")); + store.record(assignment("A", "second")); + assertThat(store.take("missing", Duration.ZERO)).isEmpty(); + assertThat(store.take("A", Duration.ZERO).orElseThrow().message().body).isEqualTo("first"); + assertThat(store.take("B", Duration.ZERO).orElseThrow().message().body).isEqualTo("other"); + assertThat(store.take("A", Duration.ZERO).orElseThrow().message().body).isEqualTo("second"); + } + } + + @ParameterizedTest + @ValueSource(ints = {1, 2}) + void failedRecordSnapshotsPublishNothingAndLeaveCapacityAvailable(int failAt) throws Exception { + AtomicInteger calls = new AtomicInteger(); + try (var store = new MemoryRoutedWorkStore(1, msg -> { + if (calls.incrementAndGet() == failAt) { + throw new IllegalStateException("snapshot failure"); + } + return msg.copy(); + })) { + store.open(); + var assignment = assignment("A", "payload"); + fails(() -> store.record(assignment), UNAVAILABLE); + assertThat(store.take("A", Duration.ZERO)).isEmpty(); + var work = store.record(assignment); + assertThat(store.take("A", Duration.ZERO).orElseThrow().id()).isEqualTo(work.id()); + } + } + + @ParameterizedTest + @ValueSource(ints = {1, 2}) + void failedTransferSnapshotsLeaveThePreviousWorkOwned(int failAt) throws Exception { + AtomicInteger calls = new AtomicInteger(); + AtomicBoolean enabled = new AtomicBoolean(false); + try (var store = new MemoryRoutedWorkStore(1, msg -> { + if (enabled.get() && calls.incrementAndGet() == failAt) { + throw new IllegalStateException("snapshot failure"); + } + return msg.copy(); + })) { + store.open(); + store.record(assignment("A", "original")); + var taken = store.take("A", Duration.ZERO).orElseThrow(); + enabled.set(true); + fails(() -> store.transfer(taken.id(), destination("B", "next")), UNAVAILABLE); + assertThat(store.take("B", Duration.ZERO)).isEmpty(); + store.release(taken); + assertThat(store.take("A", Duration.ZERO).orElseThrow().id()).isEqualTo(taken.id()); + var next = store.transfer(taken.id(), destination("B", "next")); + assertThat(store.take("B", Duration.ZERO).orElseThrow().id()).isEqualTo(next.id()); + } + } + + @Test + void failedTakeAndReturnSnapshotsRetainTheCurrentWork() throws Exception { + AtomicBoolean enabled = new AtomicBoolean(false); + try (var store = new MemoryRoutedWorkStore(1, msg -> { + if (enabled.get()) { + throw new IllegalStateException("snapshot failure"); + } + return msg.copy(); + })) { + store.open(); + var recorded = store.record(assignment("A", "one")); + enabled.set(true); + fails(() -> store.take("A", Duration.ZERO), UNAVAILABLE); + enabled.set(false); + var taken = store.take("A", Duration.ZERO).orElseThrow(); + assertThat(taken.id()).isEqualTo(recorded.id()); + taken.message().body = "updated"; + enabled.set(true); + fails(() -> store.release(taken), UNAVAILABLE); + enabled.set(false); + store.release(taken); + assertThat(store.take("A", Duration.ZERO).orElseThrow().message().body).isEqualTo("updated"); + assertThat(store.take("A", Duration.ZERO)).isEmpty(); + } + } + + @Test + void concurrentRecordAndTakeCallsKeepOneOwner() throws Exception { + try (var store = store(1); var executor = Executors.newVirtualThreadPerTaskExecutor()) { + var assignment = assignment("A", "one"); + var records = new ArrayList>>(); + for (int i = 0; i < 12; i++) { + records.add(executor.submit(() -> store.record(assignment))); + } + Set ids = ConcurrentHashMap.newKeySet(); + for (var record : records) { + ids.add(record.get(2, TimeUnit.SECONDS).id()); + } + assertThat(ids).hasSize(1); + var takes = new ArrayList>>>(); + for (int i = 0; i < 12; i++) { + takes.add(executor.submit(() -> store.take("A", Duration.ZERO))); + } + int owners = 0; + for (var take : takes) { + if (take.get(2, TimeUnit.SECONDS).isPresent()) { + owners++; + } + } + assertThat(owners).isEqualTo(1); + } + } + + @Test + void racingTransferAndCompletionCannotBothSucceedForTheSameWork() throws Exception { + try (var store = store(1); var executor = Executors.newVirtualThreadPerTaskExecutor()) { + for (int i = 0; i < 20; i++) { + var assignment = assignment("A", "one"); + store.record(assignment); + var taken = store.take("A", Duration.ZERO).orElseThrow(); + var completion = executor.submit(() -> store.complete(taken.id())); + var transfer = executor.submit(() -> { + try { + return Optional.of(store.transfer(taken.id(), destination("B", "next"))); + } catch (OutboundStorageException failure) { + assertThat(failure.reason()).isEqualTo(INVALID_TRANSITION); + return Optional.>empty(); + } + }); + var completed = completion.get(2, TimeUnit.SECONDS); + var forwarded = transfer.get(2, TimeUnit.SECONDS); + assertThat(completed.isPresent()).isNotEqualTo(forwarded.isPresent()); + if (forwarded.isPresent()) { + var successor = store.take("B", Duration.ZERO).orElseThrow(); + store.complete(successor.id()); + } else { + assertThat(store.take("B", Duration.ZERO)).isEmpty(); + } + store.forget(assignment.selection()); + } + } + } + + @ParameterizedTest + @ValueSource(strings = {"publish", "close", "interrupt"}) + void waitingTakesWakeWithoutWaitingForTheirTimeout(String action) throws Exception { + try (var store = store(1); var executor = Executors.newVirtualThreadPerTaskExecutor()) { + AtomicReference thread = new AtomicReference<>(); + var take = executor.submit(() -> { + thread.set(Thread.currentThread()); + return store.take("A", Duration.ofSeconds(2)); + }); + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(1); + while ((thread.get() == null || thread.get().getState() != Thread.State.TIMED_WAITING) && + System.nanoTime() < deadline) { + Thread.sleep(1); + } + assertThat(thread.get()).isNotNull(); + assertThat(thread.get().getState()).isEqualTo(Thread.State.TIMED_WAITING); + if (action.equals("publish")) { + store.record(assignment("A", "one")); + assertThat(take.get(1, TimeUnit.SECONDS)).isPresent(); + } else if (action.equals("close")) { + store.close(); + assertThatThrownBy(() -> take.get(1, TimeUnit.SECONDS)).hasCauseInstanceOf(OutboundStorageException.class); + } else { + thread.get().interrupt(); + assertThatThrownBy(() -> take.get(1, TimeUnit.SECONDS)).hasCauseInstanceOf(InterruptedException.class); + } + } + } + + @Test + void lifecycleAndNewInstanceLossAreExplicit() throws Exception { + var assignment = assignment("A", "one"); + var store = new MemoryRoutedWorkStore(1, Payload::copy); + assertThat(store.status()).isEqualTo(new OutboundStage.Status("memory", false, OutboundStage.State.NEW)); + fails(() -> store.record(assignment), UNAVAILABLE); + store.open(); + store.open(); + var work = store.record(assignment); + assertThat(store.status().durable()).isFalse(); + store.close(); + store.close(); + fails(store::open, INVALID_TRANSITION); + fails(() -> store.complete(work.id()), UNAVAILABLE); + try (var fresh = store(1)) { + assertThat(fresh.take("A", Duration.ZERO)).isEmpty(); + fails(() -> fresh.complete(work.id()), INVALID_TRANSITION); + fresh.forget(assignment.selection()); + } + } + + @Test + void invalidSnapshotsAndTimeoutsDoNotConsumeWork() throws Exception { + try (var store = new MemoryRoutedWorkStore(1, UnaryOperator.identity())) { + store.open(); + fails(() -> store.record(assignment("A", "one")), UNSUPPORTED); + } + try (var store = store(1)) { + var recorded = store.record(assignment("A", "one")); + assertThatThrownBy(() -> store.take(" ", Duration.ZERO)).isInstanceOf(IllegalArgumentException.class); + assertThatThrownBy(() -> store.take("A", Duration.ofNanos(-1))).isInstanceOf(IllegalArgumentException.class); + assertThat(store.take("A", Duration.ofSeconds(Long.MAX_VALUE)).orElseThrow().id()).isEqualTo(recorded.id()); + assertThat(store.take("A", Duration.ofMillis(1))).isEmpty(); + } + } + + private static MemoryRoutedWorkStore store(int capacity) { + var store = new MemoryRoutedWorkStore(capacity, Payload::copy); + store.open(); + return store; + } + + private static Assignment assignment(String destination, String body) { + return new Assignment<>(selection(), Set.of(source()), destination(destination, body)); + } + + private static Destination destination(String name, String body) { + return new Destination<>(name, new Payload(body)); + } + + private static SourceId source() { + return new SourceId(UUID.randomUUID()); + } + + private static SelectionId selection() { + return new SelectionId(UUID.randomUUID()); + } + + private static void fails(ThrowingCallable operation, OutboundStorageException.Reason reason) { + assertThatThrownBy(operation).isInstanceOfSatisfying(OutboundStorageException.class, failure -> { + assertThat(failure.stage()).isEqualTo(OutboundStage.Role.ROUTED_WORK); + assertThat(failure.reason()).isEqualTo(reason); + }); + } + + private static final class Payload extends StandardMessage { + private final List tags = new ArrayList<>(); + + private Payload(String body) { + this.body = body; + } + + private Payload copy() { + var result = new Payload(body); + result.tags.addAll(tags); + return result; + } + } +} From 5264ea15688fba526c6800bac9a4a33ad52ef347 Mon Sep 17 00:00:00 2001 From: pavlos Date: Tue, 29 Sep 2026 11:58:19 +0300 Subject: [PATCH 05/20] feat(storage): coordinate outbound lifecycle and terminal cleanup Task 2.1b - Implement the concrete outbound lifecycle coordinator. Coordinate owned routing, required handoff, retryable cleanup, and shutdown. Remove downstream-only worker forwarding and document scope ownership rules. Refs #338 Signed-off-by: pavlos --- AGENTS.md | 15 + docs/09-configuration-reference.md | 4 +- docs/14-outbound-storage.md | 169 +++-- .../outbound/DefaultOutboundCoordinator.java | 638 ++++++++++++++++++ .../core/outbound/OutboundCoordinator.java | 7 +- .../sendium/core/storage/RoutedWorkStore.java | 22 +- .../storage/memory/MemoryRoutedWorkStore.java | 123 +--- .../DefaultOutboundCoordinatorTest.java | 607 +++++++++++++++++ .../memory/MemoryRoutedWorkStoreTest.java | 116 +--- 9 files changed, 1458 insertions(+), 243 deletions(-) create mode 100644 sendium-core/src/main/java/gr/cytech/sendium/core/outbound/DefaultOutboundCoordinator.java create mode 100644 sendium-core/src/test/java/gr/cytech/sendium/core/outbound/DefaultOutboundCoordinatorTest.java diff --git a/AGENTS.md b/AGENTS.md index 3ef8c6c7..0b201e31 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -14,6 +14,21 @@ These instructions apply to all work in this repository. - Before merging, review all commit messages that will land on `main`, including their bodies. Release Please can interpret additional conventional-commit messages in a body as separate changelog entries. Prefer a squash merge with one capability-level summary for a cohesive feature, and preserve the required DCO trailer. - Before a release, review the generated changelog and consolidate redundant entries. Keep `CHANGELOG.md` and the release PR description consistent. Release Please may regenerate both, so verify the curated entries after its final run; do not assume a manual PR-description edit is persistent. +## Scope Ownership and Downstream Consumers + +- Implement only behavior required by this repository's approved task. Do not expand scope based on features discovered in downstream consumers, related repositories, or hypothetical future integrations. +- A request to review compatibility or keep contracts reusable authorizes compatibility analysis and appropriate extension boundaries. It does not authorize implementing consumer-specific workflows, adapters, policies, or lifecycle behavior. +- Requirements belonging to another repository or team remain their responsibility. Record relevant findings and explain potential integration needs; do not implement them without explicit approval. +- Before adding behavior primarily motivated by another consumer, identify: + 1. Which current-task requirement needs it. + 2. Which repository or team owns that behavior. + 3. Whether the task can be completed without it. + If it is not required for the approved task, defer it. +- Present any proposed scope expansion separately, with its motivation, implementation impact, and a recommendation. Wait for explicit approval before adding it to the implementation plan or writing code. General approval to proceed does not authorize an undisclosed expansion. +- Do not introduce speculative abstractions or implementation machinery merely to make future adoption easier. Prefer the smallest contract that satisfies the current task and leaves a clear extension path. +- When a missing prerequisite is discovered, distinguish work genuinely necessary to complete the approved task from optional support for another consumer. Do not describe optional consumer support as a required prerequisite. +- Apply this scope check when creating the plan, not only while coding. Clearly label cross-repository additions so they cannot enter scope through a broad task title or an assumed design decision. + ## Commits - Every commit must include a Developer Certificate of Origin `Signed-off-by:` trailer. Create commits with `git commit -s` and use the contributor's configured name and email. diff --git a/docs/09-configuration-reference.md b/docs/09-configuration-reference.md index 0d34075b..fdf1f922 100644 --- a/docs/09-configuration-reference.md +++ b/docs/09-configuration-reference.md @@ -64,8 +64,8 @@ fallback to a less durable backend. Validation runs before Sendium's file-watcher and router/worker startup observers. The validated profile is retained for the process lifetime. Startup logs the effective profile and a prominent `NON-DURABLE` warning: accepted messages and in-flight work can be lost on restart. The current -runtime pipeline is still memory-backed. All three memory stage stores exist as library -components; end-to-end coordinator and ingress/worker wiring are subsequent work in #338. Their +runtime pipeline is still memory-backed. All three memory stage stores and the default lifecycle +coordinator exist as library components; ingress/worker wiring is subsequent work in #338. Their capacity bounds are constructor arguments at this stage, not additional operator settings. See [Outbound Storage Contracts](14-outbound-storage.md). diff --git a/docs/14-outbound-storage.md b/docs/14-outbound-storage.md index d8cca4f3..a4d3b797 100644 --- a/docs/14-outbound-storage.md +++ b/docs/14-outbound-storage.md @@ -3,8 +3,8 @@ This describes the initial contracts for [#338](https://github.com/cytechmobile/sendium/issues/338), under [#337](https://github.com/cytechmobile/sendium/issues/337). The stage contracts are library APIs, and standalone profile selection, early validation, and startup logging are implemented. All three -memory stage stores are available as explicitly constructed library components. The coordinator and -end-to-end production wiring are subsequent tasks. Existing ingress and queue behavior remains non-durable. +memory stage stores and the default coordinator are available as explicitly constructed library components. +End-to-end production wiring is subsequent work. Existing ingress and queue behavior remains non-durable. ## Ownership model @@ -16,7 +16,6 @@ selected router work ---- take/release ---- router execution | record one destination assignment, then mark routed v destination work ------- take/release ---- worker execution - | | forward: transfer to one successor | all provider parts terminal AND successful required handoffs v current destination work terminal @@ -32,8 +31,20 @@ none is inferred from mutable message equality or a protocol serial. One selecte several source IDs, allowing later multipart integration without treating an aggregate as a new independent admission. Detailed multipart admission and reconstruction are owned by #343. -There is at most one active destination work item per selection. Forwarding transfers that ownership -to a successor with a new work ID under the same selection; it does not create simultaneous destinations. +Each routed selection has one destination and one work identity through execution, same-worker retry, +and terminal cleanup. Direct worker-to-worker forwarding is outside this abstraction's scope. + +## Sendium retry integration boundary + +The new library supports returning unfinished routing work through `returnToRouter(Selected)` and +retrying taken work at the same destination through `returnToDestination(Routed)`. The former cannot +be used after destination assignment and is not a worker-to-router transition. + +Sendium's existing `AbstractOutWorker` failure policies can call `enqueueToRouter` to let routing choose +again. That production behavior remains intact. Its source-retaining lifecycle integration is planned +separately; it must be implemented before the new pipeline replaces that path. No direct worker-to-worker +API is substituted for it. Consumer-specific dispatchers, including mCore's queue worker, remain the +consumer team's responsibility and require separate approval for Sendium support. ## Copied routing boundary @@ -41,7 +52,7 @@ The existing `+vendor`/`copied` routing feature and its public API remain availa routing path. The new storage abstraction does not support copied routing. Its routing integration must reject a copied-route request as `UNSUPPORTED` before assignment or dispatch, even if the lookup happens to produce only one destination. It must not silently choose one result, strip the -copy flag, or emulate fan-out using several assignments or forwarding calls. +copy flag, or emulate fan-out using several assignments for the same source. This guard belongs to the upcoming routing integration task; this profile-selection step does not change existing routing behavior. The single-destination scope is the owner's revision to the copied-route @@ -54,6 +65,7 @@ The public APIs are grouped by responsibility under `gr.cytech.sendium.core`: ```text outbound/ OutboundCoordinator.java + DefaultOutboundCoordinator.java OutboundWork.java storage/ OutboundStage.java @@ -81,8 +93,9 @@ These classes are absent from the `sendium-core` artifact; they do not activate |---|---| | `PendingMessageStore` | Record accepted source state under a stable caller-supplied identity and complete sources idempotently. | | `SelectedRouterStore` | Own bounded select-and-stage, exclusive runtime takes, return-to-router, and retained selected state. | -| `RoutedWorkStore` | Record one destination, own its scheduling, transfer responsibility on forwarding, and report terminal source ownership. | +| `RoutedWorkStore` | Record one destination, own same-destination scheduling/retry, and report terminal source ownership. | | `OutboundCoordinator` | Application-facing admission, transitions, required handoff, source completion, and lifecycle coordination. | +| `DefaultOutboundCoordinator` | Executable coordinator for fresh non-durable stages, with owned transitions and retryable handoff/cleanup. | | `OutboundWork` | Source/selection/work identities and typed execution, assignment, and completion values. | | `OutboundStage` | Explicit open/close, backend identity, durability declaration, and lifecycle availability. | | `OutboundStorageException` | Identify the failing stage and distinguish unavailable, capacity, ownership, unsupported, and invalid-transition failures. | @@ -96,7 +109,7 @@ ownership/state transitions use the categorized storage failure. `OutboundWork.Assignment` carries exactly one `Destination`, and `Routed` identifies its execution with a `WorkId`. The work values freeze source identity sets, not message objects. Worker and filter -mutations are intentional execution state. They continue through transitions, returns, and forwarding; +mutations are intentional execution state. They continue through routing and same-worker returns; they must not silently change a retained earlier-stage record through a shared mutable reference. The contract requires consistent ownership and stored payload/stage, not a public copying service or @@ -196,56 +209,128 @@ coordinated cleanup in the later integration tasks. `MemoryRoutedWorkStore` is independently constructed with a positive selection capacity and a `UnaryOperator` snapshot function, with the same isolation/subtype requirements as the pending memory backend. It has its own lock and does not depend on a concrete pending or selected store. -Coordination across stores belongs to the upcoming coordinator. +Coordination across stores belongs to `DefaultOutboundCoordinator`. ```text record assignment → QUEUED → take → TAKEN ↑ | +-- release --+ | - +--------------------+-------------------+ - | | - transfer complete - | | - previous: TRANSFERRED COMPLETED - successor: QUEUED | - (new ID, same selection) repeatable cleanup result - | - coordinator finishes cleanup - | - forget + complete + | + COMPLETED + | + repeatable cleanup result + | + coordinator finishes cleanup + | + forget ``` - `record` retains one assignment per selection. Matching retries return its original work ID and - original assignment payload, even after forwarding or completion, without re-enqueueing it. A + original assignment payload, even after completion, without re-enqueueing it. A changed source set/destination is invalid. A source cannot belong to two retained routed selections. -- `record` and `transfer` results describe recorded work; they do not claim execution. Only `take` +- The `record` result describes recorded work; it does not claim execution. Only `take` claims a work item. Each destination schedules in publication order. Take and return preserve one active taker and updated execution state; return the actual take result as its local attempt handle. -- `transfer` requires taken work and atomically moves responsibility to one successor under the same - selection. It works even at full selection capacity because it reuses that selection's slot. Retrying - the predecessor returns its original immediate successor, even if that successor has moved onward. - A conflicting retry destination is invalid. Old work is never scheduled again. - `complete` requires taken work after the caller has established terminal processing and required - handoff. Repeated completion returns the same source set. Completion for any transferred predecessor - returns empty, including after its successor completes, so late callbacks cannot release sources early. + handoff. It returns `Completed` directly; repeated completion returns the same selection/source set. + There is no transferred-predecessor case or empty completion result. - `forget` rejects nonterminal selections. After the coordinator completes pending/selected cleanup, - it removes that selection's entire forwarding history and source claims, freeing capacity. Repeated + it removes that selection's one work record and source claims, freeing capacity. Repeated forget is harmless. Unknown work IDs fail explicitly. Replay guarantees apply while records are - retained; the coordinator must reject stale route/transfer requests and absorb duplicate terminal + retained; the coordinator must reject stale routing requests and absorb duplicate terminal callbacks after cleanup rather than recreating old work. Capacity counts retained selections, including taken work and terminal selections awaiting cleanup. -Return and forwarding cannot create additional active selections or exhaust a separate return slot. -Forwarding history and original replay payloads are retained once per hop until forget; the selection -count is not a bound on history length or message bytes. The later worker integration owns retry and -forwarding policy. +Same-worker return reuses that selection's slot even at full capacity. The same entry is indexed by +selection ID and work ID; no successor chain or forwarding history is stored. The original assignment +payload is retained for record retries alongside current execution state. Selection count does not bound +message bytes. Sendium's retry/rerouting policy is integrated in the later worker task. -All required snapshots, including the returned result, are prepared before record/transfer publication. -If a snapshot fails, no partial assignment or successor is published, and the current work remains +All required snapshots, including the returned result, are prepared before assignment publication. +If a snapshot fails, no partial assignment is published, and the current work remains retryable. Takes are interruptible and wake on publication or close. Closure clears instance-local state without manufacturing completion; closed instances cannot reopen and fresh instances recover nothing. +## Default lifecycle coordinator + +Construct `DefaultOutboundCoordinator` with the pending, selected-router, and routed-work stores +as three direct dependencies. Construction is passive. `start()` opens them in that order; failed startup +closes every attempted stage in reverse order, including a stage whose open failed, and preserves +cleanup failures as suppressed exceptions. + +This implementation currently requires fresh, unopened, non-durable stages. Already-opened stages +are not silently adopted, and durable stages fail with `UNSUPPORTED` because startup recovery and +cross-backend reconciliation are not implemented yet. Non-durable embedding implementations can +use the interfaces directly; there is no requirement for their backend identity string to be `memory`. + +The application must use the coordinator exclusively for scheduling and transitions. `admit` does not +implicitly refill the router; invoke bounded `selectAndStage` explicitly. Blocking takes poll the owned +store with a zero timeout and then wait on the coordinator's publication/quiescence condition. +They support interruption and timeouts and wake when coordinated work is published or quiescence begins. + +### Owned transitions + +The coordinator tracks live selections, source ownership, and destination executions. It validates the +actual take projection when returning or routing work. Recording a destination precedes marking the +selection routed. A failed route retains its original destination intent; retry the same `route` operation, +not the routing/filter lookup. While that transition is unresolved, its source cannot return to routing +or be dispatched at the destination. Other existing destination work can still complete and free capacity. + +A backend that committed an assignment before reporting a failure can be reconciled using the same selection/work identities; +observing its published take associates it with the retained intent. An unexposed take whose return +failed is retained for a return retry rather than disappearing from ownership. + +### Required handoff and cleanup + +`complete(workId, requiredHandoff)` begins terminal processing for a taken destination. `discard` +does the equivalent for a taken, unassigned routing item. Concurrent duplicate reports share the active +completion attempt. No source is removed while the handoff is pending, failed, or cancelled. Retry a +failed handoff explicitly with another completion call; the work cannot meanwhile be returned for +provider redispatch. + +Once the handoff succeeds, the coordinator remembers that success and performs: + +```text +destination complete (when routed) + → verify returned selection/source identities + → pending sources complete + → selected record complete + → routed bookkeeping forget (when routed) + → remove coordinator ownership +``` + +Each successful cleanup step is remembered. A failed step is retried through `complete`/`discard` +without awaiting another handoff or dispatching provider work. Partial multi-source deletion retries +the full source set idempotently. Duplicate admission of a still-owned source does not recreate a +source record during an unfinished cleanup. The source identity must represent the same admission; +after all ownership is forgotten, long-term admission deduplication is not promised. + +Completion results are read-only stages: cancelling a caller's `toCompletableFuture()` view cannot +cancel the owned handoff or cleanup. Returned-stage listeners run outside the coordinator lock. +Handoff or cleanup failures settle that attempt exceptionally and retain ownership for retry. + +To avoid retaining completed-ID tombstones forever, terminal reports for unknown/no-longer-owned +work or selections are harmless no-ops. They never recreate work, remove another source, or wait on +an unrelated supplied handoff. Route and return requests still require live ownership and +reject stale IDs/projections. Unknown-ID errors raised for an operation on live owned work propagate; +they are not treated as successful cleanup. + +### Quiescence and close + +`quiesce()` stops new admission, selection, and takes while allowing existing transitions, returns, +handoff completion, and cleanup retries. The application stops/joins its processors before `close`. +Close returns taken routing work to the router and taken destination work to that same destination, +then closes routed, selected, and pending stores. It never completes sources merely to shut down. + +Outstanding handoff attempts, unfinished routing intents, and failed terminal cleanup must be +resolved/retried before close can finish. A refused close leaves the coordinator quiescing and its +stores open. A return failure also leaves ownership available for a subsequent close retry. Once +draining succeeds, close attempts every store and preserves close failures; closure is idempotent and +restart is forbidden. This library lifecycle does not stop application executors or provide a forced +shutdown policy. Actual ingress/worker activation and application shutdown ordering remain integration work. + ## Completion and retry boundaries - Admission retries reuse the same source ID and do not overwrite the accepted state while the @@ -255,14 +340,10 @@ without manufacturing completion; closed instances cannot reopen and fresh insta and retry mutations without silently changing earlier-stage state. The take's identities and destination cannot change on return. This does not promise durable retry counters or timing. - `record` publishes a single assignment before its work can be taken. Retrying the same selection - returns its original work ID without reactivating completed/transferred work. Conflicting source + returns its original work ID without reactivating completed work. Conflicting source identities or destinations fail as `INVALID_TRANSITION`, rather than creating another assignment. - Marking selected work routed stops router scheduling but retains the selected record until terminal source cleanup. This leaves room for a future durable router with memory-backed routed work. -- `transfer` atomically moves responsibility to one successor before that successor can be taken. - Retrying with the previous ID returns the original successor; a conflicting successor destination - fails as `INVALID_TRANSITION`. Completion of the previous work is ignored, even after its successor - completes. A late previous-worker callback must never complete the successor. - Provider multipart is aggregated by provider-processing integration: all parts must have terminal outcomes and all required handoffs must succeed before completing the one destination work item. Part submissions are not represented as multiple routed destinations. @@ -287,9 +368,9 @@ exactly-once provider submission is promised. An application assembles compatible `PendingMessageStore`, `SelectedRouterStore`, and `RoutedWorkStore` instances and passes them directly as three constructor dependencies to its -coordinator implementation. There is no dependency-bundle type or public copier contract. The -coordinator remains an interface at this step; the concrete constructor and wiring arrive with its -implementation. Constructors must have no activation side effects. The standalone profile is assembled +coordinator implementation. There is no dependency-bundle type or public copier contract. +`DefaultOutboundCoordinator` provides that constructor and the initial non-durable lifecycle. Constructors +have no activation side effects. The standalone profile is assembled through CDI in `sendium-app`; the reusable lifecycle/storage APIs require no CDI annotations or container. For example, mCore's `Message extends StandardMessage`, billing-aware batch preparation, diff --git a/sendium-core/src/main/java/gr/cytech/sendium/core/outbound/DefaultOutboundCoordinator.java b/sendium-core/src/main/java/gr/cytech/sendium/core/outbound/DefaultOutboundCoordinator.java new file mode 100644 index 00000000..28b5b8aa --- /dev/null +++ b/sendium-core/src/main/java/gr/cytech/sendium/core/outbound/DefaultOutboundCoordinator.java @@ -0,0 +1,638 @@ +package gr.cytech.sendium.core.outbound; + +import gr.cytech.sendium.core.message.StandardMessage; +import gr.cytech.sendium.core.outbound.OutboundWork.Assignment; +import gr.cytech.sendium.core.outbound.OutboundWork.Completed; +import gr.cytech.sendium.core.outbound.OutboundWork.Destination; +import gr.cytech.sendium.core.outbound.OutboundWork.Routed; +import gr.cytech.sendium.core.outbound.OutboundWork.Selected; +import gr.cytech.sendium.core.outbound.OutboundWork.SelectionId; +import gr.cytech.sendium.core.outbound.OutboundWork.SourceId; +import gr.cytech.sendium.core.outbound.OutboundWork.WorkId; +import gr.cytech.sendium.core.storage.OutboundStage; +import gr.cytech.sendium.core.storage.OutboundStage.Role; +import gr.cytech.sendium.core.storage.OutboundStorageException; +import gr.cytech.sendium.core.storage.PendingMessageStore; +import gr.cytech.sendium.core.storage.RoutedWorkStore; +import gr.cytech.sendium.core.storage.SelectedRouterStore; + +import java.time.Duration; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import java.util.Set; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionStage; +import java.util.concurrent.locks.Condition; +import java.util.concurrent.locks.ReentrantLock; + +import static gr.cytech.sendium.core.storage.OutboundStorageException.Reason.INVALID_TRANSITION; +import static gr.cytech.sendium.core.storage.OutboundStorageException.Reason.UNAVAILABLE; +import static gr.cytech.sendium.core.storage.OutboundStorageException.Reason.UNSUPPORTED; + +/** + * Coordinates fresh non-durable stages. All execution goes through this instance; its stores are not + * independently scheduled. Durable startup reconciliation is deliberately unavailable in this baseline. + * Handoff waits and callbacks on returned completion stages never run under the coordinator lock. + */ +public final class DefaultOutboundCoordinator implements OutboundCoordinator { + private final PendingMessageStore pending; + private final SelectedRouterStore router; + private final RoutedWorkStore routed; + private final ReentrantLock lock = new ReentrantLock(); + private final Condition changed = lock.newCondition(); + private final Map> selections = new HashMap<>(); + private final Map> sourceSelections = new HashMap<>(); + private final Map> executions = new HashMap<>(); + private final Set> pendingReturns = new HashSet<>(); + private Lifecycle lifecycle = Lifecycle.NEW; + + public DefaultOutboundCoordinator(PendingMessageStore pending, SelectedRouterStore router, RoutedWorkStore routed) { + this.pending = Objects.requireNonNull(pending, "pending"); + this.router = Objects.requireNonNull(router, "router"); + this.routed = Objects.requireNonNull(routed, "routed"); + } + + @Override + public void start() { + lock.lock(); + try { + if (lifecycle == Lifecycle.RUNNING) { + return; + } + if (lifecycle != Lifecycle.NEW) { + throw invalid(Role.PENDING, "Coordinator cannot restart"); + } + List attempted = new ArrayList<>(); + try { + open(pending, Role.PENDING, attempted); + open(router, Role.ROUTER_QUEUE, attempted); + open(routed, Role.ROUTED_WORK, attempted); + lifecycle = Lifecycle.RUNNING; + } catch (RuntimeException | Error failure) { + for (OutboundStage stage : attempted.reversed()) { + try { + stage.close(); + } catch (RuntimeException | Error cleanupFailure) { + if (cleanupFailure != failure) { + failure.addSuppressed(cleanupFailure); + } + } + } + lifecycle = Lifecycle.CLOSED; + throw failure; + } + } finally { + lock.unlock(); + } + } + + private void open(OutboundStage stage, Role role, List attempted) { + OutboundStage.Status status = stage.status(); + if (status.durable()) { + throw new OutboundStorageException(role, UNSUPPORTED, "Durable stage recovery is not implemented by this coordinator"); + } + if (status.state() != OutboundStage.State.NEW) { + throw invalid(role, "Coordinator requires fresh, unopened stages"); + } + attempted.add(stage); + stage.open(); + if (stage.status().state() != OutboundStage.State.READY) { + throw unavailable(role, "Stage did not become ready during startup"); + } + } + + @Override + public void quiesce() { + lock.lock(); + try { + if (lifecycle == Lifecycle.RUNNING) { + lifecycle = Lifecycle.QUIESCING; + changed.signalAll(); + } else if (lifecycle == Lifecycle.NEW) { + throw unavailable(Role.PENDING, "Coordinator has not started"); + } + } finally { + lock.unlock(); + } + } + + @Override + public void admit(SourceId source, M message) { + Objects.requireNonNull(source, "source"); + Objects.requireNonNull(message, "message"); + lock.lock(); + try { + requireRunning(Role.PENDING); + if (sourceSelections.containsKey(source)) { + return; + } + pending.admit(source, message); + } finally { + lock.unlock(); + } + } + + @Override + public int selectAndStage(int limit) { + lock.lock(); + try { + requireRunning(Role.ROUTER_QUEUE); + return router.selectAndStage(limit); + } finally { + changed.signalAll(); + lock.unlock(); + } + } + + @Override + public Optional> takeForRouting(Duration timeout) throws InterruptedException { + long remaining = timeoutNanos(timeout); + lock.lockInterruptibly(); + try { + while (true) { + requireRunning(Role.ROUTER_QUEUE); + Optional> next = router.take(Duration.ZERO); + if (next.isPresent()) { + Selected selected = next.orElseThrow(); + Context context = selections.get(selected.id()); + if (context == null) { + if (selected.sources().stream().anyMatch(sourceSelections::containsKey)) { + router.release(selected); + throw invalid(Role.ROUTER_QUEUE, "Source is already owned by another selection"); + } + context = new Context<>(selected); + selections.put(context.id, context); + for (SourceId source : context.sources) { + sourceSelections.put(source, context); + } + } + if (context.routingTaken || context.assignment != null || context.finishing || + !context.sources.equals(selected.sources())) { + router.release(selected); + throw invalid(Role.ROUTER_QUEUE, "Selected store returned work already owned by another lifecycle step"); + } + context.routingTake = selected; + context.routingTaken = true; + return next; + } + if (remaining <= 0) { + return Optional.empty(); + } + remaining = changed.awaitNanos(remaining); + } + } finally { + lock.unlock(); + } + } + + @Override + public void returnToRouter(Selected selected) { + Objects.requireNonNull(selected, "selected"); + lock.lock(); + try { + requireActive(Role.ROUTER_QUEUE); + Context context = context(selected.id()); + if (context.assignment != null || context.finishing) { + throw invalid(Role.ROUTER_QUEUE, "Work transitioning to a destination or completion cannot be returned to routing"); + } + if (context.lastRoutingReturn == selected) { + return; + } + if (!context.routingTaken || context.routingTake != selected) { + throw invalid(Role.ROUTER_QUEUE, "Only the current routing take can be returned"); + } + router.release(selected); + context.routingTaken = false; + context.lastRoutingReturn = selected; + changed.signalAll(); + } finally { + lock.unlock(); + } + } + + @Override + public Routed route(Selected selected, Destination destination) { + Objects.requireNonNull(selected, "selected"); + Objects.requireNonNull(destination, "destination"); + lock.lock(); + try { + requireActive(Role.ROUTER_QUEUE); + Context context = context(selected.id()); + if (context.routingTake != selected || context.finishing || + (!context.routingTaken && context.assignment == null)) { + throw invalid(Role.ROUTER_QUEUE, "Routing requires the current take owned by this coordinator"); + } + if (context.assignment == null) { + context.assignment = new Assignment<>(context.id, context.sources, destination); + } else if (!context.assignment.destination().name().equals(destination.name())) { + throw invalid(Role.ROUTED_WORK, "A routing retry cannot change the recorded destination intent"); + } + final Routed result = routed.record(context.assignment); + registerAssignment(context, result); + if (!context.markedRouted) { + router.markRouted(context.id); + context.markedRouted = true; + context.routingTaken = false; + } + return result; + } finally { + changed.signalAll(); + lock.unlock(); + } + } + + @Override + public Optional> takeForDestination(String destination, Duration timeout) throws InterruptedException { + Objects.requireNonNull(destination, "destination"); + if (destination.isBlank()) { + throw new IllegalArgumentException("Destination must not be blank"); + } + long remaining = timeoutNanos(timeout); + lock.lockInterruptibly(); + try { + while (true) { + requireRunning(Role.ROUTED_WORK); + var returns = pendingReturns.iterator(); + while (returns.hasNext()) { + Execution execution = returns.next(); + routed.release(execution.take); + execution.take = null; + returns.remove(); + } + Optional> next = routed.take(destination, Duration.ZERO); + if (next.isPresent()) { + Routed taken = next.orElseThrow(); + Execution execution; + try { + execution = registerTaken(taken); + } catch (RuntimeException failure) { + try { + routed.release(taken); + } catch (RuntimeException releaseFailure) { + if (releaseFailure != failure) { + failure.addSuppressed(releaseFailure); + } + } + throw failure; + } + execution.take = taken; + if (!execution.context.markedRouted) { + pendingReturns.add(execution); + routed.release(taken); + execution.take = null; + pendingReturns.remove(execution); + throw unavailable(Role.ROUTER_QUEUE, "Retry the unfinished routing transition before dispatch"); + } + execution.exposed = true; + return next; + } + if (remaining <= 0) { + return Optional.empty(); + } + remaining = changed.awaitNanos(remaining); + } + } finally { + lock.unlock(); + } + } + + @Override + public void returnToDestination(Routed taken) { + Objects.requireNonNull(taken, "taken"); + lock.lock(); + try { + requireActive(Role.ROUTED_WORK); + Execution execution = execution(taken.id()); + if (execution.context.finishing) { + throw invalid(Role.ROUTED_WORK, "Terminal work cannot be redispatched"); + } + if (execution.lastReturn == taken) { + return; + } + if (execution.take != taken) { + throw invalid(Role.ROUTED_WORK, "Only the current destination take can be returned"); + } + routed.release(taken); + execution.take = null; + execution.exposed = false; + execution.lastReturn = taken; + changed.signalAll(); + } finally { + lock.unlock(); + } + } + + @Override + public CompletionStage complete(WorkId id, CompletionStage requiredHandoff) { + Objects.requireNonNull(id, "id"); + Objects.requireNonNull(requiredHandoff, "requiredHandoff"); + Context context; + Attempt attempt; + lock.lock(); + try { + requireActive(Role.ROUTED_WORK); + Execution execution = executions.get(id); + if (execution == null) { + return CompletableFuture.completedStage(null); + } + context = execution.context; + if (!context.finishing && (execution.take == null || !execution.exposed)) { + throw invalid(Role.ROUTED_WORK, "Completion requires a destination take"); + } + if (context.attempt != null) { + return context.attempt.view; + } + context.finishing = true; + attempt = new Attempt(); + context.attempt = attempt; + } finally { + lock.unlock(); + } + awaitHandoff(context, attempt, requiredHandoff); + return attempt.view; + } + + @Override + public CompletionStage discard(SelectionId id, CompletionStage requiredHandoff) { + Objects.requireNonNull(id, "id"); + Objects.requireNonNull(requiredHandoff, "requiredHandoff"); + Context context; + Attempt attempt; + lock.lock(); + try { + requireActive(Role.ROUTER_QUEUE); + context = selections.get(id); + if (context == null) { + return CompletableFuture.completedStage(null); + } + if (context.assignment != null || !context.routingTaken) { + throw invalid(Role.ROUTER_QUEUE, "Only taken, unassigned routing work can be discarded"); + } + if (context.attempt != null) { + return context.attempt.view; + } + context.finishing = true; + attempt = new Attempt(); + context.attempt = attempt; + } finally { + lock.unlock(); + } + awaitHandoff(context, attempt, requiredHandoff); + return attempt.view; + } + + private void awaitHandoff(Context context, Attempt attempt, CompletionStage handoff) { + if (context.handoffAccepted) { + finish(context, attempt, null); + } else { + try { + handoff.whenComplete((ignored, failure) -> finish(context, attempt, failure)); + } catch (RuntimeException | Error failure) { + finish(context, attempt, failure); + } + } + } + + private void finish(Context context, Attempt attempt, Throwable failure) { + Throwable outcome = failure; + lock.lock(); + try { + if (context.attempt != attempt) { + return; + } + if (outcome == null) { + context.handoffAccepted = true; + try { + cleanup(context); + } catch (RuntimeException | Error cleanupFailure) { + outcome = cleanupFailure; + } + } + context.attempt = null; + } finally { + lock.unlock(); + } + if (outcome == null) { + attempt.result.complete(null); + } else { + attempt.result.completeExceptionally(outcome); + } + } + + private void cleanup(Context context) { + if (context.workId != null && !context.destinationCompleted) { + Completed completed = routed.complete(context.workId); + if (!completed.selection().equals(context.id) || !completed.sources().equals(context.sources)) { + throw invalid(Role.ROUTED_WORK, "Completion does not match owned source identities"); + } + context.destinationCompleted = true; + executions.get(context.workId).take = null; + } + if (!context.pendingCompleted) { + pending.complete(context.sources); + context.pendingCompleted = true; + } + if (!context.selectedCompleted) { + router.complete(context.id); + context.selectedCompleted = true; + } + if (context.workId != null) { + routed.forget(context.id); + pendingReturns.remove(executions.remove(context.workId)); + } + context.sources.forEach(sourceSelections::remove); + selections.remove(context.id); + } + + @Override + public void close() { + lock.lock(); + try { + if (lifecycle == Lifecycle.CLOSED) { + return; + } + if (lifecycle == Lifecycle.NEW) { + lifecycle = Lifecycle.CLOSED; + return; + } + lifecycle = Lifecycle.QUIESCING; + changed.signalAll(); + for (Context context : selections.values()) { + if (context.attempt != null || (context.finishing && context.handoffAccepted)) { + throw unavailable(Role.ROUTED_WORK, "Resolve outstanding handoffs and retry terminal cleanup before close"); + } + if (context.assignment != null && !context.markedRouted) { + throw unavailable(Role.ROUTER_QUEUE, "Retry unfinished routing transitions before close"); + } + } + for (Context context : selections.values()) { + if (context.routingTaken) { + router.release(context.routingTake); + context.routingTaken = false; + } + } + for (Execution execution : executions.values()) { + if (execution.take != null) { + routed.release(execution.take); + execution.take = null; + } + } + Throwable failure = null; + for (OutboundStage stage : List.of(routed, router, pending)) { + try { + stage.close(); + } catch (RuntimeException | Error closeFailure) { + if (failure == null) { + failure = closeFailure; + } else if (failure != closeFailure) { + failure.addSuppressed(closeFailure); + } + } + } + lifecycle = Lifecycle.CLOSED; + selections.clear(); + sourceSelections.clear(); + executions.clear(); + pendingReturns.clear(); + if (failure instanceof RuntimeException runtimeFailure) { + throw runtimeFailure; + } + if (failure instanceof Error error) { + throw error; + } + } finally { + lock.unlock(); + } + } + + private Execution registerAssignment(Context context, Routed result) { + if (!result.selection().equals(context.id) || !result.destination().equals(context.assignment.destination().name()) || + (context.workId != null && !context.workId.equals(result.id()))) { + throw invalid(Role.ROUTED_WORK, "Recorded work does not match the routing intent"); + } + Execution existing = executions.get(result.id()); + if (existing != null) { + if (existing.context != context || !existing.destination.equals(result.destination())) { + throw invalid(Role.ROUTED_WORK, "Work identity is already owned by another assignment"); + } + return existing; + } + Execution execution = new Execution<>(context, result.destination()); + executions.put(result.id(), execution); + context.workId = result.id(); + return execution; + } + + private Execution registerTaken(Routed taken) { + Execution execution = executions.get(taken.id()); + if (execution == null) { + Context context = context(taken.selection()); + if (context.assignment == null || context.workId != null) { + throw invalid(Role.ROUTED_WORK, "No routing intent owns this work"); + } + execution = registerAssignment(context, taken); + } + if (!execution.context.id.equals(taken.selection()) || !execution.destination.equals(taken.destination()) || + execution.take != null || execution.context.finishing) { + throw invalid(Role.ROUTED_WORK, "Destination store returned work already owned by another lifecycle step"); + } + return execution; + } + + private Context context(SelectionId id) { + Context context = selections.get(Objects.requireNonNull(id, "selection")); + if (context == null) { + throw invalid(Role.ROUTER_QUEUE, "Selection is not owned by this coordinator"); + } + return context; + } + + private Execution execution(WorkId id) { + Execution execution = executions.get(Objects.requireNonNull(id, "work")); + if (execution == null) { + throw invalid(Role.ROUTED_WORK, "Work is not owned by this coordinator"); + } + return execution; + } + + private void requireRunning(Role role) { + if (lifecycle != Lifecycle.RUNNING) { + throw unavailable(role, "Coordinator is not accepting new work"); + } + } + + private void requireActive(Role role) { + if (lifecycle != Lifecycle.RUNNING && lifecycle != Lifecycle.QUIESCING) { + throw unavailable(role, "Coordinator is not active"); + } + } + + private static OutboundStorageException invalid(Role role, String message) { + return new OutboundStorageException(role, INVALID_TRANSITION, message); + } + + private static OutboundStorageException unavailable(Role role, String message) { + return new OutboundStorageException(role, UNAVAILABLE, message); + } + + private static long timeoutNanos(Duration timeout) { + Objects.requireNonNull(timeout, "timeout"); + if (timeout.isNegative()) { + throw new IllegalArgumentException("Timeout must not be negative"); + } + try { + return timeout.toNanos(); + } catch (ArithmeticException overflow) { + return Long.MAX_VALUE; + } + } + + private enum Lifecycle { + NEW, RUNNING, QUIESCING, CLOSED + } + + private static final class Attempt { + private final CompletableFuture result = new CompletableFuture<>(); + private final CompletionStage view = result.minimalCompletionStage(); + } + + private static final class Context { + private final SelectionId id; + private final Set sources; + private Selected routingTake; + private Selected lastRoutingReturn; + private boolean routingTaken; + private Assignment assignment; + private WorkId workId; + private boolean markedRouted; + private boolean finishing; + private boolean handoffAccepted; + private boolean destinationCompleted; + private boolean pendingCompleted; + private boolean selectedCompleted; + private Attempt attempt; + + private Context(Selected selected) { + this.id = selected.id(); + this.sources = selected.sources(); + } + } + + private static final class Execution { + private final Context context; + private final String destination; + private Routed take; + private Routed lastReturn; + private boolean exposed; + + private Execution(Context context, String destination) { + this.context = context; + this.destination = destination; + } + } +} diff --git a/sendium-core/src/main/java/gr/cytech/sendium/core/outbound/OutboundCoordinator.java b/sendium-core/src/main/java/gr/cytech/sendium/core/outbound/OutboundCoordinator.java index 037372d9..bf8725c9 100644 --- a/sendium-core/src/main/java/gr/cytech/sendium/core/outbound/OutboundCoordinator.java +++ b/sendium-core/src/main/java/gr/cytech/sendium/core/outbound/OutboundCoordinator.java @@ -39,6 +39,7 @@ public interface OutboundCoordinator extends AutoClos Optional> takeForRouting(Duration timeout) throws InterruptedException; + /** Returns unfinished routing work; this is not a transition from a worker back to routing. */ void returnToRouter(Selected selected); /** @@ -53,17 +54,13 @@ public interface OutboundCoordinator extends AutoClos void returnToDestination(Routed work); - /** Transfers responsibility to one successor without treating the previous work as terminal delivery. */ - Routed forward(WorkId previous, Destination destination); - /** * Reports terminal processing of all provider parts for this work, conditional on successful required * handoff for all parts. The provider-processing integration aggregates part outcomes and handoffs. * Pass an already successful stage when no handoff is required. Failure or cancellation retains work * and source ownership; retrying completion must not redispatch provider work. The returned stage * completes after work completion and pending/selected cleanup. Duplicate callbacks are harmless, - * including after cleanup. A late callback for transferred work is a successful no-op, not a signal - * that its successor or source has completed. + * including after cleanup, and may never complete unrelated work. */ CompletionStage complete(WorkId work, CompletionStage requiredHandoff); diff --git a/sendium-core/src/main/java/gr/cytech/sendium/core/storage/RoutedWorkStore.java b/sendium-core/src/main/java/gr/cytech/sendium/core/storage/RoutedWorkStore.java index e501b70e..2b2a374d 100644 --- a/sendium-core/src/main/java/gr/cytech/sendium/core/storage/RoutedWorkStore.java +++ b/sendium-core/src/main/java/gr/cytech/sendium/core/storage/RoutedWorkStore.java @@ -3,7 +3,6 @@ import gr.cytech.sendium.core.message.StandardMessage; import gr.cytech.sendium.core.outbound.OutboundWork.Assignment; import gr.cytech.sendium.core.outbound.OutboundWork.Completed; -import gr.cytech.sendium.core.outbound.OutboundWork.Destination; import gr.cytech.sendium.core.outbound.OutboundWork.Routed; import gr.cytech.sendium.core.outbound.OutboundWork.SelectionId; import gr.cytech.sendium.core.outbound.OutboundWork.WorkId; @@ -16,9 +15,8 @@ public interface RoutedWorkStore extends OutboundStag * Records the execution payload and one destination consistently before its work may be taken. * Future recovery resumes this recorded work at its worker, not through routing again. * While retained, repeating a selection returns its original assignment and work ID without reactivating - * completed/transferred work or overwriting content. - * Callers retry with the same assignment; changing an active destination requires transfer, not - * another assignment. There is at most one active destination work item per selection. + * completed work or overwriting content. Callers retry with the same assignment; + * a selection has exactly one recorded destination and work identity. * Conflicting source identities or destinations for an existing selection fail as INVALID_TRANSITION. * After forget, the coordinator must reject stale recording attempts using its lifecycle ownership. */ @@ -35,29 +33,19 @@ public interface RoutedWorkStore extends OutboundStag /** * Retains the recorded destination and updated execution projection when retrying or draining * in-flight work. Identity, selection, and destination must match the take. Repeated release cannot - * duplicate work or reactivate completed/transferred work. Earlier-stage state cannot silently change; + * duplicate work or reactivate completed work. Earlier-stage state cannot silently change; * preserving runtime mutations does not promise durable retry counters or timing. * Pass the take result itself with its message updated, not a reconstructed projection. */ void release(Routed work); - /** - * Atomically transfers active responsibility to exactly one successor with a new work ID under - * the same selection. Retrying with the same previous ID returns the original successor without - * reactivating it. The previous item is no longer schedulable and its late callbacks cannot complete - * the successor. Transfer must not transiently complete the source or leave two active destinations. - * A retry naming a different successor destination fails as INVALID_TRANSITION. - */ - Routed transfer(WorkId previous, Destination destination); - /** * Called only after all provider parts and required handoffs for the active work are terminal. * Provider-part aggregation belongs to the provider-processing boundary, not separate routed items. - * Returns the completed selection for source cleanup. A transferred previous item returns empty, - * even after its successor completes. Until forget, repeating completion of the terminal work + * Returns the completed selection for source cleanup. Until forget, repeating completion of the terminal work * returns the same result so cleanup can be retried without sending again. Unknown IDs fail explicitly. */ - Optional complete(WorkId work); + Completed complete(WorkId work); /** * Discards terminal bookkeeping only after pending and selected completion succeed. Nonterminal diff --git a/sendium-core/src/main/java/gr/cytech/sendium/core/storage/memory/MemoryRoutedWorkStore.java b/sendium-core/src/main/java/gr/cytech/sendium/core/storage/memory/MemoryRoutedWorkStore.java index 0cbc2955..1dfe7628 100644 --- a/sendium-core/src/main/java/gr/cytech/sendium/core/storage/memory/MemoryRoutedWorkStore.java +++ b/sendium-core/src/main/java/gr/cytech/sendium/core/storage/memory/MemoryRoutedWorkStore.java @@ -3,7 +3,6 @@ import gr.cytech.sendium.core.message.StandardMessage; import gr.cytech.sendium.core.outbound.OutboundWork.Assignment; import gr.cytech.sendium.core.outbound.OutboundWork.Completed; -import gr.cytech.sendium.core.outbound.OutboundWork.Destination; import gr.cytech.sendium.core.outbound.OutboundWork.Routed; import gr.cytech.sendium.core.outbound.OutboundWork.SelectionId; import gr.cytech.sendium.core.outbound.OutboundWork.SourceId; @@ -13,9 +12,7 @@ import java.time.Duration; import java.util.ArrayDeque; -import java.util.ArrayList; import java.util.HashMap; -import java.util.List; import java.util.Map; import java.util.Objects; import java.util.Optional; @@ -31,7 +28,7 @@ import static gr.cytech.sendium.core.storage.OutboundStorageException.Reason.UNSUPPORTED; /** - * Single-destination execution with instance-local forwarding history. Capacity counts retained + * Single-destination execution with one work item per selection. Capacity counts retained * selections, including terminal selections awaiting forget. Snapshots must detach mutable fields, * preserve subtype/data, and be nonblocking, side-effect-free, and non-reentrant. */ @@ -40,7 +37,7 @@ public final class MemoryRoutedWorkStore implements R private final Condition changed = lock.newCondition(); private final int capacity; private final UnaryOperator snapshots; - private final Map> selections = new HashMap<>(); + private final Map> selections = new HashMap<>(); private final Map sourceOwners = new HashMap<>(); private final Map> work = new HashMap<>(); private final Map>> ready = new HashMap<>(); @@ -83,13 +80,13 @@ public Routed record(Assignment assignment) { lock.lock(); try { requireReady(); - Selection existing = selections.get(assignment.selection()); + Entry existing = selections.get(assignment.selection()); if (existing != null) { if (!existing.sources.equals(assignment.sources()) || - !existing.initial.destination.equals(assignment.destination().name())) { - throw invalid("An existing selection cannot change sources or its initial destination"); + !existing.destination.equals(assignment.destination().name())) { + throw invalid("An existing selection cannot change sources or destination"); } - return projection(existing.initial, existing.initial.original); + return projection(existing, existing.original); } if (assignment.sources().stream().anyMatch(sourceOwners::containsKey)) { throw invalid("A source already belongs to another routed selection"); @@ -97,13 +94,13 @@ public Routed record(Assignment assignment) { if (selections.size() >= capacity) { throw new OutboundStorageException(Role.ROUTED_WORK, CAPACITY_EXCEEDED, "Memory routed selection capacity reached"); } - Selection selection = new Selection<>(assignment.selection(), assignment.sources()); - Entry entry = entry(selection, assignment.destination()); + Entry entry = new Entry<>(new WorkId(UUID.randomUUID()), assignment.selection(), assignment.sources(), + assignment.destination().name(), snapshot(assignment.destination().message())); final Routed result = projection(entry, entry.original); - selection.initial = entry; - selections.put(selection.id, selection); - selection.sources.forEach(source -> sourceOwners.put(source, selection.id)); - publish(entry); + selections.put(entry.selection, entry); + entry.sources.forEach(source -> sourceOwners.put(source, entry.selection)); + work.put(entry.id, entry); + enqueue(entry); return result; } finally { lock.unlock(); @@ -165,55 +162,24 @@ public void release(Routed taken) { } @Override - public Routed transfer(WorkId previous, Destination destination) { - Objects.requireNonNull(previous, "previous"); - Objects.requireNonNull(destination, "destination"); - lock.lock(); - try { - requireReady(); - Entry entry = requireEntry(previous); - if (entry.phase == Phase.TRANSFERRED) { - Entry successor = requireEntry(entry.successor); - if (!successor.destination.equals(destination.name())) { - throw invalid("A transfer retry cannot change the successor destination"); - } - return projection(successor, successor.original); - } - if (entry.phase != Phase.TAKEN) { - throw invalid("Only taken destination work can be transferred"); - } - Entry successor = entry(entry.selection, destination); - final Routed result = projection(successor, successor.original); - entry.phase = Phase.TRANSFERRED; - entry.successor = successor.id; - clearExecution(entry); - publish(successor); - return result; - } finally { - lock.unlock(); - } - } - - @Override - public Optional complete(WorkId id) { + public Completed complete(WorkId id) { Objects.requireNonNull(id, "id"); lock.lock(); try { requireReady(); Entry entry = requireEntry(id); - if (entry.phase == Phase.TRANSFERRED) { - return Optional.empty(); - } if (entry.phase == Phase.COMPLETED) { - return Optional.of(entry.selection.completed); + return entry.completed; } if (entry.phase != Phase.TAKEN) { throw invalid("Only taken destination work can complete"); } - entry.selection.completed = new Completed(entry.selection.id, entry.selection.sources); + entry.completed = new Completed(entry.selection, entry.sources); entry.phase = Phase.COMPLETED; - clearExecution(entry); - return Optional.of(entry.selection.completed); + entry.message = null; + entry.inFlight = null; + entry.lastReturned = null; + return entry.completed; } finally { lock.unlock(); } @@ -225,15 +191,15 @@ public void forget(SelectionId id) { lock.lock(); try { requireReady(); - Selection selection = selections.get(id); - if (selection == null) { + Entry entry = selections.get(id); + if (entry == null) { return; } - if (selection.completed == null) { + if (entry.completed == null) { throw invalid("Nonterminal routed work cannot be forgotten"); } - selection.history.forEach(work::remove); - selection.sources.forEach(sourceOwners::remove); + work.remove(entry.id); + entry.sources.forEach(sourceOwners::remove); selections.remove(id); } finally { lock.unlock(); @@ -255,23 +221,13 @@ public void close() { } } - private Entry entry(Selection selection, Destination destination) { - return new Entry<>(new WorkId(UUID.randomUUID()), selection, destination.name(), snapshot(destination.message())); - } - - private void publish(Entry entry) { - work.put(entry.id, entry); - entry.selection.history.add(entry.id); - enqueue(entry); - } - private void enqueue(Entry entry) { ready.computeIfAbsent(entry.destination, ignored -> new ArrayDeque<>()).addLast(entry); changed.signalAll(); } private Routed projection(Entry entry, M message) { - return new Routed<>(entry.id, entry.selection.id, entry.destination, snapshot(message)); + return new Routed<>(entry.id, entry.selection, entry.destination, snapshot(message)); } private M snapshot(M message) { @@ -288,12 +244,6 @@ private M snapshot(M message) { return copy; } - private void clearExecution(Entry entry) { - entry.message = null; - entry.inFlight = null; - entry.lastReturned = null; - } - private void requireReady() { if (state != State.READY) { throw new OutboundStorageException(Role.ROUTED_WORK, UNAVAILABLE, "Memory routed-work store is not ready"); @@ -332,36 +282,25 @@ private static long timeoutNanos(Duration timeout) { } private enum Phase { - QUEUED, TAKEN, TRANSFERRED, COMPLETED - } - - private static final class Selection { - private final SelectionId id; - private final Set sources; - private final List history = new ArrayList<>(); - private Entry initial; - private Completed completed; - - private Selection(SelectionId id, Set sources) { - this.id = id; - this.sources = sources; - } + QUEUED, TAKEN, COMPLETED } private static final class Entry { private final WorkId id; - private final Selection selection; + private final SelectionId selection; + private final Set sources; private final String destination; private final M original; private M message; private Phase phase = Phase.QUEUED; - private WorkId successor; + private Completed completed; private Routed inFlight; private Routed lastReturned; - private Entry(WorkId id, Selection selection, String destination, M original) { + private Entry(WorkId id, SelectionId selection, Set sources, String destination, M original) { this.id = id; this.selection = selection; + this.sources = sources; this.destination = destination; this.original = original; this.message = original; diff --git a/sendium-core/src/test/java/gr/cytech/sendium/core/outbound/DefaultOutboundCoordinatorTest.java b/sendium-core/src/test/java/gr/cytech/sendium/core/outbound/DefaultOutboundCoordinatorTest.java new file mode 100644 index 00000000..1dc813fe --- /dev/null +++ b/sendium-core/src/test/java/gr/cytech/sendium/core/outbound/DefaultOutboundCoordinatorTest.java @@ -0,0 +1,607 @@ +package gr.cytech.sendium.core.outbound; + +import gr.cytech.sendium.core.message.StandardMessage; +import gr.cytech.sendium.core.outbound.OutboundWork.Completed; +import gr.cytech.sendium.core.outbound.OutboundWork.Destination; +import gr.cytech.sendium.core.outbound.OutboundWork.Routed; +import gr.cytech.sendium.core.outbound.OutboundWork.Selected; +import gr.cytech.sendium.core.outbound.OutboundWork.SelectionId; +import gr.cytech.sendium.core.outbound.OutboundWork.SourceId; +import gr.cytech.sendium.core.storage.OutboundStage; +import gr.cytech.sendium.core.storage.OutboundStorageException; +import gr.cytech.sendium.core.storage.SelectedRouterStore; +import gr.cytech.sendium.core.storage.memory.MemoryPendingMessageStore; +import gr.cytech.sendium.core.storage.memory.MemoryRoutedWorkStore; +import gr.cytech.sendium.core.storage.memory.MemorySelectedRouterStore; +import org.assertj.core.api.ThrowableAssert.ThrowingCallable; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.Timeout; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; + +import java.time.Duration; +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; +import java.util.Set; +import java.util.UUID; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionStage; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicReference; + +import static gr.cytech.sendium.core.storage.OutboundStorageException.Reason.CAPACITY_EXCEEDED; +import static gr.cytech.sendium.core.storage.OutboundStorageException.Reason.INVALID_TRANSITION; +import static gr.cytech.sendium.core.storage.OutboundStorageException.Reason.UNAVAILABLE; +import static gr.cytech.sendium.core.storage.OutboundStorageException.Reason.UNSUPPORTED; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.doCallRealMethod; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.inOrder; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoInteractions; +import static org.mockito.Mockito.when; + +@Timeout(10) +class DefaultOutboundCoordinatorTest { + @Test + void constructionIsPassiveAndStartupOwnsStageOrdering() { + try (var h = new Harness()) { + verifyNoInteractions(h.pending, h.router, h.routed); + fails(() -> h.coordinator.admit(source(), new Payload("one")), UNAVAILABLE); + h.start(); + h.coordinator.start(); + var order = inOrder(h.pending, h.router, h.routed); + order.verify(h.pending).open(); + order.verify(h.router).open(); + order.verify(h.routed).open(); + h.coordinator.quiesce(); + fails(h.coordinator::start, INVALID_TRANSITION); + } + } + + @ParameterizedTest + @ValueSource(booleans = {false, true}) + void failedStartupClosesEveryAttemptedStageAndPreservesCleanupErrors(boolean error) { + try (var h = new Harness()) { + Throwable opening = error ? new AssertionError("open failed") : new IllegalStateException("open failed"); + var closing = new IllegalStateException("close failed"); + doThrow(opening).when(h.routed).open(); + doThrow(closing).when(h.router).close(); + assertThatThrownBy(h.coordinator::start).isSameAs(opening); + assertThat(opening.getSuppressed()).containsExactly(closing); + var order = inOrder(h.routed, h.router, h.pending); + order.verify(h.routed).close(); + order.verify(h.router).close(); + order.verify(h.pending).close(); + fails(h.coordinator::start, INVALID_TRANSITION); + } + } + + @Test + void durableOrAlreadyOpenedStagesAreNotSilentlyAdopted() { + try (var h = new Harness()) { + when(h.routed.status()).thenReturn(new OutboundStage.Status("file", true, OutboundStage.State.NEW)); + fails(h.coordinator::start, UNSUPPORTED); + verify(h.routed, never()).open(); + verify(h.routed, never()).close(); + verify(h.router).close(); + verify(h.pending).close(); + } + try (var h = new Harness()) { + h.pending.open(); + fails(h.coordinator::start, INVALID_TRANSITION); + verify(h.pending, never()).close(); + h.pending.close(); + } + } + + @Test + void successfulHandoffCleansUpInOrderAndLateCompletionCannotRecreateWork() throws Exception { + try (var h = new Harness().start()) { + var delivery = h.delivery(); + h.coordinator.complete(new OutboundWork.WorkId(UUID.randomUUID()), new CompletableFuture<>()) + .toCompletableFuture().join(); + assertThat(h.pending.find(delivery.source())).isPresent(); + CompletableFuture handoff = new CompletableFuture<>(); + CompletionStage result = h.coordinator.complete(delivery.work.id(), handoff); + assertThat(result.toCompletableFuture()).isNotDone(); + assertThat(h.pending.find(delivery.source())).isPresent(); + assertThat((Object) h.coordinator.complete(delivery.work.id(), CompletableFuture.completedStage(null))).isSameAs(result); + fails(() -> h.coordinator.returnToDestination(delivery.work), INVALID_TRANSITION); + handoff.complete(null); + result.toCompletableFuture().join(); + var order = inOrder(h.routed, h.pending, h.router); + order.verify(h.routed).complete(delivery.work.id()); + order.verify(h.pending).complete(delivery.selected.sources()); + order.verify(h.router).complete(delivery.selected.id()); + order.verify(h.routed).forget(delivery.selected.id()); + assertThat(h.pending.find(delivery.source())).isEmpty(); + h.coordinator.complete(delivery.work.id(), new CompletableFuture<>()).toCompletableFuture().join(); + verify(h.routed, times(1)).complete(delivery.work.id()); + fails(() -> h.coordinator.route(delivery.selected, destination("A", "stale")), INVALID_TRANSITION); + fails(() -> h.coordinator.returnToDestination(delivery.work), INVALID_TRANSITION); + } + } + + @ParameterizedTest + @ValueSource(booleans = {false, true}) + void failedOrCancelledHandoffRetainsOwnershipForHandoffOnlyRetry(boolean cancel) throws Exception { + try (var h = new Harness().start()) { + var delivery = h.delivery(); + var handoff = new CompletableFuture(); + var result = h.coordinator.complete(delivery.work.id(), handoff); + if (cancel) { + handoff.cancel(false); + } else { + handoff.completeExceptionally(new IllegalStateException("handoff failed")); + } + assertThat(result.toCompletableFuture()).isCompletedExceptionally(); + assertThat(h.pending.find(delivery.source())).isPresent(); + verify(h.routed, never()).complete(any()); + assertThat(h.coordinator.takeForDestination("A", Duration.ZERO)).isEmpty(); + fails(() -> h.coordinator.returnToDestination(delivery.work), INVALID_TRANSITION); + h.coordinator.complete(delivery.work.id(), CompletableFuture.completedStage(null)).toCompletableFuture().join(); + assertThat(h.pending.find(delivery.source())).isEmpty(); + } + } + + @ParameterizedTest + @ValueSource(strings = {"routed", "pending", "selected", "forget"}) + void cleanupFailureCanBeRetriedWithoutAnotherHandoffOrDispatch(String boundary) throws Exception { + try (var h = new Harness().start()) { + var delivery = h.delivery(); + var failure = new IllegalStateException("cleanup failed"); + switch (boundary) { + case "routed" -> doThrow(failure).when(h.routed).complete(delivery.work.id()); + case "pending" -> doThrow(failure).when(h.pending).complete(delivery.selected.sources()); + case "selected" -> doThrow(failure).when(h.router).complete(delivery.selected.id()); + case "forget" -> doThrow(failure).when(h.routed).forget(delivery.selected.id()); + default -> throw new AssertionError(boundary); + } + var result = h.coordinator.complete(delivery.work.id(), CompletableFuture.completedStage(null)); + assertThat(result.toCompletableFuture()).isCompletedExceptionally(); + fails(h.coordinator::close, UNAVAILABLE); + doCallRealMethod().when(h.routed).complete(delivery.work.id()); + doCallRealMethod().when(h.pending).complete(delivery.selected.sources()); + doCallRealMethod().when(h.router).complete(delivery.selected.id()); + doCallRealMethod().when(h.routed).forget(delivery.selected.id()); + h.coordinator.complete(delivery.work.id(), new CompletableFuture<>()).toCompletableFuture().join(); + assertThat(h.pending.find(delivery.source())).isEmpty(); + verify(h.routed, never()).release(any()); + } + } + + @Test + void cleanupErrorsSettleTheCompletionStageRatherThanLeavingItPending() throws Exception { + try (var h = new Harness().start()) { + var delivery = h.delivery(); + doThrow(new AssertionError("cleanup error")).when(h.pending).complete(delivery.selected.sources()); + var result = h.coordinator.complete(delivery.work.id(), CompletableFuture.completedStage(null)); + assertThat(result.toCompletableFuture()).isCompletedExceptionally(); + assertThat(h.pending.find(delivery.source())).isPresent(); + doCallRealMethod().when(h.pending).complete(delivery.selected.sources()); + h.coordinator.complete(delivery.work.id(), new CompletableFuture<>()).toCompletableFuture().join(); + } + } + + @Test + void completionResultCannotRemoveUnrelatedSources() throws Exception { + try (var h = new Harness().start()) { + var delivery = h.delivery(); + var unrelated = source(); + h.coordinator.admit(unrelated, new Payload("unrelated")); + doReturn(new Completed(delivery.selected.id(), Set.of(unrelated))) + .when(h.routed).complete(delivery.work.id()); + var completion = h.coordinator.complete(delivery.work.id(), CompletableFuture.completedStage(null)); + assertThat(completion.toCompletableFuture()).isCompletedExceptionally(); + verify(h.pending, never()).complete(any()); + assertThat(h.pending.find(unrelated)).isPresent(); + assertThat(h.pending.find(delivery.source())).isPresent(); + doCallRealMethod().when(h.routed).complete(delivery.work.id()); + h.coordinator.complete(delivery.work.id(), new CompletableFuture<>()).toCompletableFuture().join(); + assertThat(h.pending.find(unrelated)).isPresent(); + } + } + + @Test + void returnedFutureCannotCancelOwnedCleanupAndCallbacksRunOutsideTheLock() throws Exception { + try (var h = new Harness().start(); var executor = Executors.newVirtualThreadPerTaskExecutor()) { + var delivery = h.delivery(); + var handoff = new CompletableFuture(); + var completion = h.coordinator.complete(delivery.work.id(), handoff); + completion.toCompletableFuture().cancel(false); + var nextSource = source(); + var callback = completion.thenRun(() -> { + try { + executor.submit(() -> h.coordinator.admit(nextSource, new Payload("next"))).get(1, TimeUnit.SECONDS); + } catch (Exception failure) { + throw new IllegalStateException(failure); + } + }); + handoff.complete(null); + callback.toCompletableFuture().get(2, TimeUnit.SECONDS); + assertThat(h.pending.find(delivery.source())).isEmpty(); + assertThat(h.pending.find(nextSource)).isPresent(); + } + } + + @ParameterizedTest + @ValueSource(strings = {"record", "mark"}) + void routingRetryReconcilesPublishedDestinationWithoutRoutingAgain(String boundary) throws Exception { + try (var h = new Harness().start()) { + var selected = h.select("one"); + if (boundary.equals("record")) { + AtomicBoolean first = new AtomicBoolean(true); + doAnswer(call -> { + Object result = call.callRealMethod(); + if (first.getAndSet(false)) { + throw new IllegalStateException("record acknowledged late"); + } + return result; + }).when(h.routed).record(any()); + } else { + doThrow(new IllegalStateException("mark failed")).when(h.router).markRouted(selected.id()); + } + assertThatThrownBy(() -> h.coordinator.route(selected, destination("A", "chosen"))) + .isInstanceOf(IllegalStateException.class); + fails(() -> h.coordinator.returnToRouter(selected), INVALID_TRANSITION); + fails(() -> h.coordinator.route(selected, destination("B", "different")), INVALID_TRANSITION); + fails(() -> h.coordinator.takeForDestination("A", Duration.ZERO), UNAVAILABLE); + doCallRealMethod().when(h.router).markRouted(selected.id()); + var receipt = h.coordinator.route(selected, destination("A", "retry payload")); + var taken = h.coordinator.takeForDestination("A", Duration.ZERO).orElseThrow(); + assertThat(taken.id()).isEqualTo(receipt.id()); + assertThat(taken.message().body).isEqualTo("chosen"); + h.coordinator.complete(taken.id(), CompletableFuture.completedStage(null)).toCompletableFuture().join(); + } + } + + @Test + void destinationCapacityBackpressureDoesNotPreventExistingWorkFromCompleting() throws Exception { + try (var h = new Harness(1).start()) { + var first = h.select("first"); + h.coordinator.route(first, destination("A", "first")); + var second = h.select("second"); + fails(() -> h.coordinator.route(second, destination("A", "second")), CAPACITY_EXCEEDED); + var taken = h.coordinator.takeForDestination("A", Duration.ZERO).orElseThrow(); + assertThat(taken.selection()).isEqualTo(first.id()); + h.coordinator.complete(taken.id(), CompletableFuture.completedStage(null)).toCompletableFuture().join(); + h.coordinator.route(second, destination("A", "retry")); + taken = h.coordinator.takeForDestination("A", Duration.ZERO).orElseThrow(); + assertThat(taken.message().body).isEqualTo("second"); + h.coordinator.complete(taken.id(), CompletableFuture.completedStage(null)).toCompletableFuture().join(); + } + } + + @Test + void queuedReceiptsCannotCompleteBeforeTheyAreTaken() throws Exception { + try (var h = new Harness().start()) { + var selected = h.select("one"); + var receipt = h.coordinator.route(selected, destination("A", "one")); + fails(() -> h.coordinator.complete(receipt.id(), CompletableFuture.completedStage(null)), INVALID_TRANSITION); + var taken = h.coordinator.takeForDestination("A", Duration.ZERO).orElseThrow(); + h.coordinator.complete(taken.id(), CompletableFuture.completedStage(null)).toCompletableFuture().join(); + } + } + + @Test + void unexposedTakeSurvivesFailureWhileReturningAnUnfinishedRoute() throws Exception { + try (var h = new Harness().start()) { + var selected = h.select("one"); + doThrow(new IllegalStateException("mark failed")).when(h.router).markRouted(selected.id()); + assertThatThrownBy(() -> h.coordinator.route(selected, destination("A", "one"))).isInstanceOf(IllegalStateException.class); + doThrow(new IllegalStateException("return failed")).when(h.routed).release(any()); + assertThatThrownBy(() -> h.coordinator.takeForDestination("A", Duration.ZERO)).isInstanceOf(IllegalStateException.class); + doCallRealMethod().when(h.router).markRouted(selected.id()); + doCallRealMethod().when(h.routed).release(any()); + h.coordinator.route(selected, destination("A", "one")); + var taken = h.coordinator.takeForDestination("A", Duration.ZERO).orElseThrow(); + h.coordinator.complete(taken.id(), CompletableFuture.completedStage(null)).toCompletableFuture().join(); + } + } + + @Test + void routerAndDestinationReturnsPreserveMutationsAndRejectForgedTakes() throws Exception { + try (var h = new Harness().start()) { + var selected = h.select("original"); + selected.message().body = "filtered"; + h.coordinator.returnToRouter(selected); + var next = h.coordinator.takeForRouting(Duration.ZERO).orElseThrow(); + h.coordinator.returnToRouter(selected); + assertThat(next.message().body).isEqualTo("filtered"); + assertThat(h.coordinator.takeForRouting(Duration.ZERO)).isEmpty(); + fails(() -> h.coordinator.route(new Selected<>(next.id(), next.sources(), next.message()), destination("A", "fake")), + INVALID_TRANSITION); + h.coordinator.route(next, new Destination<>("A", next.message())); + var first = h.coordinator.takeForDestination("A", Duration.ZERO).orElseThrow(); + first.message().body = "retry"; + h.coordinator.returnToDestination(first); + var second = h.coordinator.takeForDestination("A", Duration.ZERO).orElseThrow(); + h.coordinator.returnToDestination(first); + assertThat(second.message().body).isEqualTo("retry"); + assertThat(h.coordinator.takeForDestination("A", Duration.ZERO)).isEmpty(); + h.coordinator.complete(second.id(), CompletableFuture.completedStage(null)).toCompletableFuture().join(); + } + } + + @Test + void discardWaitsForHandoffAndCannotBeUsedAfterAssignment() throws Exception { + try (var h = new Harness().start()) { + var selected = h.select("drop"); + var handoff = new CompletableFuture(); + var result = h.coordinator.discard(selected.id(), handoff); + assertThat(h.pending.find(selected.sources().iterator().next())).isPresent(); + fails(() -> h.coordinator.returnToRouter(selected), INVALID_TRANSITION); + handoff.completeExceptionally(new IllegalStateException("handoff failed")); + assertThat(result.toCompletableFuture()).isCompletedExceptionally(); + h.coordinator.discard(selected.id(), CompletableFuture.completedStage(null)).toCompletableFuture().join(); + h.coordinator.discard(selected.id(), new CompletableFuture<>()).toCompletableFuture().join(); + verify(h.routed, never()).record(any()); + var delivery = h.delivery(); + fails(() -> h.coordinator.discard(delivery.selected.id(), CompletableFuture.completedStage(null)), INVALID_TRANSITION); + h.coordinator.complete(delivery.work.id(), CompletableFuture.completedStage(null)).toCompletableFuture().join(); + } + } + + @Test + @SuppressWarnings("unchecked") + void multiSourceCleanupCanRetryAfterPartialRemoval() throws Exception { + var pending = spy(new MemoryPendingMessageStore(2, Payload::copy)); + SelectedRouterStore router = mock(SelectedRouterStore.class); + var routed = spy(new MemoryRoutedWorkStore(1, Payload::copy)); + when(router.status()).thenReturn(new OutboundStage.Status("test", false, OutboundStage.State.NEW), + new OutboundStage.Status("test", false, OutboundStage.State.READY)); + var sources = Set.of(source(), source()); + var selected = new Selected<>(new SelectionId(UUID.randomUUID()), sources, new Payload("aggregate")); + when(router.take(Duration.ZERO)).thenReturn(Optional.of(selected), Optional.empty()); + try (var coordinator = new DefaultOutboundCoordinator<>(pending, router, routed)) { + coordinator.start(); + sources.forEach(source -> coordinator.admit(source, new Payload("part"))); + coordinator.takeForRouting(Duration.ZERO).orElseThrow(); + coordinator.route(selected, destination("A", "aggregate")); + var taken = coordinator.takeForDestination("A", Duration.ZERO).orElseThrow(); + AtomicBoolean first = new AtomicBoolean(true); + doAnswer(call -> { + if (first.getAndSet(false)) { + pending.complete(Set.of(sources.iterator().next())); + throw new IllegalStateException("partial source completion"); + } + return call.callRealMethod(); + }).when(pending).complete(sources); + var result = coordinator.complete(taken.id(), CompletableFuture.completedStage(null)); + assertThat(result.toCompletableFuture()).isCompletedExceptionally(); + assertThat(sources.stream().filter(source -> pending.find(source).isPresent()).count()).isEqualTo(1); + sources.forEach(source -> coordinator.admit(source, new Payload("duplicate during cleanup"))); + assertThat(sources.stream().filter(source -> pending.find(source).isPresent()).count()).isEqualTo(1); + coordinator.complete(taken.id(), new CompletableFuture<>()).toCompletableFuture().join(); + sources.forEach(source -> assertThat(pending.find(source)).isEmpty()); + verify(routed, times(1)).complete(taken.id()); + verify(pending, times(2)).complete(sources); + } + } + + @ParameterizedTest + @ValueSource(booleans = {false, true}) + void quiescenceWakesWaitingTakesAndKeepsCompletionAvailable(boolean destination) throws Exception { + try (var h = new Harness().start(); var executor = Executors.newVirtualThreadPerTaskExecutor()) { + var delivery = h.delivery(); + AtomicReference thread = new AtomicReference<>(); + var waiting = executor.submit(() -> { + thread.set(Thread.currentThread()); + return destination ? h.coordinator.takeForDestination("empty", Duration.ofSeconds(3)) : + h.coordinator.takeForRouting(Duration.ofSeconds(3)); + }); + awaitWaiting(thread); + h.coordinator.quiesce(); + assertThatThrownBy(() -> waiting.get(1, TimeUnit.SECONDS)).hasCauseInstanceOf(OutboundStorageException.class); + fails(() -> h.coordinator.admit(source(), new Payload("new")), UNAVAILABLE); + fails(() -> h.coordinator.selectAndStage(1), UNAVAILABLE); + h.coordinator.complete(delivery.work.id(), CompletableFuture.completedStage(null)).toCompletableFuture().join(); + } + } + + @Test + void publicationWakesCoordinatorTakes() throws Exception { + try (var h = new Harness().start(); var executor = Executors.newVirtualThreadPerTaskExecutor()) { + AtomicReference thread = new AtomicReference<>(); + var waiting = executor.submit(() -> { + thread.set(Thread.currentThread()); + return h.coordinator.takeForRouting(Duration.ofSeconds(3)); + }); + awaitWaiting(thread); + h.coordinator.admit(source(), new Payload("one")); + h.coordinator.selectAndStage(1); + var selected = waiting.get(1, TimeUnit.SECONDS).orElseThrow(); + thread.set(null); + var destination = executor.submit(() -> { + thread.set(Thread.currentThread()); + return h.coordinator.takeForDestination("A", Duration.ofSeconds(3)); + }); + awaitWaiting(thread); + h.coordinator.route(selected, destination("A", "one")); + var taken = destination.get(1, TimeUnit.SECONDS).orElseThrow(); + h.coordinator.complete(taken.id(), CompletableFuture.completedStage(null)).toCompletableFuture().join(); + } + } + + @Test + void closeDrainsWorkBackToItsStageWithoutCompletingSources() throws Exception { + try (var h = new Harness().start()) { + var delivery = h.delivery(); + var routing = h.select("not routed"); + h.coordinator.quiesce(); + h.coordinator.close(); + var order = inOrder(h.router, h.routed, h.pending); + order.verify(h.router).release(routing); + order.verify(h.routed).release(delivery.work); + order.verify(h.routed).close(); + order.verify(h.router).close(); + order.verify(h.pending).close(); + verify(h.pending, never()).complete(any()); + fails(h.coordinator::start, INVALID_TRANSITION); + } + } + + @Test + void closeWaitsForHandoffResolutionAndRemainsRetryableAfterDrainFailure() throws Exception { + try (var h = new Harness().start()) { + var delivery = h.delivery(); + var handoff = new CompletableFuture(); + var completed = h.coordinator.complete(delivery.work.id(), handoff); + fails(h.coordinator::close, UNAVAILABLE); + verify(h.pending, never()).close(); + handoff.complete(null); + completed.toCompletableFuture().join(); + h.coordinator.close(); + } + try (var h = new Harness().start()) { + var delivery = h.delivery(); + doThrow(new IllegalStateException("drain failed")).when(h.routed).release(delivery.work); + assertThatThrownBy(h.coordinator::close).hasMessage("drain failed"); + verify(h.pending, never()).close(); + doCallRealMethod().when(h.routed).release(delivery.work); + h.coordinator.close(); + } + } + + @Test + void concurrentCompletionReportsShareOneHandoffAndOneCleanup() throws Exception { + try (var h = new Harness().start(); var executor = Executors.newVirtualThreadPerTaskExecutor()) { + var delivery = h.delivery(); + var handoff = new CompletableFuture(); + var reports = new ArrayList>>(); + for (int i = 0; i < 12; i++) { + reports.add(executor.submit(() -> h.coordinator.complete(delivery.work.id(), handoff))); + } + var result = reports.getFirst().get(2, TimeUnit.SECONDS); + for (var report : reports) { + assertThat((Object) report.get(2, TimeUnit.SECONDS)).isSameAs(result); + } + verify(h.pending, never()).complete(any()); + handoff.complete(null); + result.toCompletableFuture().join(); + verify(h.pending, times(1)).complete(delivery.selected.sources()); + verify(h.routed, times(1)).complete(delivery.work.id()); + } + } + + @Test + void interruptedTakeDoesNotConsumeLaterWork() throws Exception { + try (var h = new Harness().start(); var executor = Executors.newVirtualThreadPerTaskExecutor()) { + AtomicReference thread = new AtomicReference<>(); + var waiting = executor.submit(() -> { + thread.set(Thread.currentThread()); + return h.coordinator.takeForRouting(Duration.ofSeconds(3)); + }); + awaitWaiting(thread); + thread.get().interrupt(); + assertThatThrownBy(() -> waiting.get(1, TimeUnit.SECONDS)).hasCauseInstanceOf(InterruptedException.class); + var selected = h.select("after interrupt"); + h.coordinator.discard(selected.id(), CompletableFuture.completedStage(null)).toCompletableFuture().join(); + } + } + + @Test + void closeAttemptsAllStoresAndPreservesFailures() { + try (var h = new Harness().start()) { + var first = new IllegalStateException("routed close"); + var second = new IllegalStateException("router close"); + doThrow(first).when(h.routed).close(); + doThrow(second).when(h.router).close(); + assertThatThrownBy(h.coordinator::close).isSameAs(first); + assertThat(first.getSuppressed()).containsExactly(second); + verify(h.pending).close(); + } + } + + private static void awaitWaiting(AtomicReference thread) throws InterruptedException { + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(1); + while ((thread.get() == null || thread.get().getState() != Thread.State.TIMED_WAITING) && System.nanoTime() < deadline) { + Thread.sleep(1); + } + assertThat(thread.get()).isNotNull(); + assertThat(thread.get().getState()).isEqualTo(Thread.State.TIMED_WAITING); + } + + private static Destination destination(String name, String body) { + return new Destination<>(name, new Payload(body)); + } + + private static SourceId source() { + return new SourceId(UUID.randomUUID()); + } + + private static void fails(ThrowingCallable operation, OutboundStorageException.Reason reason) { + assertThatThrownBy(operation).isInstanceOfSatisfying(OutboundStorageException.class, + failure -> assertThat(failure.reason()).isEqualTo(reason)); + } + + private record Delivery(Selected selected, Routed work) { + private SourceId source() { + return selected.sources().iterator().next(); + } + } + + private static final class Harness implements AutoCloseable { + private final MemoryPendingMessageStore pending = spy(new MemoryPendingMessageStore<>(4, Payload::copy)); + private final MemorySelectedRouterStore router = spy(new MemorySelectedRouterStore<>(pending, 4)); + private final MemoryRoutedWorkStore routed; + private final DefaultOutboundCoordinator coordinator; + + private Harness() { + this(4); + } + + private Harness(int routedCapacity) { + routed = spy(new MemoryRoutedWorkStore<>(routedCapacity, Payload::copy)); + coordinator = new DefaultOutboundCoordinator<>(pending, router, routed); + } + + private Harness start() { + coordinator.start(); + return this; + } + + private Selected select(String body) throws InterruptedException { + coordinator.admit(source(), new Payload(body)); + coordinator.selectAndStage(1); + return coordinator.takeForRouting(Duration.ZERO).orElseThrow(); + } + + private Delivery delivery() throws InterruptedException { + var selected = select("original"); + coordinator.route(selected, new Destination<>("A", selected.message())); + return new Delivery(selected, coordinator.takeForDestination("A", Duration.ZERO).orElseThrow()); + } + + @Override + public void close() { + coordinator.close(); + } + } + + private static final class Payload extends StandardMessage { + private final List tags = new ArrayList<>(); + + private Payload(String body) { + this.body = body; + } + + private Payload copy() { + var copy = new Payload(body); + copy.tags.addAll(tags); + return copy; + } + } +} diff --git a/sendium-core/src/test/java/gr/cytech/sendium/core/storage/memory/MemoryRoutedWorkStoreTest.java b/sendium-core/src/test/java/gr/cytech/sendium/core/storage/memory/MemoryRoutedWorkStoreTest.java index 187f8147..fbbcc263 100644 --- a/sendium-core/src/test/java/gr/cytech/sendium/core/storage/memory/MemoryRoutedWorkStoreTest.java +++ b/sendium-core/src/test/java/gr/cytech/sendium/core/storage/memory/MemoryRoutedWorkStoreTest.java @@ -104,61 +104,25 @@ void returnPreservesExecutionStateWithoutChangingAssignmentOrOverwritingANewerTa } @Test - void forwardingTransfersTheSlotAndLateCallbacksCannotCompleteTheSuccessor() throws Exception { - try (var store = store(1)) { - var assignment = assignment("A", "original"); - var original = store.record(assignment); - fails(() -> store.transfer(original.id(), destination("B", "premature")), INVALID_TRANSITION); - fails(() -> store.complete(original.id()), INVALID_TRANSITION); - var previous = store.take("A", Duration.ZERO).orElseThrow(); - var next = store.transfer(previous.id(), destination("B", "forwarded")); - assertThat(next.id()).isNotEqualTo(previous.id()); - assertThat(next.selection()).isEqualTo(previous.selection()); - assertThat(store.complete(previous.id())).isEmpty(); - fails(() -> store.release(previous), INVALID_TRANSITION); - fails(() -> store.forget(assignment.selection()), INVALID_TRANSITION); - var replay = store.transfer(previous.id(), destination("B", "different retry payload")); - assertThat(replay.id()).isEqualTo(next.id()); - assertThat(replay.message().body).isEqualTo("forwarded"); - fails(() -> store.transfer(previous.id(), destination("C", "conflict")), INVALID_TRANSITION); - assertThat(store.record(assignment).id()).isEqualTo(original.id()); - assertThat(store.take("A", Duration.ZERO)).isEmpty(); - assertThat(store.take("B", Duration.ZERO).orElseThrow().id()).isEqualTo(next.id()); - assertThat(store.take("B", Duration.ZERO)).isEmpty(); - var completed = store.complete(next.id()).orElseThrow(); - assertThat(completed.sources()).isEqualTo(assignment.sources()); - assertThat(store.complete(previous.id())).isEmpty(); - assertThat(store.complete(next.id())).contains(completed); - store.record(assignment); - store.transfer(previous.id(), destination("B", "late retry")); - assertThat(store.take("A", Duration.ZERO)).isEmpty(); - assertThat(store.take("B", Duration.ZERO)).isEmpty(); - } - } - - @Test - void cleanupRetainsAllSourceIdsAndHistoryUntilForget() throws Exception { + void cleanupRetainsAllSourceIdsAndTerminalResultUntilForget() throws Exception { try (var store = store(1)) { var sources = Set.of(source(), source()); var assignment = new Assignment<>(selection(), sources, destination("A", "multipart aggregate")); - var first = store.record(assignment); - store.take("A", Duration.ZERO).orElseThrow(); - var second = store.transfer(first.id(), destination("B", "second")); - store.take("B", Duration.ZERO).orElseThrow(); - var third = store.transfer(second.id(), destination("A", "third")); - assertThat(store.transfer(first.id(), destination("B", "retry")).id()).isEqualTo(second.id()); - assertThat(store.take("B", Duration.ZERO)).isEmpty(); - assertThat(store.take("A", Duration.ZERO).orElseThrow().id()).isEqualTo(third.id()); - var completed = store.complete(third.id()).orElseThrow(); + var recorded = store.record(assignment); + fails(() -> store.complete(recorded.id()), INVALID_TRANSITION); + var taken = store.take("A", Duration.ZERO).orElseThrow(); + fails(() -> store.forget(assignment.selection()), INVALID_TRANSITION); + var completed = store.complete(taken.id()); assertThat(completed.sources()).isEqualTo(sources); - assertThat(store.complete(first.id())).isEmpty(); - assertThat(store.complete(second.id())).isEmpty(); + assertThat(completed.selection()).isEqualTo(assignment.selection()); fails(() -> store.record(assignment("B", "capacity still held")), CAPACITY_EXCEEDED); - assertThat(store.complete(third.id())).contains(completed); + assertThat(store.complete(taken.id())).isEqualTo(completed); + assertThat(store.record(assignment).id()).isEqualTo(taken.id()); + assertThat(store.take("A", Duration.ZERO)).isEmpty(); + fails(() -> store.release(taken), INVALID_TRANSITION); store.forget(assignment.selection()); store.forget(assignment.selection()); - fails(() -> store.complete(first.id()), INVALID_TRANSITION); - fails(() -> store.complete(third.id()), INVALID_TRANSITION); + fails(() -> store.complete(taken.id()), INVALID_TRANSITION); var replacement = new Assignment<>(selection(), sources, destination("A", "new selection")); store.record(replacement); assertThat(store.take("A", Duration.ZERO).orElseThrow().selection()).isEqualTo(replacement.selection()); @@ -197,30 +161,6 @@ void failedRecordSnapshotsPublishNothingAndLeaveCapacityAvailable(int failAt) th } } - @ParameterizedTest - @ValueSource(ints = {1, 2}) - void failedTransferSnapshotsLeaveThePreviousWorkOwned(int failAt) throws Exception { - AtomicInteger calls = new AtomicInteger(); - AtomicBoolean enabled = new AtomicBoolean(false); - try (var store = new MemoryRoutedWorkStore(1, msg -> { - if (enabled.get() && calls.incrementAndGet() == failAt) { - throw new IllegalStateException("snapshot failure"); - } - return msg.copy(); - })) { - store.open(); - store.record(assignment("A", "original")); - var taken = store.take("A", Duration.ZERO).orElseThrow(); - enabled.set(true); - fails(() -> store.transfer(taken.id(), destination("B", "next")), UNAVAILABLE); - assertThat(store.take("B", Duration.ZERO)).isEmpty(); - store.release(taken); - assertThat(store.take("A", Duration.ZERO).orElseThrow().id()).isEqualTo(taken.id()); - var next = store.transfer(taken.id(), destination("B", "next")); - assertThat(store.take("B", Duration.ZERO).orElseThrow().id()).isEqualTo(next.id()); - } - } - @Test void failedTakeAndReturnSnapshotsRetainTheCurrentWork() throws Exception { AtomicBoolean enabled = new AtomicBoolean(false); @@ -275,29 +215,39 @@ void concurrentRecordAndTakeCallsKeepOneOwner() throws Exception { } @Test - void racingTransferAndCompletionCannotBothSucceedForTheSameWork() throws Exception { + void racingReturnAndCompletionCannotBothSucceedForTheSameTake() throws Exception { try (var store = store(1); var executor = Executors.newVirtualThreadPerTaskExecutor()) { for (int i = 0; i < 20; i++) { var assignment = assignment("A", "one"); store.record(assignment); var taken = store.take("A", Duration.ZERO).orElseThrow(); - var completion = executor.submit(() -> store.complete(taken.id())); - var transfer = executor.submit(() -> { + var completion = executor.submit(() -> { + try { + store.complete(taken.id()); + return true; + } catch (OutboundStorageException failure) { + assertThat(failure.reason()).isEqualTo(INVALID_TRANSITION); + return false; + } + }); + var retry = executor.submit(() -> { try { - return Optional.of(store.transfer(taken.id(), destination("B", "next"))); + store.release(taken); + return true; } catch (OutboundStorageException failure) { assertThat(failure.reason()).isEqualTo(INVALID_TRANSITION); - return Optional.>empty(); + return false; } }); var completed = completion.get(2, TimeUnit.SECONDS); - var forwarded = transfer.get(2, TimeUnit.SECONDS); - assertThat(completed.isPresent()).isNotEqualTo(forwarded.isPresent()); - if (forwarded.isPresent()) { - var successor = store.take("B", Duration.ZERO).orElseThrow(); - store.complete(successor.id()); + var returned = retry.get(2, TimeUnit.SECONDS); + assertThat(completed).isNotEqualTo(returned); + if (returned) { + var nextTake = store.take("A", Duration.ZERO).orElseThrow(); + assertThat(nextTake.id()).isEqualTo(taken.id()); + store.complete(nextTake.id()); } else { - assertThat(store.take("B", Duration.ZERO)).isEmpty(); + assertThat(store.take("A", Duration.ZERO)).isEmpty(); } store.forget(assignment.selection()); } From 70fb3a744cc4086699430fa0a2f6f56f0c5f147f Mon Sep 17 00:00:00 2001 From: pavlos Date: Tue, 29 Sep 2026 16:58:48 +0300 Subject: [PATCH 06/20] feat(storage): connect held multipart and protocol admission Signed-off-by: pavlos --- docs/04-smpp-configuration.md | 14 +- docs/06-http-api.md | 10 +- docs/09-configuration-reference.md | 3 +- docs/14-outbound-storage.md | 151 +++++- .../app/storage/SmsLifecycleHttpTest.java | 133 ++++++ .../sendium/core/http/KannelResource.java | 27 +- .../outbound/DefaultOutboundCoordinator.java | 26 +- .../core/outbound/OutboundCoordinator.java | 11 + .../sendium/core/outbound/OutboundWork.java | 7 +- .../sendium/core/smpp/server/InEvent.java | 10 + .../server/MessagePartsEventsListener.java | 4 + .../core/smpp/server/MessagePartsHandler.java | 41 +- .../core/smpp/server/SmppServerWorker.java | 157 ++++++- .../core/storage/PendingMessageStore.java | 17 +- .../memory/MemoryPendingMessageStore.java | 119 ++++- .../memory/MemorySelectedRouterStore.java | 4 +- .../StandardOutgoingWorkerHandler.java | 6 + .../sendium/core/http/KannelResourceTest.java | 86 +++- .../DefaultOutboundCoordinatorTest.java | 20 +- .../server/MessagePartsOwnershipTest.java | 101 ++++ .../server/SmppLifecycleAdmissionTest.java | 330 +++++++++++++ .../SmppServerWorkerReassemblyTest.java | 2 +- .../storage/memory/HeldAdmissionTest.java | 444 ++++++++++++++++++ .../StandardOutgoingWorkerAdmissionTest.java | 67 +++ .../java/utils/OutboundIngressFixture.java | 45 ++ 25 files changed, 1751 insertions(+), 84 deletions(-) create mode 100644 sendium-app/src/test/java/gr/cytech/sendium/app/storage/SmsLifecycleHttpTest.java create mode 100644 sendium-core/src/test/java/gr/cytech/sendium/core/smpp/server/MessagePartsOwnershipTest.java create mode 100644 sendium-core/src/test/java/gr/cytech/sendium/core/smpp/server/SmppLifecycleAdmissionTest.java create mode 100644 sendium-core/src/test/java/gr/cytech/sendium/core/storage/memory/HeldAdmissionTest.java create mode 100644 sendium-core/src/test/java/gr/cytech/sendium/routing/StandardOutgoingWorkerAdmissionTest.java create mode 100644 sendium-core/src/test/java/utils/OutboundIngressFixture.java diff --git a/docs/04-smpp-configuration.md b/docs/04-smpp-configuration.md index 8a5f6849..29e4619f 100644 --- a/docs/04-smpp-configuration.md +++ b/docs/04-smpp-configuration.md @@ -59,9 +59,21 @@ All properties below should be prefixed with your instance path. For example, if | `charset.latin1` | `ISO-8859-1` | The default charset mapping for Latin-1. | | `charset.ucs2` | `UCS-2` | The default charset mapping for UCS-2 (data coding `8`). | | `ccat.8bit` | `true` | Use 8-bit reference numbers for Concatenated (multipart) SMS instead of 16-bit. | -| `reassembling.timeoutMillis` | `30000` | Timeout (in ms) to wait for all parts of a concatenated message to arrive before failing. | +| `reassembling.timeoutMillis` | `30000` | Time (in ms) to wait for concatenated-message parts; incomplete groups release their received parts individually on expiry. | | `filters.beforeInsertMessage` | `""` | (Filters Not supported yet) Comma-separated list of filter class names to process messages before queuing. | +### Admission and acknowledgement + +The default server accepts an ingress event into its local queue before sending `submit_sm_resp STATUS_OK`. +An application explicitly binding an outbound coordinator instead admits a complete message, or retains +each concatenated part as held work, before success. Parts are acknowledged individually rather than +waiting for the whole message. Completed/expired reassembly work retains those original source IDs. + +In that coordinated path, capacity rejection returns `STATUS_THROTTLED`, unsupported input returns +`STATUS_SUBMITFAIL`, and other lifecycle/storage failures return `STATUS_SYSERR`. A failure after +acknowledgement during ready publication is retried without another client response. Standalone lifecycle +activation remains deferred until dispatch and completion are connected; see [Outbound Storage Contracts](14-outbound-storage.md). + ## 🧵 Thread Pool Configuration | Property | Default Value | Description | diff --git a/docs/06-http-api.md b/docs/06-http-api.md index 30533a95..d4b8fde1 100644 --- a/docs/06-http-api.md +++ b/docs/06-http-api.md @@ -57,11 +57,11 @@ The API returns standard HTTP status codes along with a plain-text response body | HTTP Status | Meaning | Description | | :--- | :--- | :--- | -| **`202 Accepted`** | **Success** | The message was validated and inserted into Sendium's router queue. The response body contains the unique UUID (serial) of the message. | -| **`400 Bad Request`** | **Error** | Missing a required parameter (`to`, `from`, or `text`). The response body details which parameter is missing. | +| **`202 Accepted`** | **Success** | Admission succeeded through the configured path. The response body contains the unique gateway UUID (serial) of the message. | +| **`400 Bad Request`** | **Error** | Missing a required parameter (`to`, `from`, or `text`), or the selected lifecycle cannot support the submission. | | **`401 Unauthorized`** | **Error** | Invalid or missing credentials. | | **`500 Server Error`** | **Error** | An internal error occurred while parsing or processing the message payload. | -| **`503 Unavailable`** | **Error** | Internal queue admission was interrupted. The message was not queued; the client should retry later. | +| **`503 Unavailable`** | **Error** | Admission is unavailable, at capacity, has an ownership conflict, or was interrupted. The client should retry later. | --- @@ -90,7 +90,9 @@ For a manual installation, replace the environment variables with the HTTP `syst 123e4567-e89b-12d3-a456-426614174000 ``` -`202 Accepted` means Sendium validated the message and inserted it into the in-memory router queue. It does not write an outbound message or DLR row to PostgreSQL, and it does not prove that a viable route exists, that an upstream SMSC accepted the message, or that a handset received it. Durable DLR state is created only after an upstream provider outcome. Local-only Quick Start installations have no outbound route. Check routing configuration, the SMPP client connection, message lifecycle logs, submit response, and delivery receipt for those later stages. +In the default runtime, `202 Accepted` means insertion into the in-memory router queue. An application explicitly supplying the new outbound coordinator receives `202` only after pending admission succeeds; a failing coordinator is never replaced by legacy queue admission. Standalone activation of that lifecycle is deferred until dispatch and completion are connected. See [Outbound Storage Contracts](14-outbound-storage.md). + +Neither memory admission path writes an outbound message or DLR row to PostgreSQL. Acceptance does not prove that a viable route exists, that an upstream SMSC accepted the message, or that a handset received it. Durable DLR state is created only after an upstream provider outcome. Local-only Quick Start installations have no outbound route. Check routing configuration, the SMPP client connection, message lifecycle logs, submit response, and delivery receipt for those later stages. ## Related Documentation diff --git a/docs/09-configuration-reference.md b/docs/09-configuration-reference.md index fdf1f922..3104bba2 100644 --- a/docs/09-configuration-reference.md +++ b/docs/09-configuration-reference.md @@ -65,7 +65,8 @@ Validation runs before Sendium's file-watcher and router/worker startup observer profile is retained for the process lifetime. Startup logs the effective profile and a prominent `NON-DURABLE` warning: accepted messages and in-flight work can be lost on restart. The current runtime pipeline is still memory-backed. All three memory stage stores and the default lifecycle -coordinator exist as library components; ingress/worker wiring is subsequent work in #338. Their +coordinator exist as library components. HTTP/SMPP can bind to an explicitly supplied coordinator; +standalone activation still awaits connected dispatch, completion, and retry/rerouting work in #338. Their capacity bounds are constructor arguments at this stage, not additional operator settings. See [Outbound Storage Contracts](14-outbound-storage.md). diff --git a/docs/14-outbound-storage.md b/docs/14-outbound-storage.md index a4d3b797..15fa772f 100644 --- a/docs/14-outbound-storage.md +++ b/docs/14-outbound-storage.md @@ -4,12 +4,16 @@ This describes the initial contracts for [#338](https://github.com/cytechmobile/ under [#337](https://github.com/cytechmobile/sendium/issues/337). The stage contracts are library APIs, and standalone profile selection, early validation, and startup logging are implemented. All three memory stage stores and the default coordinator are available as explicitly constructed library components. -End-to-end production wiring is subsequent work. Existing ingress and queue behavior remains non-durable. +HTTP/SMPP admission can bind to an explicitly supplied coordinator. Standalone activation remains deferred +until dispatch, completion, retry/rerouting, and shutdown are connected. The default runtime remains non-durable. ## Ownership model ```text -accepted pending source +accepted pending sources + | ready admission, or publishReady after held admission + v +ready pending work | bounded, eligible, priority-aware select-and-stage v selected router work ---- take/release ---- router execution @@ -28,8 +32,9 @@ forget routed terminal bookkeeping Pending sources remain authoritative throughout. Taking work does not delete source or stage ownership. Source IDs, selection IDs, and destination work IDs are distinct opaque UUID-backed types; none is inferred from mutable message equality or a protocol serial. One selected item may retain -several source IDs, allowing later multipart integration without treating an aggregate as a new -independent admission. Detailed multipart admission and reconstruction are owned by #343. +several source IDs. Held admission and ready-work publication retain those sources without treating +an aggregate as a new independent admission. Protocol wiring follows separately; detailed multipart +grouping, deduplication, recovery, and codec contracts are owned by #343. Each routed selection has one destination and one work identity through execution, same-worker retry, and terminal cleanup. Direct worker-to-worker forwarding is outside this abstraction's scope. @@ -91,7 +96,7 @@ These classes are absent from the `sendium-core` artifact; they do not activate | Contract | Responsibility | |---|---| -| `PendingMessageStore` | Record accepted source state under a stable caller-supplied identity and complete sources idempotently. | +| `PendingMessageStore` | Admit ready or held sources, publish prepared work using held source IDs, and complete sources idempotently. | | `SelectedRouterStore` | Own bounded select-and-stage, exclusive runtime takes, return-to-router, and retained selected state. | | `RoutedWorkStore` | Record one destination, own same-destination scheduling/retry, and report terminal source ownership. | | `OutboundCoordinator` | Application-facing admission, transitions, required handoff, source completion, and lifecycle coordination. | @@ -151,8 +156,10 @@ These two classes are plain Java components with no CDI activation or background - `MemoryPendingMessageStore` takes a positive maximum source count and a `UnaryOperator` snapshot function. An optional `Function` supplies absolute - eligibility time at admission; the default makes every admitted source immediately eligible. No new - interpretation of HTTP deferred-delivery or validity fields is introduced by this library step. + eligibility time when ready work is first published. Ordinary `admit` publishes a singleton immediately; + `admitHeld` does not evaluate eligibility or put anything in the selection index. The default time policy + makes published work immediately eligible. No new interpretation of HTTP deferred-delivery or validity + fields is introduced by this library step. - `MemorySelectedRouterStore` takes that pending store, a positive routing capacity, and optionally a `Clock` (UTC by default). Open pending first, then the selected store. A pending instance accepts only one selected-router owner for its lifetime, including after that router closes. @@ -165,21 +172,25 @@ a correct snapshot function belongs to assembly. Functions run under the store l side-effect-free, nonblocking, and must not reenter these stores. The standalone message mapper is supplied when ingress is wired; this step does not serialize messages or define a durable codec. -Pending admission is idempotent by source ID while that source exists, even at capacity. New source +Pending admission is idempotent by source ID while that source exists, even at capacity. Repeating either +admission method preserves the first accepted payload and admission disposition: an `admit` retry cannot +make a held source ready, and an `admitHeld` retry cannot withdraw ordinarily admitted work. New source admission fails with `CAPACITY_EXCEEDED` when the bound is reached. Admission or snapshot failure does not reserve a source slot. A single-source `find` returns an isolated value for backend consumers; there is no all-backlog loading API. Pending completion remains the caller's terminal-processing decision, not a side effect of selection, take, or routing. -Selection uses priority-indexed ordered sets maintained at admission. It inspects due candidates rather +Selection uses priority-indexed ordered sets maintained at ready publication. It inspects due candidates rather than copying or sorting the whole backlog on each refill. Higher numeric priorities select first; within -a priority, earlier eligibility comes first, then admission order. Future work in a higher-priority group +a priority, earlier eligibility comes first, then ready-publication order (admission order for ordinary +singletons). Future work in a higher-priority group does not block eligible work at a lower priority. A batch captures one clock instant. This is an in-process scheduling policy, not a durable global-order guarantee. `selectAndStage(limit)` publishes at most the smaller of the requested limit and free routing slots. -Each selected source is removed from the eligibility index but retained in pending storage. Shared -locking makes publication and removal from eligibility indivisible per source. If snapshotting a later +Each selected ready item is removed from the eligibility index while all its sources remain in pending storage. +A prepared aggregate occupies one routing slot, not one per source. Shared +locking makes staging and removal from eligibility indivisible per ready item. If snapshotting a later item fails, the published prefix remains available and the failed item remains eligible; retry neither duplicates selected sources nor loses accepted work. @@ -204,6 +215,122 @@ All state is instance-local. New instances start empty and provide no restart re admission and active routing, not message byte size; required terminal bookkeeping is removed by coordinated cleanup in the later integration tasks. +## Held-source admission and ready-work publication + +The pending interface and coordinator expose two operations for the initial multipart ownership boundary: + +```java +coordinator.admitHeld(firstSourceId, firstPart); +coordinator.admitHeld(secondSourceId, secondPart); + +// The existing reassembler supplies this message and its accepted source IDs. +coordinator.publishReady(Set.of(firstSourceId, secondSourceId), assembledMessage); + +coordinator.selectAndStage(batchLimit); +``` + +This example describes the library boundary used by the opt-in admission integration below. Successful held admission gives +the protocol integration an ownership boundary at which to acknowledge an individual part without +waiting for assembly. Held records consume pending source capacity but can never be selected on their +own, regardless of their priority or timestamp. A snapshot/admission failure must not be acknowledged. + +`publishReady` accepts a nonempty set of already admitted, initially held source IDs and one prepared +execution message. It validates all sources and prepares the snapshot/eligibility metadata before binding +the whole set and exposing one candidate to normal selection. It does not enqueue directly to the router, +create a new accepted source, free source capacity, or overwrite the original accepted part payloads. +Publication therefore works even when the held sources fill pending capacity. Selection still observes +batch size, routing capacity, the prepared message's priority, and its eligibility time. + +In memory, accepted-source records and ready scheduling entries are separate. Each source links to +its first ready entry; multiple sources can share one entry. The shared lock prevents a selector from +seeing a partially bound group. The ready-entry link remains after selection so retries do not publish +another candidate. The source snapshot remains available through `find(sourceId)` independently of +the prepared aggregate and its subsequent execution mutations. + +While every source is retained, repeating the exact source set is a successful no-op: the first published +payload, priority, and eligibility time win, including after the item is selected, taken, or routed. Missing +or ordinarily admitted sources and overlapping/subset/superset regroupings fail as `INVALID_TRANSITION`. +All validation precedes binding, so failure cannot consume otherwise-unpublished sources. Snapshot or +eligibility failure leaves held sources available for a whole-publication retry. Once any source has been +removed during terminal cleanup, publishing that set is invalid; publication never recreates sources or +retains completed-publication tombstones forever. + +The caller owns assembly and expiry decisions. To release an expired incomplete part independently, +publish a singleton source set and its execution message. This primitive introduces no UDH parsing, +group-key policy, ordinal deduplication, timer, persisted deadline, or codec. Those detailed contracts +remain #343 work; the admission integration reuses the existing reassembler. + +`admitHeld` requires a running coordinator, just like ordinary admission. `publishReady` is an +existing-work transition and remains available while quiescing, so an assembler may release already +accepted work during shutdown. New selection/takes remain stopped. Held and prepared memory state +is instance-local and disappears on close/restart; no new durability guarantee is implied. + +## HTTP/SMPP admission binding + +`KannelResource` and the standard outgoing-worker factory resolve a default-qualified CDI +`OutboundCoordinator` when one is supplied by application assembly. HTTP calls +`admit` on that coordinator; the factory binds it to each standard SMPP server before worker startup. +Other explicitly assembled SMPP workers can use `setIngressCoordinator` with their own message type. +The SMPP binding cannot be changed after startup or any submission has begun. + +No production coordinator producer is added at this intermediate step. With no coordinator binding, +the existing memory-queue admission path remains available. When a binding exists, resolution or +admission failure does not fall back to that queue. Applications must supply a long-lived, started +coordinator and connect selection, dispatch, completion, and retry policies before enabling the new +pipeline. The standard message snapshot mapper and standalone activation are subsequent integration work. + +### HTTP + +After authentication, parameter handling, gateway UUID generation, and DLR return-metadata creation, +HTTP admission calls `admit` with a typed source ID wrapping the same UUID as the gateway message ID. +It returns `202` only after admission succeeds; the response contains that gateway UUID as text. +The source ID remains a distinct lifecycle type so other applications can use their own protocol IDs. +Lifecycle failures map to: + +| Failure | HTTP response | +|---|---| +| Capacity, unavailable storage/lifecycle, or ownership conflict | `503` with a retry-later response | +| Unsupported submission | `400` | +| Invalid internal transition or unexpected resolution/processing error | `500` | + +Error responses do not expose storage exception details. The legacy interrupted-admission path also +returns `503` and preserves the thread's interrupt flag. + +### SMPP and existing reassembly + +Before-insert filters and gateway UUID/timestamp assignment still precede admission. A complete SMS +is admitted directly as ready work. A submission with a concatenation header must pass the existing +reassembler's supported-header check, then uses `admitHeld` before `STATUS_OK`. It enters the ingress +queue with its accepted source ID (the same UUID as that part's gateway message ID); it is not +independently selected for routing. In legacy mode, local +ingress-queue insertion now also precedes `STATUS_OK`. + +Capacity rejection returns `STATUS_THROTTLED`; unsupported input returns `STATUS_SUBMITFAIL`; +other lifecycle/storage failures return `STATUS_SYSERR`. Rejections do not publish a successful response +or enter the legacy router queue. No new UDH syntax or detailed ordinal-validation policy is introduced. + +The existing reassembler produces either an aggregate or individually expired parts. Their internal +`InEvent` values carry an immutable source-ID set and no `submitSm`; processing calls `publishReady` +instead of admitting a new source or enqueueing directly to the legacy router. Publication failure requeues +that prepared event with the same payload and source IDs. It does not reassemble again, issue another +client acknowledgement, or release accepted sources. Original part payloads remain in pending storage. + +Per-worker identity-based metadata associates accepted raw part objects with source IDs until the +association is handed to a prepared event. The existing first-part-wins ordinal behavior remains; a +default-compatible `onDuplicateMessagePart` listener hook associates an ignored duplicate's held source +with the retained original. Its source therefore finishes with that message instead of leaking pending +capacity. This does not change duplicate response IDs or downstream receipt policy, which remain part +of the detailed multipart follow-up. + +Part insertion and completion/expiry callbacks are serialized within the existing handler so source +associations cannot cross group removal. Delayed tasks capture the actual group instance: a cancelled +old timer cannot consume a newer group that reused the same reference. Expiry only publishes the +parts already in that group, not accepted parts still waiting in the ingress queue. Group-key rules, +body assembly, relative timeout settings, and first-part-wins policy otherwise remain unchanged. + +These changes do not activate provider dispatch or implement worker-to-router retry. No mCore-specific +workflow or storage adapter is added. + ## Memory routed-work implementation `MemoryRoutedWorkStore` is independently constructed with a positive selection capacity and a diff --git a/sendium-app/src/test/java/gr/cytech/sendium/app/storage/SmsLifecycleHttpTest.java b/sendium-app/src/test/java/gr/cytech/sendium/app/storage/SmsLifecycleHttpTest.java new file mode 100644 index 00000000..f1e88ee7 --- /dev/null +++ b/sendium-app/src/test/java/gr/cytech/sendium/app/storage/SmsLifecycleHttpTest.java @@ -0,0 +1,133 @@ +package gr.cytech.sendium.app.storage; + +import gr.cytech.sendium.auth.CredentialFileWatcher; +import gr.cytech.sendium.conf.SendiumConfigurationHandler; +import gr.cytech.sendium.core.http.KannelResource; +import gr.cytech.sendium.core.message.DlrReturnMetadata; +import gr.cytech.sendium.core.message.StandardMessage; +import gr.cytech.sendium.core.outbound.DefaultOutboundCoordinator; +import gr.cytech.sendium.core.outbound.OutboundCoordinator; +import gr.cytech.sendium.core.queue.InMemoryQueueProvider; +import gr.cytech.sendium.core.storage.memory.MemoryPendingMessageStore; +import gr.cytech.sendium.core.storage.memory.MemoryRoutedWorkStore; +import gr.cytech.sendium.core.storage.memory.MemorySelectedRouterStore; +import io.quarkus.test.QuarkusExtensionTest; +import io.quarkus.test.common.http.TestHTTPResource; +import jakarta.enterprise.inject.Disposes; +import jakarta.enterprise.inject.Produces; +import jakarta.enterprise.inject.Vetoed; +import jakarta.inject.Inject; +import jakarta.inject.Singleton; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.RegisterExtension; + +import java.net.URI; +import java.net.http.HttpClient; +import java.net.http.HttpRequest; +import java.net.http.HttpResponse; +import java.time.Duration; +import java.util.Map; +import java.util.Set; +import java.util.UUID; +import java.util.concurrent.CompletableFuture; + +import static org.assertj.core.api.Assertions.assertThat; + +class SmsLifecycleHttpTest { + @RegisterExtension + static final QuarkusExtensionTest APP = new QuarkusExtensionTest() + .withApplicationRoot(archive -> archive.addClasses(KannelResource.class, Beans.class, Credentials.class, + StandaloneSmsStorage.class, SmsStorageProfile.class)) + .overrideConfigKey("quarkus.arc.exclude-dependency.core.group-id", "gr.cytech") + .overrideConfigKey("quarkus.arc.exclude-dependency.core.artifact-id", "sendium-core") + .overrideConfigKey("sendium.dlr.persistence.enabled", "false") + .overrideConfigKey("quarkus.http.test-port", "0"); + + @TestHTTPResource("/sendsms") + URI endpoint; + + @Inject + OutboundCoordinator coordinator; + + @Inject + InMemoryQueueProvider legacy; + + @Test + void httpUsesTheExplicitCoordinatorBeanAndReturnsBackpressureAtCapacity() throws Exception { + URI requestUri = URI.create(endpoint + "?username=http-user&password=secret&from=Sender&to=306900000001&text=Hello" + + "&dlr-url=https%3A%2F%2Fcallback.test%2Fdlr"); + try (HttpClient client = HttpClient.newHttpClient()) { + var request = HttpRequest.newBuilder(requestUri).timeout(Duration.ofSeconds(5)).GET().build(); + var accepted = client.send(request, HttpResponse.BodyHandlers.ofString()); + assertThat(accepted.statusCode()).isEqualTo(202); + var full = client.send(request, HttpResponse.BodyHandlers.ofString()); + assertThat(full.statusCode()).isEqualTo(503); + assertThat(legacy.getRouterQueue().isEmpty()).isTrue(); + assertThat(coordinator.selectAndStage(1)).isEqualTo(1); + var selected = coordinator.takeForRouting(Duration.ZERO).orElseThrow(); + assertThat(selected.message().serial).isEqualTo(accepted.body()); + assertThat(selected.sources()).singleElement().satisfies(source -> + assertThat(source.value()).isEqualTo(UUID.fromString(accepted.body()))); + assertThat(selected.message().body).isEqualTo("Hello"); + assertThat(selected.message().dlrReturnMetadata).isEqualTo( + DlrReturnMetadata.http("http-user", "Sender", "306900000001", "https://callback.test/dlr")); + coordinator.discard(selected.id(), CompletableFuture.completedStage(null)).toCompletableFuture().join(); + } + } + + @Singleton + public static class Beans { + @Produces + @Singleton + OutboundCoordinator coordinator() { + var pending = new MemoryPendingMessageStore(1, Beans::snapshot); + var router = new MemorySelectedRouterStore<>(pending, 1); + var routed = new MemoryRoutedWorkStore(1, Beans::snapshot); + var coordinator = new DefaultOutboundCoordinator<>(pending, router, routed); + coordinator.start(); + return coordinator; + } + + void stop(@Disposes OutboundCoordinator coordinator) { + coordinator.close(); + } + + @Produces + @Singleton + CredentialFileWatcher credentials() { + return new Credentials(); + } + + @Produces + @Singleton + SendiumConfigurationHandler configuration() { + var configuration = new SendiumConfigurationHandler(); + configuration.init(); + return configuration; + } + + @Produces + @Singleton + InMemoryQueueProvider legacyQueues() { + return new InMemoryQueueProvider(); + } + + private static StandardMessage snapshot(StandardMessage message) { + // This HTTP fixture contains only scalar fields and immutable return metadata. + try { + return (StandardMessage) message.clone(); + } catch (CloneNotSupportedException failure) { + throw new AssertionError(failure); + } + } + } + + @Vetoed + public static class Credentials extends CredentialFileWatcher { + @Override + public Map getValidCredentials() { + return Map.of("http-user", new Credential(CredentialType.HTTP, null, null, + "http-user", "secret", null, Set.of())); + } + } +} diff --git a/sendium-core/src/main/java/gr/cytech/sendium/core/http/KannelResource.java b/sendium-core/src/main/java/gr/cytech/sendium/core/http/KannelResource.java index e2c45275..e4ec69d8 100644 --- a/sendium-core/src/main/java/gr/cytech/sendium/core/http/KannelResource.java +++ b/sendium-core/src/main/java/gr/cytech/sendium/core/http/KannelResource.java @@ -5,9 +5,13 @@ import gr.cytech.sendium.conf.SendiumConfigurationHandler; import gr.cytech.sendium.core.message.DlrReturnMetadata; import gr.cytech.sendium.core.message.StandardMessage; +import gr.cytech.sendium.core.outbound.OutboundCoordinator; +import gr.cytech.sendium.core.outbound.OutboundWork.SourceId; import gr.cytech.sendium.core.queue.InMemoryQueueProvider; +import gr.cytech.sendium.core.storage.OutboundStorageException; import gr.cytech.sendium.util.MessageTrace; import jakarta.annotation.security.PermitAll; +import jakarta.enterprise.inject.Instance; import jakarta.inject.Inject; import jakarta.ws.rs.GET; import jakarta.ws.rs.Path; @@ -39,6 +43,9 @@ public class KannelResource { @Inject InMemoryQueueProvider queueProvider; + @Inject + Instance> outboundCoordinators; + @Inject CredentialFileWatcher credentialFileWatcher; @@ -74,7 +81,7 @@ public class KannelResource { ), @APIResponse( responseCode = "503", - description = "Service Unavailable. Internal queue admission was interrupted.", + description = "Service Unavailable. Admission is unavailable, at capacity, or interrupted.", content = @Content(mediaType = MediaType.TEXT_PLAIN, schema = @Schema(examples = "Temporal failure, try again later.")) ) }) @@ -205,11 +212,16 @@ public Response receiveSms( msg.field4 = binfo; } msg.acked = dlrUrl != null && !dlrUrl.isBlank(); - msg.serial = UUID.randomUUID().toString(); + UUID gatewayMessageId = UUID.randomUUID(); + msg.serial = gatewayMessageId.toString(); if (msg.acked) { msg.dlrReturnMetadata = DlrReturnMetadata.http(usr, msg.from, msg.to, dlrUrl); } - queueProvider.getRouterQueue().enqueue(msg); + if (outboundCoordinators.isUnsatisfied()) { + queueProvider.getRouterQueue().enqueue(msg); + } else { + outboundCoordinators.get().admit(new SourceId(gatewayMessageId), msg); + } if (MessageTrace.shouldLog(configurationHandler, MessageTrace.EVENT_ACCEPTED)) { logger.info("message.accepted ingress=http {}", MessageTrace.identifiers(msg)); } @@ -218,7 +230,16 @@ public Response receiveSms( .entity(msg.serial) .build(); + } catch (OutboundStorageException e) { + logger.warn("HTTP SMS admission failed stage={} reason={}", e.stage(), e.reason()); + return switch (e.reason()) { + case CAPACITY_EXCEEDED, UNAVAILABLE, OWNERSHIP_CONFLICT -> Response.status(Response.Status.SERVICE_UNAVAILABLE) + .entity("Temporal failure, try again later.").build(); + case UNSUPPORTED -> Response.status(Response.Status.BAD_REQUEST).entity("Unsupported SMS submission").build(); + case INVALID_TRANSITION -> Response.status(Response.Status.INTERNAL_SERVER_ERROR).entity("Error processing SMS").build(); + }; } catch (InterruptedException e) { + Thread.currentThread().interrupt(); logger.error("Failed to enqueue message", e); return Response.status(Response.Status.SERVICE_UNAVAILABLE) .entity("Temporal failure, try again later.") diff --git a/sendium-core/src/main/java/gr/cytech/sendium/core/outbound/DefaultOutboundCoordinator.java b/sendium-core/src/main/java/gr/cytech/sendium/core/outbound/DefaultOutboundCoordinator.java index 28b5b8aa..167e9a01 100644 --- a/sendium-core/src/main/java/gr/cytech/sendium/core/outbound/DefaultOutboundCoordinator.java +++ b/sendium-core/src/main/java/gr/cytech/sendium/core/outbound/DefaultOutboundCoordinator.java @@ -123,6 +123,10 @@ public void quiesce() { @Override public void admit(SourceId source, M message) { + admit(source, message, false); + } + + private void admit(SourceId source, M message, boolean held) { Objects.requireNonNull(source, "source"); Objects.requireNonNull(message, "message"); lock.lock(); @@ -131,7 +135,27 @@ public void admit(SourceId source, M message) { if (sourceSelections.containsKey(source)) { return; } - pending.admit(source, message); + if (held) { + pending.admitHeld(source, message); + } else { + pending.admit(source, message); + } + } finally { + lock.unlock(); + } + } + + @Override + public void admitHeld(SourceId source, M message) { + admit(source, message, true); + } + + @Override + public void publishReady(Set sources, M message) { + lock.lock(); + try { + requireActive(Role.PENDING); + pending.publishReady(sources, message); } finally { lock.unlock(); } diff --git a/sendium-core/src/main/java/gr/cytech/sendium/core/outbound/OutboundCoordinator.java b/sendium-core/src/main/java/gr/cytech/sendium/core/outbound/OutboundCoordinator.java index bf8725c9..ac5db96a 100644 --- a/sendium-core/src/main/java/gr/cytech/sendium/core/outbound/OutboundCoordinator.java +++ b/sendium-core/src/main/java/gr/cytech/sendium/core/outbound/OutboundCoordinator.java @@ -10,6 +10,7 @@ import java.time.Duration; import java.util.Optional; +import java.util.Set; import java.util.concurrent.CompletionStage; /** @@ -35,6 +36,16 @@ public interface OutboundCoordinator extends AutoClos void admit(SourceId source, M message); + /** Accepts source ownership without making this source routable yet. */ + void admitHeld(SourceId source, M message); + + /** + * Makes one execution message available to bounded selection using the original held source IDs. + * The caller decides which sources form an assembled message or an individual expired part. + * This is an existing-work transition and remains available during quiescence; new admissions do not. + */ + void publishReady(Set sources, M message); + int selectAndStage(int limit); Optional> takeForRouting(Duration timeout) throws InterruptedException; diff --git a/sendium-core/src/main/java/gr/cytech/sendium/core/outbound/OutboundWork.java b/sendium-core/src/main/java/gr/cytech/sendium/core/outbound/OutboundWork.java index 3b8d2ff4..f5e799f1 100644 --- a/sendium-core/src/main/java/gr/cytech/sendium/core/outbound/OutboundWork.java +++ b/sendium-core/src/main/java/gr/cytech/sendium/core/outbound/OutboundWork.java @@ -7,9 +7,10 @@ import java.util.UUID; /** - * Storage identities are independent of protocol message IDs. Message-bearing values contain - * mutable execution projections. Mutations are intentional but must not silently change an - * earlier-stage record; implementations keep recorded payload and stage consistent. + * Storage identity types are independent of protocol message IDs. An ingress may reuse its gateway + * message UUID as a source ID; later stage IDs still identify their own lifecycle steps. + * Message-bearing values contain mutable execution projections. Mutations are intentional but must + * not silently change an earlier-stage record; implementations keep recorded payload and stage consistent. */ public final class OutboundWork { private OutboundWork() { diff --git a/sendium-core/src/main/java/gr/cytech/sendium/core/smpp/server/InEvent.java b/sendium-core/src/main/java/gr/cytech/sendium/core/smpp/server/InEvent.java index 15540930..05480988 100644 --- a/sendium-core/src/main/java/gr/cytech/sendium/core/smpp/server/InEvent.java +++ b/sendium-core/src/main/java/gr/cytech/sendium/core/smpp/server/InEvent.java @@ -2,14 +2,17 @@ import com.cloudhopper.smpp.pdu.SubmitSm; import gr.cytech.sendium.core.message.StandardMessage; +import gr.cytech.sendium.core.outbound.OutboundWork.SourceId; import java.sql.Timestamp; +import java.util.Set; public class InEvent { public final M pMsg; public final SubmitSm submitSm; public final int mpid; public final Timestamp localTimestamp; + public final Set sourceIds; public boolean notifyClient; public boolean concatenated; public boolean waitingForResponse; @@ -20,10 +23,17 @@ public InEvent(M pMsg, SubmitSm submitSm, int mpid, Timestamp localTimestamp) { } public InEvent(M pMsg, SubmitSm submitSm, int mpid, Timestamp localTimestamp, boolean waitingForResponse, String messageId) { + this(pMsg, submitSm, mpid, localTimestamp, waitingForResponse, messageId, Set.of()); + } + + /** Coordinated events retain source IDs; an event without submitSm publishes already-accepted ready work. */ + public InEvent(M pMsg, SubmitSm submitSm, int mpid, Timestamp localTimestamp, boolean waitingForResponse, + String messageId, Set sourceIds) { this.pMsg = pMsg; this.submitSm = submitSm; this.mpid = mpid; this.localTimestamp = localTimestamp; + this.sourceIds = Set.copyOf(sourceIds); this.notifyClient = true; this.concatenated = false; this.waitingForResponse = waitingForResponse; diff --git a/sendium-core/src/main/java/gr/cytech/sendium/core/smpp/server/MessagePartsEventsListener.java b/sendium-core/src/main/java/gr/cytech/sendium/core/smpp/server/MessagePartsEventsListener.java index fa52275a..fbec4220 100644 --- a/sendium-core/src/main/java/gr/cytech/sendium/core/smpp/server/MessagePartsEventsListener.java +++ b/sendium-core/src/main/java/gr/cytech/sendium/core/smpp/server/MessagePartsEventsListener.java @@ -7,5 +7,9 @@ public interface MessagePartsEventsListener { void onMessagePartsHandlingEvent(MessagePartsHandler.MessagePartsEventType type, List parts); + /** Reports an ignored duplicate ordinal without changing the existing first-part-wins policy. */ + default void onDuplicateMessagePart(M original, M duplicate) { + } + String getName(); } diff --git a/sendium-core/src/main/java/gr/cytech/sendium/core/smpp/server/MessagePartsHandler.java b/sendium-core/src/main/java/gr/cytech/sendium/core/smpp/server/MessagePartsHandler.java index af501eda..63622466 100644 --- a/sendium-core/src/main/java/gr/cytech/sendium/core/smpp/server/MessagePartsHandler.java +++ b/sendium-core/src/main/java/gr/cytech/sendium/core/smpp/server/MessagePartsHandler.java @@ -53,7 +53,7 @@ public MessagePartsHandler(MessagePartsEventsListener listener, long timeoutI .factory())); } - public boolean stop() { + public synchronized boolean stop() { logger.info("Stopping MessagePartsHandler: {} ...", listener.getName()); //process any scheduled tasks and stop the executor @@ -83,7 +83,7 @@ public void setTimeoutInMilliseconds(long timeoutInMilliseconds) { * @param part The part to be added. It must be a valid part based on the {@link MessagePartsHandler#isSupportedMessagePart} * @throws IllegalArgumentException If the {@link MessagePartsHandler#isSupportedMessagePart} return false */ - public void addMessagePart(M part) throws IllegalArgumentException { + public synchronized void addMessagePart(M part) throws IllegalArgumentException { if (!isSupportedMessagePart(part)) { throw new IllegalArgumentException("The provided message part does not contain a supported UDH"); } @@ -94,7 +94,9 @@ public void addMessagePart(M part) throws IllegalArgumentException { k -> new ConcurrentSkipListSet<>(new MessagePartsComparator())); logger.debug("adding message part for msgRefNum: {} with udh: {}", msgRefNum, part.binheader); - receivedParts.add(part); + if (!receivedParts.add(part)) { + listener.onDuplicateMessagePart(receivedParts.tailSet(part).first(), part); + } scheduledTasks.computeIfAbsent(msgRefNum, this::scheduleDelayedMessagePartsTask); @@ -124,15 +126,11 @@ private Future scheduleDelayedMessagePartsTask(String msgRefNum) { */ private List cancelScheduledDelayedMessagePartsTask(String msgRefNum) { Future futureCall = scheduledTasks.remove(msgRefNum); - List receivedParts = null; - //Remove all the related information because all message parts have been received - if (futureCall != null && futureCall.cancel(false)) { - Set parts = pendingMessageParts.remove(msgRefNum); - if (parts != null) { - receivedParts = new ArrayList<>(parts); - } + if (futureCall != null) { + futureCall.cancel(false); } - return receivedParts; + Set parts = pendingMessageParts.remove(msgRefNum); + return parts == null ? null : new ArrayList<>(parts); } /** @@ -170,9 +168,11 @@ public int compare(M part1, M part2) { */ public class DelayedMessagePartsTask implements Runnable, Callable { private final String msgRefNum; + private final SortedSet expectedParts; public DelayedMessagePartsTask(String msgRefNum) { this.msgRefNum = msgRefNum; + this.expectedParts = pendingMessageParts.get(msgRefNum); } public void run() { @@ -184,13 +184,22 @@ public Boolean call() { } private boolean handleReceivedParts() { + synchronized (MessagePartsHandler.this) { + return handleCurrentGroup(); + } + } + + private boolean handleCurrentGroup() { try { - Set parts = pendingMessageParts.remove(msgRefNum); - if (parts != null && !parts.isEmpty()) { - //Just remove the reference to the task as it has already been executed. - scheduledTasks.remove(msgRefNum); + // A cancelled timer may still run after the reference has been reused for a new group. + if (expectedParts != null && pendingMessageParts.get(msgRefNum) == expectedParts) { + pendingMessageParts.remove(msgRefNum); + Future task = scheduledTasks.remove(msgRefNum); + if (task != null) { + task.cancel(false); + } //notify the registered listener - listener.onMessagePartsHandlingEvent(MessagePartsEventType.DELAYED, new ArrayList<>(parts)); + listener.onMessagePartsHandlingEvent(MessagePartsEventType.DELAYED, new ArrayList<>(expectedParts)); } } catch (Exception ex) { logger.error("exception caught while handling the delayed parts with msgRefNum: " + msgRefNum); diff --git a/sendium-core/src/main/java/gr/cytech/sendium/core/smpp/server/SmppServerWorker.java b/sendium-core/src/main/java/gr/cytech/sendium/core/smpp/server/SmppServerWorker.java index 9f8358d6..da2dbcec 100644 --- a/sendium-core/src/main/java/gr/cytech/sendium/core/smpp/server/SmppServerWorker.java +++ b/sendium-core/src/main/java/gr/cytech/sendium/core/smpp/server/SmppServerWorker.java @@ -20,12 +20,16 @@ import gr.cytech.sendium.conf.SendiumConfigurationProvider; import gr.cytech.sendium.core.AbstractOutWorker; import gr.cytech.sendium.core.message.StandardMessage; +import gr.cytech.sendium.core.outbound.OutboundCoordinator; +import gr.cytech.sendium.core.outbound.OutboundWork.SourceId; import gr.cytech.sendium.core.queue.Queue; import gr.cytech.sendium.core.smpp.server.tasks.InTask; import gr.cytech.sendium.core.smpp.server.tasks.InactivityTimeTask; import gr.cytech.sendium.core.smpp.server.tasks.OutTask; import gr.cytech.sendium.core.smpp.server.tasks.PrintStatisticsTask; import gr.cytech.sendium.core.smpp.util.SmppServerUtil; +import gr.cytech.sendium.core.storage.OutboundStage; +import gr.cytech.sendium.core.storage.OutboundStorageException; import gr.cytech.sendium.core.worker.FailDelayPolicy; import gr.cytech.sendium.core.worker.FailDelayPolicyAction; import gr.cytech.sendium.core.worker.WorkerType; @@ -47,8 +51,11 @@ import java.time.ZonedDateTime; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collections; import java.util.HashSet; +import java.util.IdentityHashMap; import java.util.List; +import java.util.Map; import java.util.Objects; import java.util.OptionalInt; import java.util.Set; @@ -141,6 +148,9 @@ public class SmppServerWorker extends AbstractOutWork protected MessagePartsHandler messagePartsHandler; protected boolean isFastUnsafeStop = false; private final ReentrantReadWriteLock ingressLifecycleLock = new ReentrantReadWriteLock(true); + private final Map> pendingPartSources = Collections.synchronizedMap(new IdentityHashMap<>()); + private volatile OutboundCoordinator ingressCoordinator; + private volatile boolean ingressStarted; public SmppServerWorker() { this.authProvider = new BasicSmppAuthenticationProvider(this); @@ -194,6 +204,23 @@ public void setMessageStore(SmppServerMessageStore messageStore) { this.messageStore = messageStore; } + /** Select the ingress lifecycle before starting this worker or admitting any submissions. */ + public void setIngressCoordinator(OutboundCoordinator coordinator) { + Objects.requireNonNull(coordinator, "coordinator"); + if (ingressStarted || inExecutor != null) { + throw new IllegalStateException("Ingress lifecycle must be selected before worker startup/admission"); + } + ingressLifecycleLock.writeLock().lock(); + try { + if (ingressStarted || inExecutor != null || (ingressCoordinator != null && ingressCoordinator != coordinator)) { + throw new IllegalStateException("Ingress lifecycle cannot change after selection/startup"); + } + ingressCoordinator = coordinator; + } finally { + ingressLifecycleLock.writeLock().unlock(); + } + } + @Override public String getType() { return TYPE_SMPP_SERVER; @@ -201,6 +228,12 @@ public String getType() { @Override public Thread start() { + ingressLifecycleLock.writeLock().lock(); + try { + ingressStarted = true; + } finally { + ingressLifecycleLock.writeLock().unlock(); + } logger.info("starting SmppServer"); this.keepOnRunning = true; stopExecutor(inExecutor, "in"); @@ -943,6 +976,16 @@ public void outTaskFailed(Pdu pdu, M message) { } public void enqueueIn(InEvent ine) { + ingressLifecycleLock.readLock().lock(); + try { + ingressStarted = true; + acceptIngress(ine); + } finally { + ingressLifecycleLock.readLock().unlock(); + } + } + + private void acceptIngress(InEvent ine) { if (printMsgs) { logger.debug("IN: {}", ine); } @@ -955,15 +998,68 @@ public void enqueueIn(InEvent ine) { if (filtered == null) { return; } - filtered.pMsg.serial = UUID.randomUUID().toString(); + UUID gatewayMessageId = UUID.randomUUID(); + filtered.pMsg.serial = gatewayMessageId.toString(); filtered.pMsg.ctstamp = ine.localTimestamp.getTime(); filtered.pMsg.onetwork = ine.mpid; + if (ingressCoordinator != null) { + if (!admitToLifecycle(filtered, gatewayMessageId)) { + return; + } + } else { + inEventQueue.add(filtered); + } if (filtered.submitSm != null) { enqueueOut(SmppServerUtil.createSubmitRsp( filtered.submitSm, SmppConstants.STATUS_OK, filtered.pMsg.serial)); filtered.waitingForResponse = false; } - inEventQueue.add(filtered); + } + + private boolean admitToLifecycle(InEvent event, UUID gatewayMessageId) { + try { + SourceId source = new SourceId(gatewayMessageId); + if (event.submitSm != null && !Strings.isNullOrEmpty(event.pMsg.binheader)) { + if (!messagePartsHandler.isSupportedMessagePart(event.pMsg)) { + throw new OutboundStorageException(OutboundStage.Role.PENDING, OutboundStorageException.Reason.UNSUPPORTED, + "Unsupported multipart header"); + } + ingressCoordinator.admitHeld(source, event.pMsg); + synchronized (pendingPartSources) { + pendingPartSources.computeIfAbsent(event.pMsg, ignored -> new HashSet<>()).add(source); + } + InEvent accepted = new InEvent<>(event.pMsg, event.submitSm, event.mpid, event.localTimestamp, + false, event.responseMessageId, Set.of(source)); + accepted.notifyClient = event.notifyClient; + inEventQueue.add(accepted); + } else { + ingressCoordinator.admit(source, event.pMsg); + } + if (MessageTrace.shouldLog(configurationProvider, MessageTrace.EVENT_ACCEPTED)) { + logger.info("message.accepted ingress=smppserver worker={} {}", getFullName(), MessageTrace.identifiers(event.pMsg)); + } + return true; + } catch (OutboundStorageException failure) { + logger.warn("SMPP admission failed stage={} reason={}", failure.stage(), failure.reason()); + int status = switch (failure.reason()) { + case CAPACITY_EXCEEDED -> SmppConstants.STATUS_THROTTLED; + case UNSUPPORTED -> SmppConstants.STATUS_SUBMITFAIL; + default -> SmppConstants.STATUS_SYSERR; + }; + rejectIngress(event, status); + return false; + } catch (RuntimeException failure) { + logger.error("SMPP admission failed before acknowledgement", failure); + rejectIngress(event, SmppConstants.STATUS_SYSERR); + return false; + } + } + + private void rejectIngress(InEvent event, int status) { + if (event.submitSm != null) { + enqueueOut(SmppServerUtil.createSubmitRsp(event.submitSm, status, null)); + event.waitingForResponse = false; + } } protected boolean checkReassembling(M msg) { @@ -989,6 +1085,18 @@ public void handleIngressMessages(List> events) { } try { + if (ingressCoordinator != null) { + if (event.sourceIds.isEmpty()) { + throw new IllegalStateException("Accepted ingress event has no lifecycle source ownership"); + } + if (event.submitSm == null) { + ingressCoordinator.publishReady(event.sourceIds, event.pMsg); + } else if (pendingPartSources.containsKey(event.pMsg)) { + messagePartsHandler.addMessagePart(event.pMsg); + } + // A replayed raw event may already have handed its sources to a retained part or ready event. + continue; + } if (event.submitSm != null && !Strings.isNullOrEmpty(event.pMsg.binheader)) { messagePartsHandler.addMessagePart(event.pMsg); } else { @@ -1301,9 +1409,30 @@ private int getMessageType(StandardMessage msg) throws IOException { } public class CcatMessagePartsEventsListener implements MessagePartsEventsListener { + @Override + public void onDuplicateMessagePart(M original, M duplicate) { + if (ingressCoordinator == null || original == duplicate) { + return; + } + synchronized (pendingPartSources) { + Set originalSources = requirePartSources(original); + Set duplicateSources = requirePartSources(duplicate); + originalSources.addAll(duplicateSources); + pendingPartSources.remove(duplicate); + } + } + @Override public void onMessagePartsHandlingEvent(MessagePartsHandler.MessagePartsEventType type, List parts) { + List> sources = new ArrayList<>(); + if (ingressCoordinator != null) { + synchronized (pendingPartSources) { + for (M part : parts) { + sources.add(Set.copyOf(requirePartSources(part))); + } + } + } if (type == MessagePartsHandler.MessagePartsEventType.COMPLETE) { //The first part contains the id that we need to send to the provider M message = parts.getFirst(); @@ -1312,12 +1441,32 @@ public void onMessagePartsHandlingEvent(MessagePartsHandler.MessagePartsEventTyp message.body = parts.stream().map(m -> m.body).collect(Collectors.joining("")); message.reassembledParts = parts.stream().map(m -> m.serial).collect(Collectors.toCollection(ArrayList::new)); - InEvent event = new InEvent(message, null, message.onetwork, new Timestamp(message.ctstamp)); + Set allSources = sources.stream().flatMap(Set::stream).collect(Collectors.toSet()); + InEvent event = new InEvent<>(message, null, message.onetwork, new Timestamp(message.ctstamp), + ingressCoordinator == null, null, allSources); reEnqueueIn(List.of(event)); } else { - var messages = parts.stream().map(m -> new InEvent(m, null, m.onetwork, new Timestamp(m.ctstamp))).collect(Collectors.toList()); + List> messages = new ArrayList<>(); + for (int i = 0; i < parts.size(); i++) { + M part = parts.get(i); + messages.add(new InEvent<>(part, null, part.onetwork, new Timestamp(part.ctstamp), + ingressCoordinator == null, null, ingressCoordinator == null ? Set.of() : sources.get(i))); + } reEnqueueIn(messages); } + if (ingressCoordinator != null) { + synchronized (pendingPartSources) { + parts.forEach(pendingPartSources::remove); + } + } + } + + private Set requirePartSources(M part) { + Set sources = pendingPartSources.get(part); + if (sources == null || sources.isEmpty()) { + throw new IllegalStateException("Reassembled part has no lifecycle source ownership"); + } + return sources; } @Override diff --git a/sendium-core/src/main/java/gr/cytech/sendium/core/storage/PendingMessageStore.java b/sendium-core/src/main/java/gr/cytech/sendium/core/storage/PendingMessageStore.java index 383bb641..7a5e4447 100644 --- a/sendium-core/src/main/java/gr/cytech/sendium/core/storage/PendingMessageStore.java +++ b/sendium-core/src/main/java/gr/cytech/sendium/core/storage/PendingMessageStore.java @@ -10,10 +10,25 @@ public interface PendingMessageStore extends Outbound * Records accepted source state before protocol success. Later execution mutations must not * silently change that state. Repeating the same source ID while it remains pending is idempotent * and never overwrites its accepted content. A failure must - * not be acknowledged. Deduplication after terminal source removal is not promised. + * not be acknowledged. This admits ready work, subject to the selection eligibility policy. + * Repeating either admission method preserves the first accepted payload and held/ready disposition. + * Deduplication after terminal source removal is not promised. */ void admit(SourceId source, M message); + /** Retains an accepted source without making it eligible for selection until publishReady succeeds. */ + void admitHeld(SourceId source, M message); + + /** + * Publishes one execution message referring to a nonempty set of admitted held sources. Original + * source records remain authoritative. Publication must bind the whole set before exposing work + * to selection; it does not bypass selection limits or create another accepted source. + * While all sources remain retained, repeating the exact source set preserves the first published + * payload and cannot duplicate work, including after selection/routing. Missing, ordinarily admitted, + * or differently grouped sources fail explicitly. The caller owns assembly/expiry policy. + */ + void publishReady(Set sources, M message); + /** * Called only after all work and required handoffs are terminal. Missing sources are already * complete. Partial multi-source completion is allowed; retrying the full set must be safe. diff --git a/sendium-core/src/main/java/gr/cytech/sendium/core/storage/memory/MemoryPendingMessageStore.java b/sendium-core/src/main/java/gr/cytech/sendium/core/storage/memory/MemoryPendingMessageStore.java index f2501889..4d61242e 100644 --- a/sendium-core/src/main/java/gr/cytech/sendium/core/storage/memory/MemoryPendingMessageStore.java +++ b/sendium-core/src/main/java/gr/cytech/sendium/core/storage/memory/MemoryPendingMessageStore.java @@ -39,7 +39,7 @@ public final class MemoryPendingMessageStore implemen private final int capacity; private final UnaryOperator snapshots; private final Function eligibleAt; - private final Map> records = new HashMap<>(); + private final Map> records = new HashMap<>(); private final NavigableMap>> waiting = new TreeMap<>(Comparator.reverseOrder()); private State state = State.NEW; private Object selectorOwner; @@ -49,7 +49,7 @@ public MemoryPendingMessageStore(int capacity, UnaryOperator snapshots) { this(capacity, snapshots, message -> Instant.MIN); } - /** Eligibility is evaluated once at admission; subsequent message mutations cannot change it. */ + /** Eligibility is evaluated when ready work is first published; held admission does not evaluate it. */ public MemoryPendingMessageStore(int capacity, UnaryOperator snapshots, Function eligibleAt) { if (capacity <= 0) { throw new IllegalArgumentException("Pending capacity must be positive"); @@ -84,6 +84,10 @@ public Status status() { @Override public void admit(SourceId source, M message) { + admit(source, message, false); + } + + private void admit(SourceId source, M message, boolean held) { Objects.requireNonNull(source, "source"); Objects.requireNonNull(message, "message"); lock.lock(); @@ -96,29 +100,95 @@ public void admit(SourceId source, M message) { throw new OutboundStorageException(Role.PENDING, CAPACITY_EXCEEDED, "Memory pending capacity reached"); } M snapshot = snapshot(message, Role.PENDING); - Instant available; - try { - available = Objects.requireNonNull(eligibleAt.apply(snapshot), "eligibleAt result"); - } catch (RuntimeException failure) { - throw new OutboundStorageException(Role.PENDING, UNSUPPORTED, "Cannot determine source eligibility", failure); + Source accepted = new Source<>(snapshot, held); + Entry ready = held ? null : readyEntry(Set.of(source), snapshot); + records.put(source, accepted); + if (ready != null) { + accepted.ready = ready; + addWaiting(ready); } - Entry entry = new Entry<>(source, snapshot, available, sequence++); - records.put(source, entry); - waiting.computeIfAbsent(entry.priority, ignored -> new TreeSet<>(Comparator - .comparing((Entry item) -> item.available) - .thenComparingLong(item -> item.sequence))).add(entry); } finally { lock.unlock(); } } + @Override + public void admitHeld(SourceId source, M message) { + admit(source, message, true); + } + + @Override + public void publishReady(Set sources, M message) { + Set checked = Set.copyOf(sources); + Objects.requireNonNull(message, "message"); + if (checked.isEmpty()) { + throw new IllegalArgumentException("Ready work must reference at least one source"); + } + lock.lock(); + try { + requireReady(Role.PENDING); + Entry existing = null; + for (SourceId source : checked) { + Source accepted = records.get(source); + if (accepted == null || !accepted.initiallyHeld) { + throw invalidPublication(); + } + if (accepted.ready != null) { + if (existing != null && accepted.ready != existing) { + throw invalidPublication(); + } + existing = accepted.ready; + } + } + if (existing != null) { + if (!existing.sources.equals(checked)) { + throw invalidPublication(); + } + for (SourceId source : checked) { + if (records.get(source).ready != existing) { + throw invalidPublication(); + } + } + return; + } + Entry entry = readyEntry(checked, snapshot(message, Role.PENDING)); + for (SourceId source : checked) { + records.get(source).ready = entry; + } + addWaiting(entry); + } finally { + lock.unlock(); + } + } + + private Entry readyEntry(Set sources, M message) { + Instant available; + try { + available = Objects.requireNonNull(eligibleAt.apply(message), "eligibleAt result"); + } catch (RuntimeException failure) { + throw new OutboundStorageException(Role.PENDING, UNSUPPORTED, "Cannot determine work eligibility", failure); + } + return new Entry<>(sources, message, available, sequence++); + } + + private void addWaiting(Entry entry) { + waiting.computeIfAbsent(entry.priority, ignored -> new TreeSet<>(Comparator + .comparing((Entry item) -> item.available) + .thenComparingLong(item -> item.sequence))).add(entry); + } + + private static OutboundStorageException invalidPublication() { + return new OutboundStorageException(Role.PENDING, INVALID_TRANSITION, + "Ready publication requires retained held sources with one matching source set"); + } + /** Single-source lookup for memory-backend consumers; returns an isolated execution value. */ public Optional find(SourceId source) { Objects.requireNonNull(source, "source"); lock.lock(); try { requireReady(Role.PENDING); - Entry entry = records.get(source); + Source entry = records.get(source); return entry == null ? Optional.empty() : Optional.of(snapshot(entry.message, Role.PENDING)); } finally { lock.unlock(); @@ -132,9 +202,9 @@ public void complete(Set sources) { try { requireReady(Role.PENDING); for (SourceId source : checked) { - Entry entry = records.remove(source); - if (entry != null) { - removeWaiting(entry); + Source entry = records.remove(source); + if (entry != null && entry.ready != null) { + removeWaiting(entry.ready); } } } finally { @@ -209,18 +279,29 @@ M snapshot(M message, Role operation) { } static final class Entry { - final SourceId source; + final Set sources; final M message; final int priority; final Instant available; final long sequence; - Entry(SourceId source, M message, Instant available, long sequence) { - this.source = source; + Entry(Set sources, M message, Instant available, long sequence) { + this.sources = sources; this.message = message; this.priority = message.priority; this.available = available; this.sequence = sequence; } } + + private static final class Source { + private final M message; + private final boolean initiallyHeld; + private Entry ready; + + private Source(M message, boolean held) { + this.message = message; + this.initiallyHeld = held; + } + } } diff --git a/sendium-core/src/main/java/gr/cytech/sendium/core/storage/memory/MemorySelectedRouterStore.java b/sendium-core/src/main/java/gr/cytech/sendium/core/storage/memory/MemorySelectedRouterStore.java index 30e275af..e9946a9b 100644 --- a/sendium-core/src/main/java/gr/cytech/sendium/core/storage/memory/MemorySelectedRouterStore.java +++ b/sendium-core/src/main/java/gr/cytech/sendium/core/storage/memory/MemorySelectedRouterStore.java @@ -23,7 +23,7 @@ /** * One selected-router owner per pending-store lifetime. A shared lock makes memory selection and - * publication indivisible per source to other operations. Capacity includes queued and taken routing work; + * publication indivisible per ready item to other operations. Capacity includes queued and taken routing work; * routed records remain retained until source cleanup but no longer occupy a routing slot. */ public final class MemorySelectedRouterStore implements SelectedRouterStore { @@ -90,7 +90,7 @@ public int selectAndStage(int limit) { break; } M message = pending.snapshot(source.message, Role.ROUTER_QUEUE); - Entry entry = new Entry<>(new SelectionId(UUID.randomUUID()), Set.of(source.source), message); + Entry entry = new Entry<>(new SelectionId(UUID.randomUUID()), source.sources, message); records.put(entry.id, entry); ready.addLast(entry); pending.removeWaiting(source); diff --git a/sendium-core/src/main/java/gr/cytech/sendium/routing/StandardOutgoingWorkerHandler.java b/sendium-core/src/main/java/gr/cytech/sendium/routing/StandardOutgoingWorkerHandler.java index 3648b4bc..36c0c356 100644 --- a/sendium-core/src/main/java/gr/cytech/sendium/routing/StandardOutgoingWorkerHandler.java +++ b/sendium-core/src/main/java/gr/cytech/sendium/routing/StandardOutgoingWorkerHandler.java @@ -7,6 +7,7 @@ import gr.cytech.sendium.conf.SendiumConfigurationHandler; import gr.cytech.sendium.core.AbstractOutWorker; import gr.cytech.sendium.core.message.StandardMessage; +import gr.cytech.sendium.core.outbound.OutboundCoordinator; import gr.cytech.sendium.core.queue.InMemoryQueueProvider; import gr.cytech.sendium.core.smpp.server.SmppServerWorker; import gr.cytech.sendium.core.smpp.server.StandardSmppServerMessageStore; @@ -57,6 +58,8 @@ public class StandardOutgoingWorkerHandler implements PropertyChangeListener, Ou @Any Instance> availableWorkers; @Inject + Instance> outboundCoordinators; + @Inject SendiumConfigurationHandler configurationHandler; @Inject InMemoryQueueProvider queueProvider; @@ -206,6 +209,9 @@ protected AbstractOutWorker startWorker(String instName, String worker.init(workerResourceProvider, new StandardMessageTracker(worker)); if (SmppServerWorker.TYPE_SMPP_SERVER.equals(worker.getType())) { var smppServer = (SmppServerWorker) worker; + if (!outboundCoordinators.isUnsatisfied()) { + smppServer.setIngressCoordinator(outboundCoordinators.get()); + } smppServer.setMessageStore(new StandardSmppServerMessageStore(smppServer)); } } catch (Exception e) { diff --git a/sendium-core/src/test/java/gr/cytech/sendium/core/http/KannelResourceTest.java b/sendium-core/src/test/java/gr/cytech/sendium/core/http/KannelResourceTest.java index dcb5f0a4..acd9b1ed 100644 --- a/sendium-core/src/test/java/gr/cytech/sendium/core/http/KannelResourceTest.java +++ b/sendium-core/src/test/java/gr/cytech/sendium/core/http/KannelResourceTest.java @@ -4,21 +4,31 @@ import gr.cytech.sendium.conf.SendiumConfigurationHandler; import gr.cytech.sendium.core.message.DlrReturnMetadata; import gr.cytech.sendium.core.message.StandardMessage; +import gr.cytech.sendium.core.outbound.OutboundCoordinator; +import gr.cytech.sendium.core.outbound.OutboundWork.SourceId; import gr.cytech.sendium.core.queue.InMemoryQueueProvider; import gr.cytech.sendium.core.queue.Queue; +import gr.cytech.sendium.core.storage.OutboundStage; +import gr.cytech.sendium.core.storage.OutboundStorageException; +import jakarta.enterprise.inject.Instance; import jakarta.ws.rs.core.Response; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.mockito.ArgumentCaptor; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.EnumSource; +import utils.OutboundIngressFixture; import java.util.Map; import java.util.Set; +import java.util.UUID; import static org.assertj.core.api.Assertions.assertThat; import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoInteractions; import static org.mockito.Mockito.when; class KannelResourceTest { @@ -42,6 +52,8 @@ void setUp() { resource = new KannelResource(); resource.queueProvider = queueProvider; + resource.outboundCoordinators = mock(Instance.class); + when(resource.outboundCoordinators.isUnsatisfied()).thenReturn(true); resource.credentialFileWatcher = credentials; resource.configurationHandler = mock(SendiumConfigurationHandler.class); } @@ -78,14 +90,19 @@ void callbackSubmissionCarriesHttpReturnMetadata() throws InterruptedException { } @Test - void returnsRetryableFailureWhenQueueAdmissionIsInterruptedAfterPersistence() throws InterruptedException { + void returnsRetryableFailureWhenQueueAdmissionIsInterrupted() throws InterruptedException { doThrow(new InterruptedException("interrupted")) .when(routerQueue).enqueue(any(StandardMessage.class)); - Response response = submit(null); + try { + Response response = submit(null); - assertThat(response.getStatus()).isEqualTo(Response.Status.SERVICE_UNAVAILABLE.getStatusCode()); - assertThat(response.getEntity()).isEqualTo("Temporal failure, try again later."); + assertThat(response.getStatus()).isEqualTo(Response.Status.SERVICE_UNAVAILABLE.getStatusCode()); + assertThat(response.getEntity()).isEqualTo("Temporal failure, try again later."); + assertThat(Thread.currentThread().isInterrupted()).isTrue(); + } finally { + Thread.interrupted(); + } } private Response submit(String dlrUrl) { @@ -93,4 +110,65 @@ private Response submit(String dlrUrl) { USERNAME, PASSWORD, "Sender", "306910000000", "Hello", null, null, null, null, null, null, null, dlrUrl, null, null, null, null, null, null, null, null); } + + @Test + void coordinatorAdmissionOwnsTheMessageBeforeHttpSuccess() { + try (var lifecycle = new OutboundIngressFixture(1)) { + when(resource.outboundCoordinators.isUnsatisfied()).thenReturn(false); + when(resource.outboundCoordinators.get()).thenReturn(lifecycle.coordinator); + Response response = submit("https://callback.test/dlr"); + var source = ArgumentCaptor.forClass(SourceId.class); + var message = ArgumentCaptor.forClass(StandardMessage.class); + verify(lifecycle.coordinator).admit(source.capture(), message.capture()); + assertThat(response.getStatus()).isEqualTo(202); + assertThat(source.getValue().value()).isEqualTo(UUID.fromString((String) response.getEntity())); + var retained = lifecycle.pending.find(source.getValue()).orElseThrow(); + assertThat(retained.serial).isEqualTo(response.getEntity()); + assertThat(retained.body).isEqualTo("Hello"); + assertThat(retained.dlrReturnMetadata).isEqualTo(message.getValue().dlrReturnMetadata); + assertThat(retained.dlrReturnMetadata.channel()).isEqualTo(DlrReturnMetadata.DeliveryChannel.HTTP); + verifyNoInteractions(routerQueue); + } + } + + @Test + void fullOrQuiescingCoordinatorRejectsWithoutFallingBackToTheQueue() { + try (var lifecycle = new OutboundIngressFixture(1)) { + when(resource.outboundCoordinators.isUnsatisfied()).thenReturn(false); + when(resource.outboundCoordinators.get()).thenReturn(lifecycle.coordinator); + assertThat(submit(null).getStatus()).isEqualTo(202); + assertThat(submit(null).getStatus()).isEqualTo(503); + lifecycle.coordinator.quiesce(); + assertThat(submit(null).getStatus()).isEqualTo(503); + verifyNoInteractions(routerQueue); + } + } + + @ParameterizedTest + @EnumSource(OutboundStorageException.Reason.class) + @SuppressWarnings("unchecked") + void mapsStorageFailuresWithoutLeakingTheirDetails(OutboundStorageException.Reason reason) { + OutboundCoordinator coordinator = mock(OutboundCoordinator.class); + when(resource.outboundCoordinators.isUnsatisfied()).thenReturn(false); + when(resource.outboundCoordinators.get()).thenReturn(coordinator); + doThrow(new OutboundStorageException(OutboundStage.Role.PENDING, reason, "private storage details")) + .when(coordinator).admit(any(), any()); + Response response = submit(null); + int expected = switch (reason) { + case UNSUPPORTED -> 400; + case INVALID_TRANSITION -> 500; + default -> 503; + }; + assertThat(response.getStatus()).isEqualTo(expected); + assertThat(response.getEntity().toString()).doesNotContain("private storage details"); + verifyNoInteractions(routerQueue); + } + + @Test + void failingCoordinatorResolutionDoesNotUseLegacyAdmission() { + when(resource.outboundCoordinators.isUnsatisfied()).thenReturn(false); + when(resource.outboundCoordinators.get()).thenThrow(new IllegalStateException("coordinator unavailable")); + assertThat(submit(null).getStatus()).isEqualTo(500); + verifyNoInteractions(routerQueue); + } } diff --git a/sendium-core/src/test/java/gr/cytech/sendium/core/outbound/DefaultOutboundCoordinatorTest.java b/sendium-core/src/test/java/gr/cytech/sendium/core/outbound/DefaultOutboundCoordinatorTest.java index 1dc813fe..fde4e6aa 100644 --- a/sendium-core/src/test/java/gr/cytech/sendium/core/outbound/DefaultOutboundCoordinatorTest.java +++ b/sendium-core/src/test/java/gr/cytech/sendium/core/outbound/DefaultOutboundCoordinatorTest.java @@ -5,11 +5,9 @@ import gr.cytech.sendium.core.outbound.OutboundWork.Destination; import gr.cytech.sendium.core.outbound.OutboundWork.Routed; import gr.cytech.sendium.core.outbound.OutboundWork.Selected; -import gr.cytech.sendium.core.outbound.OutboundWork.SelectionId; import gr.cytech.sendium.core.outbound.OutboundWork.SourceId; import gr.cytech.sendium.core.storage.OutboundStage; import gr.cytech.sendium.core.storage.OutboundStorageException; -import gr.cytech.sendium.core.storage.SelectedRouterStore; import gr.cytech.sendium.core.storage.memory.MemoryPendingMessageStore; import gr.cytech.sendium.core.storage.memory.MemoryRoutedWorkStore; import gr.cytech.sendium.core.storage.memory.MemorySelectedRouterStore; @@ -22,7 +20,6 @@ import java.time.Duration; import java.util.ArrayList; import java.util.List; -import java.util.Optional; import java.util.Set; import java.util.UUID; import java.util.concurrent.CompletableFuture; @@ -44,7 +41,6 @@ import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.inOrder; -import static org.mockito.Mockito.mock; import static org.mockito.Mockito.never; import static org.mockito.Mockito.spy; import static org.mockito.Mockito.times; @@ -355,20 +351,18 @@ void discardWaitsForHandoffAndCannotBeUsedAfterAssignment() throws Exception { } @Test - @SuppressWarnings("unchecked") void multiSourceCleanupCanRetryAfterPartialRemoval() throws Exception { var pending = spy(new MemoryPendingMessageStore(2, Payload::copy)); - SelectedRouterStore router = mock(SelectedRouterStore.class); + var router = spy(new MemorySelectedRouterStore<>(pending, 1)); var routed = spy(new MemoryRoutedWorkStore(1, Payload::copy)); - when(router.status()).thenReturn(new OutboundStage.Status("test", false, OutboundStage.State.NEW), - new OutboundStage.Status("test", false, OutboundStage.State.READY)); var sources = Set.of(source(), source()); - var selected = new Selected<>(new SelectionId(UUID.randomUUID()), sources, new Payload("aggregate")); - when(router.take(Duration.ZERO)).thenReturn(Optional.of(selected), Optional.empty()); try (var coordinator = new DefaultOutboundCoordinator<>(pending, router, routed)) { coordinator.start(); - sources.forEach(source -> coordinator.admit(source, new Payload("part"))); - coordinator.takeForRouting(Duration.ZERO).orElseThrow(); + sources.forEach(source -> coordinator.admitHeld(source, new Payload("part"))); + coordinator.publishReady(sources, new Payload("aggregate")); + coordinator.selectAndStage(1); + var selected = coordinator.takeForRouting(Duration.ZERO).orElseThrow(); + assertThat(selected.sources()).isEqualTo(sources); coordinator.route(selected, destination("A", "aggregate")); var taken = coordinator.takeForDestination("A", Duration.ZERO).orElseThrow(); AtomicBoolean first = new AtomicBoolean(true); @@ -383,7 +377,9 @@ void multiSourceCleanupCanRetryAfterPartialRemoval() throws Exception { assertThat(result.toCompletableFuture()).isCompletedExceptionally(); assertThat(sources.stream().filter(source -> pending.find(source).isPresent()).count()).isEqualTo(1); sources.forEach(source -> coordinator.admit(source, new Payload("duplicate during cleanup"))); + sources.forEach(source -> coordinator.admitHeld(source, new Payload("held duplicate during cleanup"))); assertThat(sources.stream().filter(source -> pending.find(source).isPresent()).count()).isEqualTo(1); + fails(() -> coordinator.publishReady(sources, new Payload("late publication")), INVALID_TRANSITION); coordinator.complete(taken.id(), new CompletableFuture<>()).toCompletableFuture().join(); sources.forEach(source -> assertThat(pending.find(source)).isEmpty()); verify(routed, times(1)).complete(taken.id()); diff --git a/sendium-core/src/test/java/gr/cytech/sendium/core/smpp/server/MessagePartsOwnershipTest.java b/sendium-core/src/test/java/gr/cytech/sendium/core/smpp/server/MessagePartsOwnershipTest.java new file mode 100644 index 00000000..adc75ee3 --- /dev/null +++ b/sendium-core/src/test/java/gr/cytech/sendium/core/smpp/server/MessagePartsOwnershipTest.java @@ -0,0 +1,101 @@ +package gr.cytech.sendium.core.smpp.server; + +import gr.cytech.sendium.core.message.StandardMessage; +import org.junit.jupiter.api.Test; + +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.Callable; +import java.util.concurrent.ScheduledFuture; +import java.util.concurrent.ScheduledThreadPoolExecutor; +import java.util.concurrent.TimeUnit; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +class MessagePartsOwnershipTest { + @Test + void lateCancelledTimerCannotExpireAReusedReference() throws Exception { + try (var scheduler = new CapturingScheduler()) { + var listener = new Listener(); + var handler = new MessagePartsHandler<>(listener, 1000, scheduler); + handler.addMessagePart(part(1, "old-first")); + Callable oldTimer = scheduler.tasks.getFirst(); + handler.addMessagePart(part(2, "old-second")); + assertThat(listener.events).containsExactly(MessagePartsHandler.MessagePartsEventType.COMPLETE); + handler.addMessagePart(part(1, "new-first")); + oldTimer.call(); + assertThat(listener.events).hasSize(1); + handler.addMessagePart(part(2, "new-second")); + assertThat(listener.events).containsExactly(MessagePartsHandler.MessagePartsEventType.COMPLETE, + MessagePartsHandler.MessagePartsEventType.COMPLETE); + assertThat(listener.groups.getLast()).extracting(message -> message.body).containsExactly("new-first", "new-second"); + handler.stop(); + } + } + + @Test + void duplicateCallbackIdentifiesTheRetainedPartWithoutChangingAssembly() { + try (var scheduler = new CapturingScheduler()) { + var listener = new Listener(); + var handler = new MessagePartsHandler<>(listener, 1000, scheduler); + var first = part(1, "first"); + var duplicate = part(1, "duplicate"); + handler.addMessagePart(first); + handler.addMessagePart(duplicate); + handler.addMessagePart(part(2, "second")); + assertThat(listener.original).isSameAs(first); + assertThat(listener.duplicate).isSameAs(duplicate); + assertThat(listener.groups.getFirst()).extracting(message -> message.body).containsExactly("first", "second"); + assertThat(scheduler.tasks).hasSize(1); + handler.stop(); + } + } + + private static StandardMessage part(int ordinal, String body) { + var message = new StandardMessage(); + message.owner_id = "account"; + message.binheader = "0500037F020" + ordinal; + message.body = body; + return message; + } + + private static final class Listener implements MessagePartsEventsListener { + private final List events = new ArrayList<>(); + private final List> groups = new ArrayList<>(); + private StandardMessage original; + private StandardMessage duplicate; + + public void onMessagePartsHandlingEvent(MessagePartsHandler.MessagePartsEventType type, List parts) { + events.add(type); + groups.add(parts); + } + + public void onDuplicateMessagePart(StandardMessage original, StandardMessage duplicate) { + this.original = original; + this.duplicate = duplicate; + } + + public String getName() { + return "ownership-test"; + } + } + + private static final class CapturingScheduler extends ScheduledThreadPoolExecutor { + private final List> tasks = new ArrayList<>(); + + private CapturingScheduler() { + super(1); + } + + @Override + @SuppressWarnings("unchecked") + public ScheduledFuture schedule(Callable callable, long delay, TimeUnit unit) { + tasks.add(callable); + ScheduledFuture result = mock(ScheduledFuture.class); + when(result.cancel(false)).thenReturn(false); + return result; + } + } +} diff --git a/sendium-core/src/test/java/gr/cytech/sendium/core/smpp/server/SmppLifecycleAdmissionTest.java b/sendium-core/src/test/java/gr/cytech/sendium/core/smpp/server/SmppLifecycleAdmissionTest.java new file mode 100644 index 00000000..e7b607f3 --- /dev/null +++ b/sendium-core/src/test/java/gr/cytech/sendium/core/smpp/server/SmppLifecycleAdmissionTest.java @@ -0,0 +1,330 @@ +package gr.cytech.sendium.core.smpp.server; + +import com.cloudhopper.smpp.SmppConstants; +import com.cloudhopper.smpp.pdu.Pdu; +import com.cloudhopper.smpp.pdu.SubmitSm; +import com.cloudhopper.smpp.pdu.SubmitSmResp; +import gr.cytech.sendium.core.message.DlrReturnMetadata; +import gr.cytech.sendium.core.message.StandardMessage; +import gr.cytech.sendium.core.outbound.OutboundWork.Destination; +import gr.cytech.sendium.core.outbound.OutboundWork.SourceId; +import gr.cytech.sendium.core.queue.Queue; +import gr.cytech.sendium.core.storage.OutboundStage; +import gr.cytech.sendium.core.storage.OutboundStorageException; +import gr.cytech.sendium.util.MessageUtil; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.Timeout; +import org.mockito.ArgumentCaptor; +import utils.OutboundIngressFixture; + +import java.sql.Timestamp; +import java.time.Duration; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import java.util.UUID; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ScheduledThreadPoolExecutor; +import java.util.function.Consumer; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.doCallRealMethod; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.verify; + +@Timeout(10) +class SmppLifecycleAdmissionTest { + @Test + void completeSubmissionIsOwnedBeforeStatusOkAndDoesNotEnterLegacyQueues() throws Exception { + try (var lifecycle = new OutboundIngressFixture(1); var worker = new TestWorker()) { + worker.setIngressCoordinator(lifecycle.coordinator); + var event = submission(null, "hello", 42); + worker.onResponse = response -> { + var source = ArgumentCaptor.forClass(SourceId.class); + verify(lifecycle.coordinator).admit(source.capture(), eq(event.pMsg)); + assertThat(source.getValue().value()).isEqualTo(UUID.fromString(response.getMessageId())); + assertThat(lifecycle.pending.find(source.getValue()).orElseThrow().serial).isEqualTo(response.getMessageId()); + }; + worker.enqueueIn(event); + assertThat(worker.responses).singleElement().satisfies(response -> { + assertThat(response.getCommandStatus()).isEqualTo(SmppConstants.STATUS_OK); + assertThat(response.getSequenceNumber()).isEqualTo(42); + assertThat(response.getMessageId()).isEqualTo(event.pMsg.serial); + }); + assertThat(event.waitingForResponse).isFalse(); + assertThat(worker.getInEventQueue()).isEmpty(); + assertThat(worker.legacyQueue.isEmpty()).isTrue(); + assertThat(lifecycle.coordinator.selectAndStage(1)).isEqualTo(1); + var selected = lifecycle.coordinator.takeForRouting(Duration.ZERO).orElseThrow(); + assertThat(selected.message().dlrReturnMetadata).isEqualTo(event.pMsg.dlrReturnMetadata); + assertThat(selected.message().ctstamp).isEqualTo(event.localTimestamp.getTime()); + lifecycle.coordinator.discard(selected.id(), CompletableFuture.completedStage(null)).toCompletableFuture().join(); + } + } + + @Test + void multipartPartsAreAcknowledgedIndividuallyThenPublishedWithAllSources() throws Exception { + try (var lifecycle = new OutboundIngressFixture(2); var worker = new TestWorker()) { + worker.setIngressCoordinator(lifecycle.coordinator); + worker.onResponse = response -> { + var accepted = worker.getInEventQueue().peek(); + assertThat(accepted.sourceIds).hasSize(1); + assertThat(accepted.sourceIds.iterator().next().value()).isEqualTo( + UUID.fromString(response.getMessageId())); + accepted.sourceIds.forEach(source -> assertThat(lifecycle.pending.find(source)).isPresent()); + assertThat(lifecycle.coordinator.selectAndStage(2)).isZero(); + }; + var first = submission("0500037F0201", "Hello ", 1); + worker.enqueueIn(first); + var firstAccepted = worker.getInEventQueue().remove(); + worker.handleIngressMessages(List.of(firstAccepted)); + assertThat(worker.responses).hasSize(1); + assertThat(lifecycle.coordinator.selectAndStage(2)).isZero(); + var second = submission("0500037F0202", "world", 2); + worker.enqueueIn(second); + var secondAccepted = worker.getInEventQueue().remove(); + worker.handleIngressMessages(List.of(secondAccepted)); + var ready = worker.getInEventQueue().remove(); + Set sources = new HashSet<>(firstAccepted.sourceIds); + sources.addAll(secondAccepted.sourceIds); + assertThat(ready.sourceIds).isEqualTo(sources); + assertThat(sources).extracting(SourceId::value).containsExactlyInAnyOrder( + UUID.fromString(first.pMsg.serial), UUID.fromString(second.pMsg.serial)); + assertThat(ready.submitSm).isNull(); + assertThat(ready.waitingForResponse).isFalse(); + assertThat(ready.pMsg.body).isEqualTo("Hello world"); + assertThat(ready.pMsg.reassembledParts).containsExactly(first.pMsg.serial, second.pMsg.serial); + assertThat(lifecycle.pending.find(firstAccepted.sourceIds.iterator().next()).orElseThrow().body).isEqualTo("Hello "); + worker.handleIngressMessages(List.of(ready)); + assertThat(lifecycle.coordinator.selectAndStage(2)).isEqualTo(1); + var selected = lifecycle.coordinator.takeForRouting(Duration.ZERO).orElseThrow(); + assertThat(selected.sources()).isEqualTo(sources); + assertThat(selected.message().body).isEqualTo("Hello world"); + lifecycle.coordinator.route(selected, new Destination<>("provider", selected.message())); + var work = lifecycle.coordinator.takeForDestination("provider", Duration.ZERO).orElseThrow(); + lifecycle.coordinator.complete(work.id(), CompletableFuture.completedStage(null)).toCompletableFuture().join(); + sources.forEach(source -> assertThat(lifecycle.pending.find(source)).isEmpty()); + assertThat(worker.responses).hasSize(2).allSatisfy(response -> + assertThat(response.getCommandStatus()).isEqualTo(SmppConstants.STATUS_OK)); + assertThat(worker.legacyQueue.isEmpty()).isTrue(); + } + } + + @Test + void publicationFailureRetriesPreparedWorkWithoutAnotherAcknowledgementOrReassembly() throws Exception { + try (var lifecycle = new OutboundIngressFixture(2); var worker = new TestWorker()) { + worker.setIngressCoordinator(lifecycle.coordinator); + worker.enqueueIn(submission("0500037F0201", "one", 1)); + worker.enqueueIn(submission("0500037F0202", "two", 2)); + worker.processQueued(); + var ready = worker.getInEventQueue().remove(); + doThrow(new OutboundStorageException(OutboundStage.Role.PENDING, + OutboundStorageException.Reason.UNAVAILABLE, "publication unavailable")) + .when(lifecycle.coordinator).publishReady(any(), any()); + worker.handleIngressMessages(List.of(ready)); + assertThat(worker.getInEventQueue()).containsExactly(ready); + assertThat(lifecycle.coordinator.selectAndStage(2)).isZero(); + ready.sourceIds.forEach(source -> assertThat(lifecycle.pending.find(source)).isPresent()); + doCallRealMethod().when(lifecycle.coordinator).publishReady(any(), any()); + worker.processQueued(); + assertThat(lifecycle.coordinator.selectAndStage(2)).isEqualTo(1); + assertThat(lifecycle.coordinator.takeForRouting(Duration.ZERO).orElseThrow().message().body).isEqualTo("onetwo"); + assertThat(worker.responses).hasSize(2); + } + } + + @Test + void expiryPublishesEachAcceptedPartIndependently() throws Exception { + try (var lifecycle = new OutboundIngressFixture(2); var worker = new TestWorker()) { + worker.setIngressCoordinator(lifecycle.coordinator); + var first = submission("0500037F0301", "one", 1); + var third = submission("0500037F0303", "three", 3); + worker.enqueueIn(first); + worker.enqueueIn(third); + worker.processQueued(); + worker.expire(first.pMsg); + var events = worker.queued(); + assertThat(events).hasSize(2).allSatisfy(event -> { + assertThat(event.sourceIds).hasSize(1); + assertThat(event.submitSm).isNull(); + }); + worker.handleIngressMessages(events); + assertThat(lifecycle.coordinator.selectAndStage(2)).isEqualTo(2); + var one = lifecycle.coordinator.takeForRouting(Duration.ZERO).orElseThrow(); + var three = lifecycle.coordinator.takeForRouting(Duration.ZERO).orElseThrow(); + assertThat(one.message().body).isEqualTo("one"); + assertThat(three.message().body).isEqualTo("three"); + assertThat(one.sources()).doesNotContainAnyElementsOf(three.sources()); + assertThat(worker.responses).hasSize(2); + } + } + + @Test + void ignoredDuplicateOrdinalStillReleasesItsAcceptedSourceWithTheOriginalGroup() throws Exception { + try (var lifecycle = new OutboundIngressFixture(3); var worker = new TestWorker()) { + worker.setIngressCoordinator(lifecycle.coordinator); + worker.enqueueIn(submission("0500037F0201", "first", 1)); + worker.enqueueIn(submission("0500037F0201", "ignored", 2)); + worker.enqueueIn(submission("0500037F0202", "second", 3)); + worker.processQueued(); + var ready = worker.getInEventQueue().remove(); + assertThat(ready.sourceIds).hasSize(3); + assertThat(ready.pMsg.body).isEqualTo("firstsecond"); + assertThat(ready.pMsg.reassembledParts).hasSize(2); + worker.handleIngressMessages(List.of(ready)); + lifecycle.coordinator.selectAndStage(1); + var selected = lifecycle.coordinator.takeForRouting(Duration.ZERO).orElseThrow(); + assertThat(selected.sources()).hasSize(3); + lifecycle.coordinator.discard(selected.id(), CompletableFuture.completedStage(null)).toCompletableFuture().join(); + ready.sourceIds.forEach(source -> assertThat(lifecycle.pending.find(source)).isEmpty()); + assertThat(worker.responses).hasSize(3); + } + } + + @Test + void replayingProcessedRawEventsDoesNotReassembleOrAcknowledgeThemAgain() throws Exception { + try (var lifecycle = new OutboundIngressFixture(2); var worker = new TestWorker()) { + worker.setIngressCoordinator(lifecycle.coordinator); + worker.enqueueIn(submission("0500037F0201", "one", 1)); + worker.enqueueIn(submission("0500037F0202", "two", 2)); + var raw = worker.queued(); + worker.handleIngressMessages(raw); + worker.handleIngressMessages(raw); + assertThat(worker.getInEventQueue()).hasSize(1); + worker.processQueued(); + assertThat(lifecycle.coordinator.selectAndStage(2)).isEqualTo(1); + var selected = lifecycle.coordinator.takeForRouting(Duration.ZERO).orElseThrow(); + assertThat(selected.message().body).isEqualTo("onetwo"); + assertThat(selected.sources()).hasSize(2); + assertThat(worker.responses).hasSize(2); + } + } + + @Test + void expiryDoesNotClaimAnAcceptedPartStillWaitingInTheIngressQueue() throws Exception { + try (var lifecycle = new OutboundIngressFixture(2); var worker = new TestWorker()) { + worker.setIngressCoordinator(lifecycle.coordinator); + var first = submission("0500037F0201", "first", 1); + var second = submission("0500037F0202", "second", 2); + worker.enqueueIn(first); + worker.processQueued(); + worker.enqueueIn(second); + worker.expire(first.pMsg); + var events = worker.queued(); + var unprocessed = events.stream().filter(event -> event.submitSm != null).findFirst().orElseThrow(); + var expired = events.stream().filter(event -> event.submitSm == null).findFirst().orElseThrow(); + assertThat(expired.sourceIds).doesNotContainAnyElementsOf(unprocessed.sourceIds); + worker.handleIngressMessages(List.of(expired)); + assertThat(lifecycle.coordinator.selectAndStage(2)).isEqualTo(1); + assertThat(lifecycle.coordinator.takeForRouting(Duration.ZERO).orElseThrow().message().body).isEqualTo("first"); + worker.handleIngressMessages(List.of(unprocessed)); + assertThat(lifecycle.coordinator.selectAndStage(2)).isZero(); + worker.expire(second.pMsg); + worker.processQueued(); + assertThat(lifecycle.coordinator.selectAndStage(2)).isEqualTo(1); + assertThat(lifecycle.coordinator.takeForRouting(Duration.ZERO).orElseThrow().message().body).isEqualTo("second"); + assertThat(worker.responses).hasSize(2); + } + } + + @Test + void capacityAndQuiescenceRejectBeforeSuccessAndNeverFallBackToLegacyQueue() { + try (var lifecycle = new OutboundIngressFixture(1); var worker = new TestWorker()) { + worker.setIngressCoordinator(lifecycle.coordinator); + worker.enqueueIn(submission("0500037F0201", "held", 1)); + worker.enqueueIn(submission("0500037F0202", "full", 2)); + lifecycle.coordinator.quiesce(); + worker.enqueueIn(submission(null, "unavailable", 3)); + assertThat(worker.responses).extracting(SubmitSmResp::getCommandStatus) + .containsExactly(SmppConstants.STATUS_OK, SmppConstants.STATUS_THROTTLED, SmppConstants.STATUS_SYSERR); + assertThat(worker.getInEventQueue()).hasSize(1); + assertThat(worker.legacyQueue.isEmpty()).isTrue(); + } + } + + @Test + void unsupportedHeaderIsRejectedUsingTheExistingReassemblyValidator() { + try (var lifecycle = new OutboundIngressFixture(1); var worker = new TestWorker()) { + worker.setIngressCoordinator(lifecycle.coordinator); + worker.enqueueIn(submission("not-a-supported-concatenation-header", "bad", 1)); + assertThat(worker.responses).singleElement().satisfies(response -> + assertThat(response.getCommandStatus()).isEqualTo(SmppConstants.STATUS_SUBMITFAIL)); + assertThat(worker.getInEventQueue()).isEmpty(); + assertThat(lifecycle.coordinator.selectAndStage(1)).isZero(); + } + } + + @Test + void ingressBindingCannotChangeAfterAdmission() { + try (var lifecycle = new OutboundIngressFixture(1); var worker = new TestWorker()) { + worker.setIngressCoordinator(lifecycle.coordinator); + worker.enqueueIn(submission(null, "one", 1)); + assertThatThrownBy(() -> worker.setIngressCoordinator(lifecycle.coordinator)) + .isInstanceOf(IllegalStateException.class); + } + } + + private static InEvent submission(String udh, String body, int sequence) { + var message = new StandardMessage(); + message.from = "sender"; + message.to = "306900000001"; + message.owner_id = "account-a"; + message.systemId = "system-a"; + message.body = body; + message.binheader = udh; + message.acked = true; + message.dlrReturnMetadata = DlrReturnMetadata.smpp("account-a", "system-a", message.from, message.to); + var request = new SubmitSm(); + request.setSequenceNumber(sequence); + return new InEvent<>(message, request, 1, new Timestamp(1_000)); + } + + private static final class TestWorker extends SmppServerWorker implements AutoCloseable { + private final Queue legacyQueue; + private final List responses = new ArrayList<>(); + private final ScheduledThreadPoolExecutor scheduler = new ScheduledThreadPoolExecutor(1); + private Consumer onResponse = ignored -> {}; + + private TestWorker() { + this(new Queue<>()); + } + + private TestWorker(Queue queue) { + super(new SmppServerWorkerReassemblyTest.TestConfigurationProvider(), "ingress", queue); + legacyQueue = queue; + messagePartsHandler.stop(); + setMessagePartsHandler(new MessagePartsHandler<>(new CcatMessagePartsEventsListener(), 60_000, scheduler)); + } + + @Override + public void enqueueOut(Pdu event) { + var response = (SubmitSmResp) event; + onResponse.accept(response); + responses.add(response); + } + + private List> queued() { + var events = new ArrayList>(); + getInEventQueue().drainTo(events); + return events; + } + + private void processQueued() { + handleIngressMessages(queued()); + } + + private void expire(StandardMessage part) { + messagePartsHandler.new DelayedMessagePartsTask(MessageUtil.getMessageReference(part)).call(); + } + + @Override + public void close() { + messagePartsHandler.stop(); + scheduler.shutdownNow(); + } + } +} diff --git a/sendium-core/src/test/java/gr/cytech/sendium/core/smpp/server/SmppServerWorkerReassemblyTest.java b/sendium-core/src/test/java/gr/cytech/sendium/core/smpp/server/SmppServerWorkerReassemblyTest.java index cb3e398e..4c5d6b02 100644 --- a/sendium-core/src/test/java/gr/cytech/sendium/core/smpp/server/SmppServerWorkerReassemblyTest.java +++ b/sendium-core/src/test/java/gr/cytech/sendium/core/smpp/server/SmppServerWorkerReassemblyTest.java @@ -448,7 +448,7 @@ public void enqueueOut(Pdu event) { } - private static class TestConfigurationProvider implements SendiumConfigurationProvider { + static class TestConfigurationProvider implements SendiumConfigurationProvider { private final Map props = new HashMap<>(); @Override diff --git a/sendium-core/src/test/java/gr/cytech/sendium/core/storage/memory/HeldAdmissionTest.java b/sendium-core/src/test/java/gr/cytech/sendium/core/storage/memory/HeldAdmissionTest.java new file mode 100644 index 00000000..7d7e8502 --- /dev/null +++ b/sendium-core/src/test/java/gr/cytech/sendium/core/storage/memory/HeldAdmissionTest.java @@ -0,0 +1,444 @@ +package gr.cytech.sendium.core.storage.memory; + +import gr.cytech.sendium.core.message.StandardMessage; +import gr.cytech.sendium.core.outbound.DefaultOutboundCoordinator; +import gr.cytech.sendium.core.outbound.OutboundWork.Destination; +import gr.cytech.sendium.core.outbound.OutboundWork.SourceId; +import gr.cytech.sendium.core.storage.OutboundStorageException; +import org.assertj.core.api.ThrowableAssert.ThrowingCallable; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.Timeout; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; + +import java.time.Clock; +import java.time.Duration; +import java.time.Instant; +import java.time.ZoneOffset; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import java.util.UUID; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.function.UnaryOperator; + +import static gr.cytech.sendium.core.storage.OutboundStorageException.Reason.CAPACITY_EXCEEDED; +import static gr.cytech.sendium.core.storage.OutboundStorageException.Reason.INVALID_TRANSITION; +import static gr.cytech.sendium.core.storage.OutboundStorageException.Reason.UNAVAILABLE; +import static gr.cytech.sendium.core.storage.OutboundStorageException.Reason.UNSUPPORTED; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +@Timeout(10) +class HeldAdmissionTest { + private static final Instant NOW = Instant.parse("2026-09-29T00:00:00Z"); + private static final Clock CLOCK = Clock.fixed(NOW, ZoneOffset.UTC); + + @Test + void heldSourcesAreRetainedButNeverIndividuallySelected() throws Exception { + try (var stores = new Stores(3, 3)) { + SourceId first = source(); + SourceId second = source(); + Payload part = payload("first", 3); + part.tags.add("original"); + stores.pending.admitHeld(first, part); + stores.pending.admitHeld(second, payload("second", 3)); + part.body = "changed"; + part.tags.clear(); + assertThat(stores.pending.find(first).orElseThrow().body).isEqualTo("first"); + assertThat(stores.pending.find(first).orElseThrow().tags).containsExactly("original"); + assertThat(stores.router.selectAndStage(3)).isZero(); + assertThat(stores.router.take(Duration.ZERO)).isEmpty(); + SourceId ordinary = source(); + stores.pending.admit(ordinary, payload("ordinary", 1)); + assertThat(stores.router.selectAndStage(3)).isEqualTo(1); + assertThat(stores.router.take(Duration.ZERO).orElseThrow().sources()).containsExactly(ordinary); + } + } + + @Test + void publicationAtFullCapacitySelectsOneAggregateWithAllOriginalSources() throws Exception { + try (var stores = new Stores(2, 1)) { + SourceId first = source(); + SourceId second = source(); + Set sources = new HashSet<>(Set.of(first, second)); + stores.pending.admitHeld(first, payload("first", 2)); + stores.pending.admitHeld(second, payload("second", 2)); + fails(() -> stores.pending.admitHeld(source(), payload("full", 2)), CAPACITY_EXCEEDED); + Payload aggregate = payload("firstsecond", 2); + aggregate.tags.add("assembled"); + stores.pending.publishReady(sources, aggregate); + sources.clear(); + aggregate.body = "changed"; + aggregate.tags.clear(); + assertThat(stores.router.selectAndStage(10)).isEqualTo(1); + var selected = stores.router.take(Duration.ZERO).orElseThrow(); + assertThat(selected.sources()).containsExactlyInAnyOrder(first, second); + assertThat(selected.message().body).isEqualTo("firstsecond"); + assertThat(selected.message().tags).containsExactly("assembled"); + assertThat(stores.pending.find(first).orElseThrow().body).isEqualTo("first"); + assertThat(stores.pending.find(second).orElseThrow().body).isEqualTo("second"); + assertThat(stores.router.selectAndStage(10)).isZero(); + assertThat(stores.router.take(Duration.ZERO)).isEmpty(); + stores.router.markRouted(selected.id()); + fails(() -> stores.pending.admit(source(), payload("still full", 2)), CAPACITY_EXCEEDED); + stores.pending.complete(selected.sources()); + stores.router.complete(selected.id()); + stores.pending.admit(source(), payload("next", 2)); + } + } + + @Test + void publicationIsFirstWinsBeforeAndAfterSelectionAndRouting() throws Exception { + AtomicInteger snapshots = new AtomicInteger(); + try (var stores = new Stores(2, 1, message -> { + snapshots.incrementAndGet(); + return message.copy(); + })) { + var sources = Set.of(source(), source()); + sources.forEach(source -> stores.pending.admitHeld(source, payload("part", 2))); + stores.pending.publishReady(sources, payload("first publication", 2)); + int count = snapshots.get(); + stores.pending.publishReady(new HashSet<>(sources), payload("replacement", 3)); + assertThat(snapshots).hasValue(count); + stores.router.selectAndStage(1); + stores.pending.publishReady(sources, payload("after selection", 3)); + var selected = stores.router.take(Duration.ZERO).orElseThrow(); + assertThat(selected.message().body).isEqualTo("first publication"); + stores.pending.publishReady(sources, payload("while taken", 3)); + stores.router.markRouted(selected.id()); + stores.pending.publishReady(sources, payload("after routing", 3)); + assertThat(stores.router.selectAndStage(1)).isZero(); + assertThat(stores.router.take(Duration.ZERO)).isEmpty(); + stores.pending.complete(sources); + stores.router.complete(selected.id()); + fails(() -> stores.pending.publishReady(sources, payload("after cleanup", 2)), INVALID_TRANSITION); + assertThat(stores.router.selectAndStage(1)).isZero(); + } + } + + @Test + void duplicateAdmissionCannotPromoteHeldWorkOrHoldOrdinaryWork() throws Exception { + try (var stores = new Stores(2, 2)) { + SourceId held = source(); + SourceId ordinary = source(); + stores.pending.admitHeld(held, payload("held", 2)); + stores.pending.admit(ordinary, payload("ordinary", 1)); + stores.pending.admit(held, payload("promote", 3)); + stores.pending.admitHeld(held, payload("overwrite", 3)); + stores.pending.admitHeld(ordinary, payload("hold", 3)); + assertThat(stores.router.selectAndStage(2)).isEqualTo(1); + assertThat(stores.router.take(Duration.ZERO).orElseThrow().sources()).containsExactly(ordinary); + assertThat(stores.pending.find(held).orElseThrow().body).isEqualTo("held"); + fails(() -> stores.pending.publishReady(Set.of(ordinary), payload("ordinary replacement", 2)), INVALID_TRANSITION); + } + } + + @Test + void callerCanPublishAnExpiredPartIndividuallyWithoutReleasingOtherHeldParts() throws Exception { + try (var stores = new Stores(2, 2)) { + SourceId first = source(); + SourceId second = source(); + stores.pending.admitHeld(first, payload("first", 2)); + stores.pending.admitHeld(second, payload("second", 2)); + stores.pending.publishReady(Set.of(first), stores.pending.find(first).orElseThrow()); + assertThat(stores.router.selectAndStage(2)).isEqualTo(1); + var selected = stores.router.take(Duration.ZERO).orElseThrow(); + assertThat(selected.sources()).containsExactly(first); + assertThat(selected.message().body).isEqualTo("first"); + assertThat(stores.pending.find(second)).isPresent(); + assertThat(stores.router.selectAndStage(2)).isZero(); + fails(() -> stores.pending.publishReady(Set.of(first, second), payload("late aggregate", 2)), INVALID_TRANSITION); + stores.pending.publishReady(Set.of(second), stores.pending.find(second).orElseThrow()); + assertThat(stores.router.selectAndStage(2)).isEqualTo(1); + } + } + + @Test + void missingOrOverlappingSourceSetsFailWithoutBindingUnpublishedSources() throws Exception { + try (var stores = new Stores(3, 3)) { + SourceId first = source(); + SourceId second = source(); + SourceId third = source(); + for (var source : Set.of(first, second, third)) { + stores.pending.admitHeld(source, payload("part", 2)); + } + fails(() -> stores.pending.publishReady(Set.of(first, source()), payload("unknown", 2)), INVALID_TRANSITION); + assertThat(stores.router.selectAndStage(3)).isZero(); + stores.pending.publishReady(Set.of(first, second), payload("aggregate", 2)); + fails(() -> stores.pending.publishReady(Set.of(first), payload("subset", 2)), INVALID_TRANSITION); + fails(() -> stores.pending.publishReady(Set.of(first, second, third), payload("superset", 2)), INVALID_TRANSITION); + fails(() -> stores.pending.publishReady(Set.of(first, third), payload("overlap", 2)), INVALID_TRANSITION); + stores.pending.publishReady(Set.of(third), payload("third", 2)); + assertThat(stores.router.selectAndStage(3)).isEqualTo(2); + assertThat(stores.router.take(Duration.ZERO).orElseThrow().sources()).containsExactlyInAnyOrder(first, second); + assertThat(stores.router.take(Duration.ZERO).orElseThrow().sources()).containsExactly(third); + } + } + + @Test + void snapshotFailureLeavesHeldSourcesAvailableForACompletePublicationRetry() throws Exception { + AtomicBoolean fail = new AtomicBoolean(false); + try (var stores = new Stores(2, 2, message -> { + if (fail.get()) { + throw new IllegalStateException("snapshot unavailable"); + } + return message.copy(); + })) { + var sources = Set.of(source(), source()); + sources.forEach(source -> stores.pending.admitHeld(source, payload("part", 2))); + fail.set(true); + fails(() -> stores.pending.publishReady(sources, payload("aggregate", 2)), UNAVAILABLE); + fail.set(false); + sources.forEach(source -> assertThat(stores.pending.find(source)).isPresent()); + assertThat(stores.router.selectAndStage(2)).isZero(); + stores.pending.publishReady(sources, payload("aggregate", 2)); + assertThat(stores.router.selectAndStage(2)).isEqualTo(1); + assertThat(stores.router.take(Duration.ZERO).orElseThrow().sources()).isEqualTo(sources); + } + } + + @Test + void eligibilityIsEvaluatedAtPublicationAndFailureDoesNotPublishPartOfAGroup() throws Exception { + AtomicInteger evaluations = new AtomicInteger(); + AtomicBoolean fail = new AtomicBoolean(true); + try (var pending = new MemoryPendingMessageStore(2, Payload::copy, message -> { + evaluations.incrementAndGet(); + if (fail.get()) { + throw new IllegalStateException("eligibility unavailable"); + } + return message.available; + }); var router = new MemorySelectedRouterStore<>(pending, 1, CLOCK)) { + pending.open(); + router.open(); + var sources = Set.of(source(), source()); + sources.forEach(source -> pending.admitHeld(source, new Payload("part", 3, null))); + assertThat(evaluations).hasValue(0); + fails(() -> pending.publishReady(sources, payload("aggregate", 2)), UNSUPPORTED); + assertThat(router.selectAndStage(1)).isZero(); + fail.set(false); + pending.publishReady(sources, payload("aggregate", 2)); + assertThat(router.selectAndStage(1)).isEqualTo(1); + assertThat(router.take(Duration.ZERO).orElseThrow().message().body).isEqualTo("aggregate"); + assertThat(evaluations).hasValue(2); + } + } + + @Test + void preparedPriorityAndEligibilityStillPassThroughBoundedSelection() throws Exception { + try (var stores = new Stores(4, 1)) { + SourceId ordinary = source(); + SourceId low = source(); + SourceId due = source(); + SourceId future = source(); + stores.pending.admit(ordinary, payload("ordinary", 2)); + for (var source : Set.of(low, due, future)) { + stores.pending.admitHeld(source, payload("raw high priority", 5)); + } + stores.pending.publishReady(Set.of(low), payload("prepared low", 1)); + stores.pending.publishReady(Set.of(due), payload("prepared high", 3)); + stores.pending.publishReady(Set.of(future), new Payload("future", 9, NOW.plusSeconds(1))); + assertThat(stores.router.take(Duration.ZERO)).isEmpty(); + assertThat(stores.router.selectAndStage(10)).isEqualTo(1); + var selected = stores.router.take(Duration.ZERO).orElseThrow(); + assertThat(selected.message().body).isEqualTo("prepared high"); + assertThat(stores.router.selectAndStage(10)).isZero(); + stores.router.markRouted(selected.id()); + assertThat(stores.router.selectAndStage(10)).isEqualTo(1); + selected = stores.router.take(Duration.ZERO).orElseThrow(); + assertThat(selected.message().body).isEqualTo("ordinary"); + stores.router.markRouted(selected.id()); + stores.router.selectAndStage(10); + selected = stores.router.take(Duration.ZERO).orElseThrow(); + assertThat(selected.message().body).isEqualTo("prepared low"); + stores.router.markRouted(selected.id()); + assertThat(stores.router.selectAndStage(10)).isZero(); + } + } + + @Test + void concurrentPublicationAndSelectionYieldOneWholeGroup() throws Exception { + try (var stores = new Stores(2, 2); var executor = Executors.newVirtualThreadPerTaskExecutor()) { + var sources = Set.of(source(), source()); + sources.forEach(source -> stores.pending.admitHeld(source, payload("part", 2))); + var jobs = new ArrayList>(); + for (int i = 0; i < 16; i++) { + final int index = i; + jobs.add(executor.submit(() -> { + stores.pending.publishReady(sources, payload("aggregate-" + index, 2)); + stores.router.selectAndStage(2); + })); + } + for (var job : jobs) { + job.get(2, TimeUnit.SECONDS); + } + var selected = stores.router.take(Duration.ZERO).orElseThrow(); + assertThat(selected.sources()).isEqualTo(sources); + assertThat(selected.message().body).startsWith("aggregate-"); + assertThat(stores.router.take(Duration.ZERO)).isEmpty(); + assertThat(stores.router.selectAndStage(2)).isZero(); + } + } + + @Test + void competingGroupsCannotBothOwnAnOverlappingSource() throws Exception { + try (var stores = new Stores(3, 3); var executor = Executors.newVirtualThreadPerTaskExecutor()) { + SourceId shared = source(); + SourceId left = source(); + SourceId right = source(); + for (var source : Set.of(shared, left, right)) { + stores.pending.admitHeld(source, payload("part", 2)); + } + var groups = List.of(Set.of(shared, left), Set.of(shared, right)); + var jobs = new ArrayList>(); + for (var group : groups) { + jobs.add(executor.submit(() -> { + try { + stores.pending.publishReady(group, payload("aggregate", 2)); + return true; + } catch (OutboundStorageException failure) { + assertThat(failure.reason()).isEqualTo(INVALID_TRANSITION); + return false; + } + })); + } + boolean leftWon = jobs.getFirst().get(2, TimeUnit.SECONDS); + assertThat(jobs.getLast().get(2, TimeUnit.SECONDS)).isNotEqualTo(leftWon); + assertThat(stores.router.selectAndStage(3)).isEqualTo(1); + var selected = stores.router.take(Duration.ZERO).orElseThrow(); + assertThat(selected.sources()).isEqualTo(groups.get(leftWon ? 0 : 1)); + SourceId unbound = leftWon ? right : left; + stores.pending.publishReady(Set.of(unbound), payload("unbound", 2)); + assertThat(stores.router.selectAndStage(3)).isEqualTo(1); + } + } + + @ParameterizedTest + @ValueSource(booleans = {false, true}) + void coordinatorRetainsHeldSourcesThroughHandoffOrUnroutedDiscard(boolean discard) throws Exception { + try (var pending = new MemoryPendingMessageStore(2, Payload::copy); + var router = new MemorySelectedRouterStore<>(pending, 1); + var routed = new MemoryRoutedWorkStore(1, Payload::copy); + var coordinator = new DefaultOutboundCoordinator<>(pending, router, routed)) { + coordinator.start(); + var sources = Set.of(source(), source()); + sources.forEach(source -> coordinator.admitHeld(source, payload("part", 2))); + assertThat(coordinator.selectAndStage(1)).isZero(); + coordinator.publishReady(sources, payload("aggregate", 2)); + assertThat(coordinator.takeForRouting(Duration.ZERO)).isEmpty(); + coordinator.selectAndStage(1); + var selected = coordinator.takeForRouting(Duration.ZERO).orElseThrow(); + assertThat(selected.sources()).isEqualTo(sources); + var handoff = new CompletableFuture(); + java.util.concurrent.CompletionStage completion; + if (discard) { + completion = coordinator.discard(selected.id(), handoff); + } else { + coordinator.route(selected, new Destination<>("A", selected.message())); + var work = coordinator.takeForDestination("A", Duration.ZERO).orElseThrow(); + completion = coordinator.complete(work.id(), handoff); + } + coordinator.publishReady(sources, payload("late duplicate", 2)); + sources.forEach(source -> assertThat(pending.find(source).orElseThrow().body).isEqualTo("part")); + assertThat(coordinator.selectAndStage(1)).isZero(); + handoff.complete(null); + completion.toCompletableFuture().join(); + sources.forEach(source -> assertThat(pending.find(source)).isEmpty()); + fails(() -> coordinator.publishReady(sources, payload("after cleanup", 2)), INVALID_TRANSITION); + } + } + + @Test + void quiescenceAllowsExistingHeldWorkToBecomeReadyButRejectsNewAdmissionAndSelection() { + try (var pending = new MemoryPendingMessageStore(1, Payload::copy); + var router = new MemorySelectedRouterStore<>(pending, 1); + var routed = new MemoryRoutedWorkStore(1, Payload::copy); + var coordinator = new DefaultOutboundCoordinator<>(pending, router, routed)) { + SourceId source = source(); + fails(() -> coordinator.admitHeld(source, payload("part", 2)), UNAVAILABLE); + coordinator.start(); + coordinator.admitHeld(source, payload("part", 2)); + coordinator.quiesce(); + coordinator.publishReady(Set.of(source), payload("released at shutdown", 2)); + assertThat(pending.find(source)).isPresent(); + fails(() -> coordinator.admitHeld(source(), payload("new", 2)), UNAVAILABLE); + fails(() -> coordinator.selectAndStage(1), UNAVAILABLE); + } + } + + @Test + void invalidPublicationCannotCreateSourcesAndNewInstancesRecoverNoHeldOrReadyState() { + var sources = Set.of(source()); + try (var stores = new Stores(1, 1)) { + assertThatThrownBy(() -> stores.pending.publishReady(Set.of(), payload("empty", 2))) + .isInstanceOf(IllegalArgumentException.class); + fails(() -> stores.pending.publishReady(sources, payload("unknown", 2)), INVALID_TRANSITION); + stores.pending.admitHeld(sources.iterator().next(), payload("held", 2)); + assertThatThrownBy(() -> stores.pending.publishReady(sources, null)).isInstanceOf(NullPointerException.class); + } + try (var fresh = new Stores(1, 1)) { + assertThat(fresh.pending.find(sources.iterator().next())).isEmpty(); + fails(() -> fresh.pending.publishReady(sources, payload("unknown", 2)), INVALID_TRANSITION); + fresh.pending.close(); + fails(() -> fresh.pending.admitHeld(source(), payload("closed", 2)), UNAVAILABLE); + fails(() -> fresh.pending.publishReady(sources, payload("closed", 2)), UNAVAILABLE); + } + } + + private static Payload payload(String body, int priority) { + return new Payload(body, priority, NOW); + } + + private static SourceId source() { + return new SourceId(UUID.randomUUID()); + } + + private static void fails(ThrowingCallable operation, OutboundStorageException.Reason reason) { + assertThatThrownBy(operation).isInstanceOfSatisfying(OutboundStorageException.class, + failure -> assertThat(failure.reason()).isEqualTo(reason)); + } + + private static final class Stores implements AutoCloseable { + private final MemoryPendingMessageStore pending; + private final MemorySelectedRouterStore router; + + private Stores(int pendingCapacity, int routerCapacity) { + this(pendingCapacity, routerCapacity, Payload::copy); + } + + private Stores(int pendingCapacity, int routerCapacity, UnaryOperator snapshots) { + pending = new MemoryPendingMessageStore<>(pendingCapacity, snapshots, message -> message.available); + router = new MemorySelectedRouterStore<>(pending, routerCapacity, CLOCK); + pending.open(); + router.open(); + } + + @Override + public void close() { + router.close(); + pending.close(); + } + } + + private static final class Payload extends StandardMessage { + private final Instant available; + private final List tags = new ArrayList<>(); + + private Payload(String body, int priority, Instant available) { + this.body = body; + this.priority = priority; + this.available = available; + } + + private Payload copy() { + var copy = new Payload(body, priority, available); + copy.tags.addAll(tags); + return copy; + } + } +} diff --git a/sendium-core/src/test/java/gr/cytech/sendium/routing/StandardOutgoingWorkerAdmissionTest.java b/sendium-core/src/test/java/gr/cytech/sendium/routing/StandardOutgoingWorkerAdmissionTest.java new file mode 100644 index 00000000..54ffeea0 --- /dev/null +++ b/sendium-core/src/test/java/gr/cytech/sendium/routing/StandardOutgoingWorkerAdmissionTest.java @@ -0,0 +1,67 @@ +package gr.cytech.sendium.routing; + +import gr.cytech.sendium.conf.SendiumConfigurationHandler; +import gr.cytech.sendium.core.AbstractOutWorker; +import gr.cytech.sendium.core.message.StandardMessage; +import gr.cytech.sendium.core.outbound.OutboundCoordinator; +import gr.cytech.sendium.core.queue.InMemoryQueueProvider; +import gr.cytech.sendium.core.queue.Queue; +import gr.cytech.sendium.core.smpp.server.SmppServerWorker; +import gr.cytech.sendium.external.WorkerResourceProvider; +import jakarta.enterprise.inject.Instance; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; + +import java.lang.annotation.Annotation; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.inOrder; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +class StandardOutgoingWorkerAdmissionTest { + @ParameterizedTest + @ValueSource(strings = {"legacy", "coordinated", "unavailable"}) + @SuppressWarnings("unchecked") + void ingressBindingIsResolvedBeforeStartupWithoutFailureFallback(String binding) { + boolean coordinated = !binding.equals("legacy"); + var handler = new StandardOutgoingWorkerHandler(); + handler.availableWorkers = mock(Instance.class); + Instance> selected = mock(Instance.class); + SmppServerWorker worker = mock(SmppServerWorker.class); + when(handler.availableWorkers.select(any(Annotation[].class))).thenReturn(selected); + when(selected.get()).thenReturn(worker); + when(worker.getType()).thenReturn(SmppServerWorker.TYPE_SMPP_SERVER); + when(worker.isKeepOnRunning()).thenReturn(true); + handler.configurationHandler = mock(SendiumConfigurationHandler.class); + handler.queueProvider = mock(InMemoryQueueProvider.class); + when(handler.queueProvider.getRouterQueue()).thenReturn(new Queue<>()); + handler.workerResourceProvider = mock(WorkerResourceProvider.class); + handler.routingManager = mock(StandardRoutingManager.class); + handler.outboundCoordinators = mock(Instance.class); + OutboundCoordinator coordinator = mock(OutboundCoordinator.class); + when(handler.outboundCoordinators.isUnsatisfied()).thenReturn(!coordinated); + when(handler.outboundCoordinators.get()).thenReturn(coordinator); + + if (binding.equals("unavailable")) { + when(handler.outboundCoordinators.get()).thenThrow(new IllegalStateException("coordinator unavailable")); + assertThat(handler.startWorker("server", SmppServerWorker.TYPE_SMPP_SERVER)).isNull(); + verify(worker, never()).start(); + verify(worker, never()).setIngressCoordinator(any()); + return; + } + assertThat(handler.startWorker("server", SmppServerWorker.TYPE_SMPP_SERVER)).isSameAs(worker); + + if (coordinated) { + var order = inOrder(worker); + order.verify(worker).setIngressCoordinator(coordinator); + order.verify(worker).start(); + } else { + verify(worker, never()).setIngressCoordinator(any()); + verify(handler.outboundCoordinators, never()).get(); + } + } +} diff --git a/sendium-core/src/test/java/utils/OutboundIngressFixture.java b/sendium-core/src/test/java/utils/OutboundIngressFixture.java new file mode 100644 index 00000000..3cdacc9e --- /dev/null +++ b/sendium-core/src/test/java/utils/OutboundIngressFixture.java @@ -0,0 +1,45 @@ +package utils; + +import gr.cytech.sendium.core.message.StandardMessage; +import gr.cytech.sendium.core.outbound.DefaultOutboundCoordinator; +import gr.cytech.sendium.core.storage.memory.MemoryPendingMessageStore; +import gr.cytech.sendium.core.storage.memory.MemoryRoutedWorkStore; +import gr.cytech.sendium.core.storage.memory.MemorySelectedRouterStore; + +import java.util.ArrayList; +import java.util.HashMap; + +import static org.mockito.Mockito.spy; + +public final class OutboundIngressFixture implements AutoCloseable { + public final MemoryPendingMessageStore pending; + public final MemorySelectedRouterStore router; + public final MemoryRoutedWorkStore routed; + public final DefaultOutboundCoordinator coordinator; + + public OutboundIngressFixture(int capacity) { + pending = new MemoryPendingMessageStore<>(capacity, OutboundIngressFixture::snapshot); + router = new MemorySelectedRouterStore<>(pending, capacity); + routed = new MemoryRoutedWorkStore<>(capacity, OutboundIngressFixture::snapshot); + coordinator = spy(new DefaultOutboundCoordinator<>(pending, router, routed)); + coordinator.start(); + } + + // These wire fixtures use scalar extension fields and string-valued collections, not arbitrary object graphs. + private static StandardMessage snapshot(StandardMessage message) { + try { + var copy = (StandardMessage) message.clone(); + copy.attrs = message.attrs == null ? null : new HashMap<>(message.attrs); + copy.tlvs = message.tlvs == null ? null : new HashMap<>(message.tlvs); + copy.reassembledParts = message.reassembledParts == null ? null : new ArrayList<>(message.reassembledParts); + return copy; + } catch (CloneNotSupportedException failure) { + throw new AssertionError(failure); + } + } + + @Override + public void close() { + coordinator.close(); + } +} From 3051c34e36f53deeabf34cc4d7253eb74970be43 Mon Sep 17 00:00:00 2001 From: pavlos Date: Thu, 1 Oct 2026 16:45:22 +0300 Subject: [PATCH 07/20] feat(storage): connect lifecycle routing and provider completion Signed-off-by: pavlos --- docs/14-outbound-storage.md | 51 +++- .../sendium/core/AbstractOutWorker.java | 73 ++++++ .../client/CoordinatedSmppSubmission.java | 113 ++++++++ .../smpp/client/SmppClientSessionHandler.java | 14 + .../core/smpp/client/SmppClientWorker.java | 107 ++++++++ .../core/worker/StandardMessageTracker.java | 27 +- .../cytech/sendium/core/worker/Tracker.java | 18 ++ .../routing/StandardOutboundDispatch.java | 139 ++++++++++ .../routing/StandardRoutingManager.java | 30 ++- .../smpp/client/SmppClientWorkerTest.java | 241 ++++++++++++++++++ .../worker/StandardMessageTrackerTest.java | 19 ++ .../routing/StandardOutboundDispatchTest.java | 156 ++++++++++++ 12 files changed, 978 insertions(+), 10 deletions(-) create mode 100644 sendium-core/src/main/java/gr/cytech/sendium/core/smpp/client/CoordinatedSmppSubmission.java create mode 100644 sendium-core/src/main/java/gr/cytech/sendium/routing/StandardOutboundDispatch.java create mode 100644 sendium-core/src/test/java/gr/cytech/sendium/routing/StandardOutboundDispatchTest.java diff --git a/docs/14-outbound-storage.md b/docs/14-outbound-storage.md index 15fa772f..00bef252 100644 --- a/docs/14-outbound-storage.md +++ b/docs/14-outbound-storage.md @@ -54,15 +54,56 @@ consumer team's responsibility and require separate approval for Sendium support ## Copied routing boundary The existing `+vendor`/`copied` routing feature and its public API remain available on the existing -routing path. The new storage abstraction does not support copied routing. Its routing integration -must reject a copied-route request as `UNSUPPORTED` before assignment or dispatch, even if the -lookup happens to produce only one destination. It must not silently choose one result, strip the +routing path. The new storage abstraction does not support copied routing. The opt-in lifecycle +lookup rejects a matching copied-route rule as `UNSUPPORTED` before assignment or dispatch, even +if lookup would produce only one destination. It does not silently choose one result, strip the copy flag, or emulate fan-out using several assignments for the same source. -This guard belongs to the upcoming routing integration task; this profile-selection step does not change -existing routing behavior. The single-destination scope is the owner's revision to the copied-route +This guard does not change existing routing behavior. The single-destination scope is the owner's revision to the copied-route requirements previously described in #337/#338. Those issue bodies have not been edited here. +`StandardOutboundDispatch`, constructed explicitly with a coordinator and Sendium's routing manager, +can route one already selected item to one recorded destination and take that work for provider +processing. The caller retains the selected handle if recording the destination needs a retry. +A routing miss returns the selection to the router; a matched copied rule fails before +assignment and likewise returns the selection. `finishProviderParts` is called only after the provider +has reported an outcome for every part. It waits for each part's required DLR/tracking handoff before +invoking coordinated completion. While a part handoff is outstanding, its sources remain retained; +failure leaves them owned for completion-only retry with successful handoff stages. Duplicate terminal +callbacks do not release another source. A successful stage represents a part requiring no handoff. + +### Coordinated SMPP provider processing + +`submitToProvider` connects an explicitly taken work item to `SmppClientWorker.submitCoordinated`. +It registers every request before sending and uses typed request references for provider callbacks. +The session handler recognizes these references for submit responses, request expiry and recoverable +PDU errors before applying its legacy message casts. Ordinary message references retain their existing behavior. + +Each request is claimed once, so duplicate callbacks cannot start another retry or tracking handoff. +Provider acceptance and rejection use the tracker's completion-stage handoff boundary; asynchronous +trackers can return a stage for the actual handoff. The standard rejection handoff propagates an +interrupted DLR router enqueue instead of classifying it as success. A failed handoff stays pending +and is retried through `ProviderExecution.retryPendingHandoffs`, without resending an accepted SMS. +Terminal store cleanup can be retried through `retryTerminalCleanup` without repeating provider processing. + +The production multipart retry payload choices are retained: the first request references the full +original message and later requests reference individual cloned parts. A first-send exception aborts +the remaining sends and retries the full payload; a later send exception retries that part. Negative +message IDs on cloned later parts retain the existing no-tracking behavior. All request outcomes and +replacement attempts remain under the parent work, without new pending admissions or copied-route branches. + +For retry responses, the worker executes its existing failure-filter/retry-counter/delay-policy logic +inside a scoped scheduling capture. Worker, router and delayed queue actions are redirected to tracked +replacement attempts. Router retries perform lifecycle routing lookup, including copied-route rejection; +same-worker retries reuse that worker. The scope is removed before replacement work begins and does +not change scheduling for ordinary legacy messages. Parent sources complete only once the original +outstanding requests, their replacement attempts, and required tracking handoffs all finish. + +This provider boundary is explicitly assembled and is not automatically wired into the standalone +router/worker loops. Worker-loop preparation, application lifecycle activation and coordinated shutdown +remain subsequent integration work. The new runtime attempt bookkeeping is non-durable; it does not +provide provider-outcome checkpoints or restart recovery. + ## Components The public APIs are grouped by responsibility under `gr.cytech.sendium.core`: diff --git a/sendium-core/src/main/java/gr/cytech/sendium/core/AbstractOutWorker.java b/sendium-core/src/main/java/gr/cytech/sendium/core/AbstractOutWorker.java index a5b39b9d..fd64dbab 100644 --- a/sendium-core/src/main/java/gr/cytech/sendium/core/AbstractOutWorker.java +++ b/sendium-core/src/main/java/gr/cytech/sendium/core/AbstractOutWorker.java @@ -34,6 +34,8 @@ import java.util.Iterator; import java.util.List; import java.util.Map; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionStage; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; import java.util.concurrent.DelayQueue; @@ -43,6 +45,7 @@ import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.ScheduledThreadPoolExecutor; import java.util.concurrent.TimeUnit; +import java.util.function.BiFunction; public abstract class AbstractOutWorker implements HealthCheckReporter { @@ -205,6 +208,7 @@ private void saveMessage(final M msg) { private double transactionsPerSecond; private RateLimiter rateLimiter; private FailDelayPolicy failDelayPolicy; + private final ThreadLocal> coordinatedRetryScheduling = new ThreadLocal<>(); protected AbstractOutWorker() { // for fast initialization @@ -530,6 +534,9 @@ private void applyFilters(List> filters, M msg, FilterLifec } public void enqueue(M pMsg) throws InterruptedException { + if (scheduleCoordinatedRetry(pMsg, false, 0)) { + return; + } if (isFilter()) { logger.debug("Filters do not have queues"); return; @@ -696,6 +703,9 @@ public String getQueueName() { * @param msg The message to enqueue to the router */ public final void enqueueToRouter(M msg) throws InterruptedException { + if (scheduleCoordinatedRetry(msg, true, 0)) { + return; + } routerQueue.enqueue(msg); if (MessageTrace.shouldLog(configurationProvider, MessageTrace.EVENT_ENQUEUED)) { logger.info("message.enqueued destination=router {}", MessageTrace.identifiers(msg)); @@ -715,6 +725,9 @@ public void enqueueToRouterNoExceptions(M msg) { } public final void enqueueDelayed(M msg, long delay) { + if (scheduleCoordinatedRetry(msg, false, delay)) { + return; + } if (isFilter()) { throw new UnsupportedOperationException("Filters do not have queues"); } @@ -1134,6 +1147,51 @@ protected void onMessageFailed(M m, boolean shouldAttemptWorkerRetry) { } } + /** Applies the production failure policy while replacing its runtime queue scheduling boundary. */ + protected CompletionStage coordinateFailure(M message, boolean workerRetry, + BiFunction> retry) { + if (coordinatedRetryScheduling.get() != null) { + throw new IllegalStateException("Nested failure policy capture is not supported"); + } + var scheduling = new RetryScheduling(retry); + coordinatedRetryScheduling.set(scheduling); + try { + onMessageFailed(message, workerRetry); + } finally { + coordinatedRetryScheduling.remove(); + } + scheduling.dispatches.forEach(Runnable::run); + return CompletableFuture.allOf(scheduling.operations.toArray(CompletableFuture[]::new)); + } + + private boolean scheduleCoordinatedRetry(M message, boolean router, long delay) { + RetryScheduling scheduling = coordinatedRetryScheduling.get(); + if (scheduling == null) { + return false; + } + var result = new CompletableFuture(); + scheduling.operations.add(result); + Runnable submit = () -> { + try { + scheduling.retry.apply(message, router).whenComplete((ignored, failure) -> { + if (failure == null) { + result.complete(null); + } else { + result.completeExceptionally(failure); + } + }); + } catch (RuntimeException failure) { + result.completeExceptionally(failure); + } + }; + if (delay > 0) { + scheduling.dispatches.add(() -> CompletableFuture.delayedExecutor(delay, TimeUnit.MILLISECONDS).execute(submit)); + } else { + scheduling.dispatches.add(submit); + } + return true; + } + public void onMessageSuccess(M msg) throws IOException { try { failedMsgCounter.remove(msg.msgId); @@ -1164,6 +1222,11 @@ protected void handleMessageFailInWorker(String id, M m, boolean enqueueInstead) doFailDelayWorkerEndRetryPolicyAction(m); } } catch (InterruptedException ie) { + RetryScheduling scheduling = coordinatedRetryScheduling.get(); + if (scheduling != null) { + scheduling.operations.add(CompletableFuture.failedFuture(ie)); + Thread.currentThread().interrupt(); + } handleException(ie); } failedMsgCounter.remove(m.msgId); @@ -1329,6 +1392,16 @@ protected boolean customFailAction(M msg, FailDelayPolicy.Stage stage, int trial return true; } + private static final class RetryScheduling { + private final BiFunction> retry; + private final java.util.ArrayList> operations = new java.util.ArrayList<>(); + private final java.util.ArrayList dispatches = new java.util.ArrayList<>(); + + private RetryScheduling(BiFunction> retry) { + this.retry = retry; + } + } + public enum FilterLifecyclePhase { DURING_ROUTING, BEFORE_PROCESSING, AFTER_SUCCESS, AFTER_FAILURE, BEFORE_INSERT; } diff --git a/sendium-core/src/main/java/gr/cytech/sendium/core/smpp/client/CoordinatedSmppSubmission.java b/sendium-core/src/main/java/gr/cytech/sendium/core/smpp/client/CoordinatedSmppSubmission.java new file mode 100644 index 00000000..7f210ee4 --- /dev/null +++ b/sendium-core/src/main/java/gr/cytech/sendium/core/smpp/client/CoordinatedSmppSubmission.java @@ -0,0 +1,113 @@ +package gr.cytech.sendium.core.smpp.client; + +import gr.cytech.sendium.core.message.StandardMessage; + +import java.util.ArrayList; +import java.util.List; +import java.util.Objects; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionStage; +import java.util.function.Supplier; + +/** Tracks provider requests and their replacement attempts without admitting additional sources. */ +public final class CoordinatedSmppSubmission { + private final List> parts = new ArrayList<>(); + private CompletionStage completion; + + Part add(M message) { + if (completion != null) { + throw new IllegalStateException("Provider request registration is already sealed"); + } + Part part = new Part<>(message); + parts.add(part); + return part; + } + + CompletionStage seal() { + if (parts.isEmpty()) { + throw new IllegalArgumentException("Provider submission must contain a request"); + } + completion = CompletableFuture.allOf(parts.stream().map(part -> part.completed) + .toArray(CompletableFuture[]::new)).minimalCompletionStage(); + return completion; + } + + /** Retries failed tracking or retry handoffs, never a successfully completed provider request. */ + public void retryPendingHandoffs() { + parts.forEach(Part::retry); + } + + public CompletionStage completion() { + return Objects.requireNonNull(completion, "Submission has not been sealed"); + } + + public List handoffFailures() { + return parts.stream().map(Part::failure).filter(Objects::nonNull).toList(); + } + + static final class Part { + final M message; + private final CompletableFuture completed = new CompletableFuture<>(); + private Supplier> handoff; + private boolean responding; + private boolean active; + private boolean terminal; + private Throwable failure; + + Part(M message) { + this.message = Objects.requireNonNull(message, "message"); + } + + synchronized boolean claimResponse() { + if (responding) { + return false; + } + responding = true; + return true; + } + + CompletionStage completion() { + return completed.minimalCompletionStage(); + } + + void finish(Supplier> operation) { + synchronized (this) { + handoff = Objects.requireNonNull(operation, "operation"); + } + retry(); + } + + void retry() { + Supplier> operation; + synchronized (this) { + if (handoff == null || active || terminal) { + return; + } + active = true; + failure = null; + operation = handoff; + } + try { + Objects.requireNonNull(operation.get(), "handoff stage").whenComplete((ignored, error) -> { + synchronized (this) { + active = false; + failure = error; + terminal = error == null; + } + if (error == null) { + completed.complete(null); + } + }); + } catch (RuntimeException error) { + synchronized (this) { + active = false; + failure = error; + } + } + } + + synchronized Throwable failure() { + return failure; + } + } +} diff --git a/sendium-core/src/main/java/gr/cytech/sendium/core/smpp/client/SmppClientSessionHandler.java b/sendium-core/src/main/java/gr/cytech/sendium/core/smpp/client/SmppClientSessionHandler.java index 0f737e80..bcdd4110 100644 --- a/sendium-core/src/main/java/gr/cytech/sendium/core/smpp/client/SmppClientSessionHandler.java +++ b/sendium-core/src/main/java/gr/cytech/sendium/core/smpp/client/SmppClientSessionHandler.java @@ -152,6 +152,10 @@ public void firePduRequestExpired(PduRequest pduRequest) { //if I have saved a message object as a reference object //then just re-enqueue the msg if (pduRequest.getReferenceObject() != null) { + if (smppClientWorker.handleCoordinatedResponse(pduRequest.getReferenceObject(), + SmppConstants.STATUS_DELIVERYFAILURE, null)) { + return; + } StandardMessage msg = (StandardMessage) pduRequest.getReferenceObject(); smppClientWorker.handleResponse(this, SmppConstants.STATUS_DELIVERYFAILURE, null, msg); @@ -204,6 +208,11 @@ public void fireExpectedPduResponseReceived(PduAsyncResponse pduAsyncResponse) { switch (pduAsyncResponse.getRequest().getCommandId()) { case SmppConstants.CMD_ID_SUBMIT_SM: SubmitSm submit = (SubmitSm) pduAsyncResponse.getRequest(); + if (smppClientWorker.handleCoordinatedResponse(submit.getReferenceObject(), + pduAsyncResponse.getResponse().getCommandStatus(), + pduAsyncResponse.getResponse() instanceof SubmitSmResp response ? response.getMessageId() : null)) { + return; + } SubmitSmResp resp = (SubmitSmResp) pduAsyncResponse.getResponse(); int statusCode = resp.getCommandStatus(); StandardMessage msg = (StandardMessage) submit.getReferenceObject(); @@ -270,6 +279,11 @@ public void fireRecoverablePduException(RecoverablePduException e) { logger.warn("{}: received: {}", this, e); PartialPdu partialPdu = (PartialPdu) e.getPartialPdu(); + if (smppClientWorker.handleCoordinatedResponse(partialPdu.getReferenceObject(), + partialPdu.getCommandStatus() == SmppConstants.STATUS_OK ? SmppConstants.STATUS_UNKNOWNERR : + partialPdu.getCommandStatus(), null)) { + return; + } if (partialPdu.getReferenceObject() != null && partialPdu.getReferenceObject() instanceof StandardMessage) { StandardMessage msg = (StandardMessage) partialPdu.getReferenceObject(); int commandStatus = partialPdu.getCommandStatus(); diff --git a/sendium-core/src/main/java/gr/cytech/sendium/core/smpp/client/SmppClientWorker.java b/sendium-core/src/main/java/gr/cytech/sendium/core/smpp/client/SmppClientWorker.java index 2564b09e..7692c4fe 100644 --- a/sendium-core/src/main/java/gr/cytech/sendium/core/smpp/client/SmppClientWorker.java +++ b/sendium-core/src/main/java/gr/cytech/sendium/core/smpp/client/SmppClientWorker.java @@ -59,6 +59,8 @@ import java.util.Map; import java.util.Objects; import java.util.Set; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionStage; import java.util.concurrent.Executors; import java.util.concurrent.Future; import java.util.concurrent.ScheduledExecutorService; @@ -541,6 +543,102 @@ public M doMessage(int pThreadIndex, M pMsg) throws IOException { return null; } + /** + * Submits lifecycle-owned work explicitly. Retry callbacks must retain the parent's ownership + * and return a stage covering the replacement attempt, not merely its queue insertion. + */ + public CoordinatedSmppSubmission submitCoordinated(M message, CoordinatedRetry retry) + throws SmppInvalidArgumentException { + Objects.requireNonNull(retry, "retry"); + List requests = generateSubmitRequest(message); + var submission = new CoordinatedSmppSubmission(); + for (SubmitSm request : requests) { + M payload = (M) request.getReferenceObject(); + var part = submission.add(payload); + request.setReferenceObject(new CoordinatedResponse<>(part, retry)); + } + submission.seal(); + for (int index = 0; index < requests.size(); index++) { + SubmitSm request = requests.get(index); + try { + sendCoordinatedRequest(request); + } catch (Exception failure) { + var response = (CoordinatedResponse) request.getReferenceObject(); + if (response.part.claimResponse()) { + if (index == 0) { + response.part.finish(() -> coordinateFailure(message, true, (payload, router) -> + retry.retry(payload, router ? NackHandlePolicy.RETRY_ROUTER : NackHandlePolicy.RETRY_WORKER))); + for (int unsent = 1; unsent < requests.size(); unsent++) { + var replacement = (CoordinatedResponse) requests.get(unsent).getReferenceObject(); + replacement.part.claimResponse(); + replacement.part.finish(response.part::completion); + } + break; + } + response.part.finish(() -> retry.retry(response.part.message, NackHandlePolicy.RETRY_WORKER)); + } + } + } + return submission; + } + + protected void sendCoordinatedRequest(SubmitSm request) throws Exception { + getAvailableHandlerForSending().getSession().sendRequestPdu( + request, configurationProvider.getLongPrpt(_requestTout), false); + } + + boolean handleCoordinatedResponse(Object reference, int status, String responseMessageId) { + if (!(reference instanceof CoordinatedResponse)) { + return false; + } + var response = (CoordinatedResponse) reference; + if (!response.part.claimResponse()) { + return true; + } + M message = response.part.message; + NackHandlePolicy policy = status == SmppConstants.STATUS_OK ? NackHandlePolicy.FAIL : findPolicyForStatusCode(status); + if (status != SmppConstants.STATUS_OK && policy != NackHandlePolicy.FAIL) { + if (policy == NackHandlePolicy.RETRY_ROUTER && retryRouterRemoveHlr && message.cnetwork > 0) { + message.cnetwork = 0; + message.outgateway = ""; + } + response.part.finish(() -> coordinateFailure(message, policy == NackHandlePolicy.RETRY_WORKER, + (payload, router) -> response.retry.retry(payload, + router ? NackHandlePolicy.RETRY_ROUTER : NackHandlePolicy.RETRY_WORKER))); + } else { + response.part.finish(() -> coordinatedProviderHandoff(message, status, responseMessageId)); + } + return true; + } + + protected CompletionStage coordinatedProviderHandoff(M message, int status, String responseMessageId) { + if (message.msgId < 0) { + return CompletableFuture.completedStage(null); + } + String providerMessageId = getProviderMessageId(responseMessageId); + String hashed = getHashedMessageID(providerMessageId); + if (status == SmppConstants.STATUS_OK) { + if (message.dlrReturnMetadata != null && providerMessageId == null) { + throw new IllegalStateException("Provider acceptance has no usable correlation ID"); + } + message.extrid = providerMessageId; + message.field1 = hashed; + return messageTracker.handoffProviderAccepted(hashed, message, providerMessageId).thenCompose(ignored -> { + try { + onMessageSuccess(message); + return CompletableFuture.completedStage(null); + } catch (IOException failure) { + return CompletableFuture.failedStage(failure); + } + }); + } else { + String errorCode = respErrCodeMap == null ? String.valueOf(StandardMessage.DLR_ERR_SMS_FAILED) : + respErrCodeMap.getOrDefault(String.valueOf(status), String.valueOf(StandardMessage.DLR_ERR_SMS_FAILED)); + return messageTracker.handoffSubmissionFailure(message, providerMessageId, hashed, + String.valueOf(status), StandardMessage.DLR_STAT_FAILED, errorCode); + } + } + public boolean checkConnectivity() { if (connectionHealthCheck) { return this.verifyConnectivity(); @@ -1855,6 +1953,15 @@ public static HashMap extractTlvs(Map registeredT return extracted.isEmpty() ? null : extracted; } + @FunctionalInterface + public interface CoordinatedRetry { + CompletionStage retry(M message, NackHandlePolicy policy); + } + + private record CoordinatedResponse(CoordinatedSmppSubmission.Part part, + CoordinatedRetry retry) { + } + public enum MsgIdType { StringLiteral, SubmitRespHexDlrDec, diff --git a/sendium-core/src/main/java/gr/cytech/sendium/core/worker/StandardMessageTracker.java b/sendium-core/src/main/java/gr/cytech/sendium/core/worker/StandardMessageTracker.java index 55d124ed..03c01460 100644 --- a/sendium-core/src/main/java/gr/cytech/sendium/core/worker/StandardMessageTracker.java +++ b/sendium-core/src/main/java/gr/cytech/sendium/core/worker/StandardMessageTracker.java @@ -109,7 +109,19 @@ public void createAndEnqueueDLR(int mqid, String providerMessageId, String hashe public void createAndEnqueueSubmissionFailure(StandardMessage message, String providerMessageId, String hashedProviderMessageId, String body, int state, String errorCode, - HashMap tlvs) { + HashMap tlvs) { + recordSubmissionFailure(message, providerMessageId, body, state, errorCode, false); + } + + @Override + public java.util.concurrent.CompletionStage handoffSubmissionFailure(StandardMessage message, + String providerMessageId, String hashedProviderMessageId, String body, int state, String errorCode) { + recordSubmissionFailure(message, providerMessageId, body, state, errorCode, true); + return java.util.concurrent.CompletableFuture.completedStage(null); + } + + private void recordSubmissionFailure(StandardMessage message, String providerMessageId, String body, + int state, String errorCode, boolean strict) { if (!outWorker.getWorkerResources().isDlrPersistenceEnabled()) { return; } @@ -121,7 +133,7 @@ public void createAndEnqueueSubmissionFailure(StandardMessage message, String pr Optional rejected = outWorker.getWorkerResources().getDlrService() .recordProviderRejected(returnState.get(), providerName, providerMessageId, state, errorCode); rejected.filter(result -> result.getDeliveryChannel() == MessageState.DeliveryChannel.SMPP) - .ifPresent(result -> enqueueDlr(result, providerMessageId, body, state, errorCode)); + .ifPresent(result -> enqueueDlr(result, providerMessageId, body, state, errorCode, strict)); } private Optional toMessageState(StandardMessage message) { @@ -137,7 +149,12 @@ private Optional toMessageState(StandardMessage message) { } private void enqueueDlr(MessageState state, String providerMessageId, String body, - int dlrState, String errorCode) { + int dlrState, String errorCode) { + enqueueDlr(state, providerMessageId, body, dlrState, errorCode, false); + } + + private void enqueueDlr(MessageState state, String providerMessageId, String body, + int dlrState, String errorCode, boolean strict) { StandardMessage dlrMsg = new StandardMessage(); dlrMsg.serial = state.getGatewayMsgId(); dlrMsg.from = state.getDestAddr(); @@ -153,6 +170,10 @@ private void enqueueDlr(MessageState state, String providerMessageId, String bod try { outWorker.enqueueToRouter(dlrMsg); } catch (InterruptedException ie) { + if (strict) { + Thread.currentThread().interrupt(); + throw new DlrStorageException("Submission failure DLR handoff interrupted", ie); + } outWorker.handleException(ie); } if (MessageTrace.shouldLog(outWorker.getConfigurationProvider(), MessageTrace.EVENT_DLR)) { diff --git a/sendium-core/src/main/java/gr/cytech/sendium/core/worker/Tracker.java b/sendium-core/src/main/java/gr/cytech/sendium/core/worker/Tracker.java index fe4827ef..4690827c 100644 --- a/sendium-core/src/main/java/gr/cytech/sendium/core/worker/Tracker.java +++ b/sendium-core/src/main/java/gr/cytech/sendium/core/worker/Tracker.java @@ -3,6 +3,8 @@ import gr.cytech.sendium.core.message.StandardMessage; import java.util.HashMap; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionStage; public interface Tracker { @@ -38,4 +40,20 @@ default void createAndEnqueueSubmissionFailure(M message, String providerMessage } int getConfiguredMccMnc(); + + /** Asynchronous trackers can override this boundary to wait for the actual acceptance handoff. */ + default CompletionStage handoffProviderAccepted(String hashedProviderMessageId, M message, + String providerMessageId) { + updateSendStatusAndExtID(hashedProviderMessageId, message, providerMessageId); + return CompletableFuture.completedStage(null); + } + + /** Lifecycle handoff must report failures rather than treating a failed enqueue as success. */ + default CompletionStage handoffSubmissionFailure(M message, String providerMessageId, + String hashedProviderMessageId, String body, + int state, String errorCode) { + createAndEnqueueSubmissionFailure(message, providerMessageId, hashedProviderMessageId, + body, state, errorCode, null); + return CompletableFuture.completedStage(null); + } } diff --git a/sendium-core/src/main/java/gr/cytech/sendium/routing/StandardOutboundDispatch.java b/sendium-core/src/main/java/gr/cytech/sendium/routing/StandardOutboundDispatch.java new file mode 100644 index 00000000..17801689 --- /dev/null +++ b/sendium-core/src/main/java/gr/cytech/sendium/routing/StandardOutboundDispatch.java @@ -0,0 +1,139 @@ +package gr.cytech.sendium.routing; + +import gr.cytech.sendium.core.message.StandardMessage; +import gr.cytech.sendium.core.outbound.OutboundCoordinator; +import gr.cytech.sendium.core.outbound.OutboundWork.Destination; +import gr.cytech.sendium.core.outbound.OutboundWork.Routed; +import gr.cytech.sendium.core.outbound.OutboundWork.Selected; +import gr.cytech.sendium.core.smpp.client.CoordinatedSmppSubmission; +import gr.cytech.sendium.core.smpp.client.SmppClientWorker; +import gr.cytech.sendium.core.storage.OutboundStage; +import gr.cytech.sendium.core.storage.OutboundStorageException; + +import java.io.IOException; +import java.time.Duration; +import java.util.List; +import java.util.Objects; +import java.util.Optional; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionStage; +import java.util.concurrent.CopyOnWriteArrayList; + +/** Explicitly assembled routing and provider-completion boundary for the single-destination lifecycle. */ +public final class StandardOutboundDispatch { + private final OutboundCoordinator coordinator; + private final StandardRoutingManager routing; + + public StandardOutboundDispatch(OutboundCoordinator coordinator, StandardRoutingManager routing) { + this.coordinator = Objects.requireNonNull(coordinator, "coordinator"); + this.routing = Objects.requireNonNull(routing, "routing"); + } + + /** + * Routes one owned selection. A miss or lookup failure returns it to routing. If recording the + * destination fails, the caller retains the selection handle to retry the same routing intent. + */ + public Optional> routeSelected(Selected selected) throws IOException { + Objects.requireNonNull(selected, "selected"); + RoutingLookupResult result; + try { + result = routing.lookupForLifecycle(selected.message()); + } catch (IOException | RuntimeException failure) { + coordinator.returnToRouter(selected); + throw failure; + } + if (result.getDestinations().isEmpty()) { + coordinator.returnToRouter(selected); + return Optional.empty(); + } + if (result.getDestinations().size() != 1) { + coordinator.returnToRouter(selected); + throw new OutboundStorageException(OutboundStage.Role.ROUTER_QUEUE, + OutboundStorageException.Reason.UNSUPPORTED, "Lifecycle routing requires one destination"); + } + String destination = result.getDestinations().getFirst().getFullName(); + return Optional.of(coordinator.route(selected, new Destination<>(destination, selected.message()))); + } + + /** Takes recorded work for provider processing; taking or enqueuing it is not terminal completion. */ + public Optional> takeForProvider(String destination, Duration timeout) throws InterruptedException { + return coordinator.takeForDestination(destination, timeout); + } + + /** Runs an explicitly taken SMPP work item and carries its ownership through replacement attempts. */ + public ProviderExecution submitToProvider(Routed work, SmppClientWorker worker) + throws Exception { + if (!work.destination().equals(worker.getFullName())) { + throw new IllegalArgumentException("Provider does not match the recorded destination"); + } + var execution = new ProviderExecution(); + execution.retryCompletion = () -> coordinator.complete(work.id(), execution.attempts); + CompletionStage attempts = submitAttempt(work.message(), worker, execution); + execution.attempts = attempts; + execution.completion = coordinator.complete(work.id(), attempts); + return execution; + } + + private CompletionStage submitAttempt(StandardMessage message, SmppClientWorker worker, + ProviderExecution execution) throws Exception { + var submission = worker.submitCoordinated(message, (retryMessage, policy) -> + CompletableFuture.supplyAsync(() -> { + try { + SmppClientWorker destination = worker; + if (policy == SmppClientWorker.NackHandlePolicy.RETRY_ROUTER) { + RoutingLookupResult result = routing.lookupForLifecycle(retryMessage); + if (result.getDestinations().size() != 1 || + !(result.getDestinations().getFirst() instanceof SmppClientWorker)) { + throw new OutboundStorageException(OutboundStage.Role.ROUTER_QUEUE, + OutboundStorageException.Reason.UNAVAILABLE, "No single SMPP destination for retry"); + } + destination = (SmppClientWorker) result.getDestinations().getFirst(); + } + return submitAttempt(retryMessage, destination, execution); + } catch (Exception failure) { + return CompletableFuture.failedStage(failure); + } + }).thenCompose(stage -> stage)); + execution.submissions.add(submission); + return submission.completion(); + } + + /** + * Called only once the provider has produced an outcome for every part. Each stage represents + * the required handoff for that part (an already successful stage if none is required). Pending + * handoffs retain ownership; failure retains the work for a completion-only retry with new stages. + */ + public CompletionStage finishProviderParts(Routed work, + List> requiredHandoffs) { + Objects.requireNonNull(work, "work"); + List> handoffs = List.copyOf(requiredHandoffs); + if (handoffs.isEmpty()) { + throw new IllegalArgumentException("A provider outcome must include at least one part"); + } + CompletableFuture[] parts = handoffs.stream().map(CompletionStage::toCompletableFuture) + .toArray(CompletableFuture[]::new); + return coordinator.complete(work.id(), CompletableFuture.allOf(parts)); + } + + /** Runtime attempt handles remain associated with the parent's recorded work, never new admissions. */ + public static final class ProviderExecution { + private final List> submissions = new CopyOnWriteArrayList<>(); + private CompletionStage completion; + private CompletionStage attempts; + private java.util.function.Supplier> retryCompletion; + + public CompletionStage completion() { + return completion; + } + + public void retryPendingHandoffs() { + List.copyOf(submissions).forEach(CoordinatedSmppSubmission::retryPendingHandoffs); + } + + /** Retries store cleanup after successful handoffs without sending the message again. */ + public CompletionStage retryTerminalCleanup() { + completion = retryCompletion.get(); + return completion; + } + } +} diff --git a/sendium-core/src/main/java/gr/cytech/sendium/routing/StandardRoutingManager.java b/sendium-core/src/main/java/gr/cytech/sendium/routing/StandardRoutingManager.java index 8e983c40..814741a5 100644 --- a/sendium-core/src/main/java/gr/cytech/sendium/routing/StandardRoutingManager.java +++ b/sendium-core/src/main/java/gr/cytech/sendium/routing/StandardRoutingManager.java @@ -5,6 +5,8 @@ import gr.cytech.sendium.core.AbstractOutWorker; import gr.cytech.sendium.core.message.StandardMessage; import gr.cytech.sendium.core.queue.InMemoryQueueProvider; +import gr.cytech.sendium.core.storage.OutboundStage; +import gr.cytech.sendium.core.storage.OutboundStorageException; import gr.cytech.sendium.external.filter.FilterException; import gr.cytech.sendium.util.MessageTrace; import gr.cytech.sendium.util.TimeUtils; @@ -195,8 +197,22 @@ protected void handleMessageUnexpectedFailure(StandardMessage msg, Throwable e) } } + /** Resolves a lifecycle destination without interpreting copied routing as a single assignment. */ + public RoutingLookupResult lookupForLifecycle(StandardMessage message) throws IOException { + if (targets == null) { + throw new OutboundStorageException(OutboundStage.Role.ROUTER_QUEUE, + OutboundStorageException.Reason.UNAVAILABLE, "Routing targets are not initialized"); + } + return lookupRoutingForMessage(message, targets.defaultTable, true); + } + @Override protected RoutingLookupResult lookupRoutingForMessage(StandardMessage pMsg, RoutingTable table) throws IOException { + return lookupRoutingForMessage(pMsg, table, false); + } + + private RoutingLookupResult lookupRoutingForMessage(StandardMessage pMsg, RoutingTable table, + boolean rejectCopied) throws IOException { if (pMsg == null || table == null || table.rules == null || table.rules.isEmpty()) { return RoutingLookupResult.EMPTY_RESULT; } @@ -210,10 +226,15 @@ protected RoutingLookupResult lookupRoutingForMessage(StandardMessage pMsg, Rout } continue; } + if (rejectCopied && route.isCopied()) { + throw new OutboundStorageException(OutboundStage.Role.ROUTER_QUEUE, + OutboundStorageException.Reason.UNSUPPORTED, "Copied routes are not supported by lifecycle dispatch"); + } if (debugRouting) { logger.info("Message: ({}) matches route: ({})", pMsg, route); } - result.mergeRoutingLookupResult(getRoutingLookupResultFromRouteForNormal(route, pMsg)); + result.mergeRoutingLookupResult(rejectCopied ? getRoutingLookupResultFromRouteForNormal(route, pMsg, true) : + getRoutingLookupResultFromRouteForNormal(route, pMsg)); if (result.hasReachedLast()) { if (pMsg.rtxCnt == 0 && !Strings.isNullOrEmpty(pMsg.nextTarget) && pMsg.nextTarget.equals(Strings.nullToEmpty(originalNextTarget))) { @@ -233,6 +254,11 @@ protected RoutingLookupResult lookupRoutingForMessage(StandardMessage pMsg, Rout public RoutingLookupResult getRoutingLookupResultFromRouteForNormal( RoutingRule route, StandardMessage pMsg) throws IOException { + return getRoutingLookupResultFromRouteForNormal(route, pMsg, false); + } + + private RoutingLookupResult getRoutingLookupResultFromRouteForNormal( + RoutingRule route, StandardMessage pMsg, boolean rejectCopied) throws IOException { final String targetName = route.getTarget(); final var targetTable = getTargets().getTable(targetName); final var targetWorker = targetTable != null ? null : getTargets().getWorker(targetName); @@ -258,7 +284,7 @@ public RoutingLookupResult getRoutingLookupResultFromRouteForNormal( return new RoutingLookupResult(Collections.singletonList(targetWorker), !route.isCopied()); } - return lookupRoutingForMessage(pMsg, targetTable); + return rejectCopied ? lookupRoutingForMessage(pMsg, targetTable, true) : lookupRoutingForMessage(pMsg, targetTable); } private void applyFilterToMessage(AbstractOutWorker filter, StandardMessage pMsg) throws IOException { diff --git a/sendium-core/src/test/java/gr/cytech/sendium/core/smpp/client/SmppClientWorkerTest.java b/sendium-core/src/test/java/gr/cytech/sendium/core/smpp/client/SmppClientWorkerTest.java index 8590767e..05628c4c 100644 --- a/sendium-core/src/test/java/gr/cytech/sendium/core/smpp/client/SmppClientWorkerTest.java +++ b/sendium-core/src/test/java/gr/cytech/sendium/core/smpp/client/SmppClientWorkerTest.java @@ -2,6 +2,7 @@ import com.cloudhopper.commons.charset.CharsetUtil; import com.cloudhopper.smpp.SmppConstants; +import com.cloudhopper.smpp.PduAsyncResponse; import com.cloudhopper.smpp.pdu.DeliverSm; import com.cloudhopper.smpp.pdu.PduResponse; import com.cloudhopper.smpp.pdu.SubmitSm; @@ -11,21 +12,233 @@ import gr.cytech.sendium.conf.SendiumConfigurationProvider; import gr.cytech.sendium.core.message.StandardMessage; import gr.cytech.sendium.core.queue.Queue; +import gr.cytech.sendium.core.outbound.OutboundWork.Destination; +import gr.cytech.sendium.core.outbound.OutboundWork.SourceId; import gr.cytech.sendium.core.worker.DlrStorageException; import gr.cytech.sendium.core.worker.ForwardMoService; import gr.cytech.sendium.core.worker.Tracker; import gr.cytech.sendium.external.WorkerResourceProvider; +import gr.cytech.sendium.routing.StandardOutboundDispatch; +import gr.cytech.sendium.routing.StandardRoutingManager; +import gr.cytech.sendium.routing.RoutingLookupResult; import org.junit.jupiter.api.Test; +import utils.OutboundIngressFixture; import java.util.HashMap; import java.util.Map; import java.util.Set; +import java.util.UUID; +import java.time.Duration; import java.util.concurrent.ScheduledThreadPoolExecutor; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionStage; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; class SmppClientWorkerTest { + @Test + void firstSendFailureAbortsRemainingSubmitsAndRetriesOriginalPayload() throws Exception { + var worker = new TestSmppClientWorker(new TestConfigurationProvider(), new Queue<>(), new CapturingTracker()); + worker.failSendNumber = 1; + var message = messageWithNetwork(); + message.body = "a".repeat(200); + var replacement = new CompletableFuture(); + var retries = new java.util.ArrayList(); + var submission = worker.submitCoordinated(message, (payload, policy) -> { + retries.add(payload); + return replacement; + }); + assertThat(worker.coordinatedRequests).hasSize(1); + assertThat(retries).containsExactly(message); + assertThat(submission.completion().toCompletableFuture()).isNotDone(); + replacement.complete(null); + submission.completion().toCompletableFuture().join(); + } + + @Test + void coordinatedAcceptanceWaitsForAsynchronousTrackerHandoff() throws Exception { + var accepted = new CompletableFuture(); + var tracker = new CapturingTracker() { + @Override + public CompletionStage handoffProviderAccepted(String hash, StandardMessage message, String providerId) { + return accepted; + } + }; + var worker = new TestSmppClientWorker(new TestConfigurationProvider(), new Queue<>(), tracker); + worker.realCoordinatedHandoff = true; + var message = messageWithNetwork(); + message.body = "hello"; + var submission = worker.submitCoordinated(message, (payload, policy) -> { + throw new AssertionError("No retry expected"); + }); + respond(handler(worker), worker.coordinatedRequests.getFirst(), SmppConstants.STATUS_OK); + assertThat(submission.completion().toCompletableFuture()).isNotDone(); + accepted.complete(null); + submission.completion().toCompletableFuture().join(); + } + + @Test + void coordinatedExpiryGoesThroughFailurePolicyWithoutLegacyQueueInsertion() throws Exception { + var routerQueue = new Queue(); + var worker = new TestSmppClientWorker(new TestConfigurationProvider(), routerQueue, new CapturingTracker()); + var message = messageWithNetwork(); + message.body = "hello"; + var retried = new java.util.ArrayList(); + var replacement = new CompletableFuture(); + var submission = worker.submitCoordinated(message, (payload, policy) -> { + retried.add(payload); + return replacement; + }); + handler(worker).firePduRequestExpired(worker.coordinatedRequests.getFirst()); + assertThat(retried).containsExactly(message); + assertThat(routerQueue.isEmpty()).isTrue(); + assertThat(submission.completion().toCompletableFuture()).isNotDone(); + replacement.complete(null); + submission.completion().toCompletableFuture().join(); + } + + @Test + void firstPartRouterRetryKeepsParentSourcesThroughOriginalAndReplacementParts() throws Exception { + try (var fixture = new OutboundIngressFixture(1)) { + var worker = new TestSmppClientWorker(new TestConfigurationProvider(Map.of( + "status.retry.router", Integer.toString(SmppConstants.STATUS_INVDSTADR))), new Queue<>(), new CapturingTracker()); + var message = messageWithNetwork(); + message.body = "a".repeat(200); + SourceId source = new SourceId(UUID.randomUUID()); + fixture.coordinator.admit(source, message); + fixture.coordinator.selectAndStage(1); + var selected = fixture.coordinator.takeForRouting(Duration.ZERO).orElseThrow(); + fixture.coordinator.route(selected, new Destination<>(worker.getFullName(), selected.message())); + var work = fixture.coordinator.takeForDestination(worker.getFullName(), Duration.ZERO).orElseThrow(); + var routing = mock(StandardRoutingManager.class); + when(routing.lookupForLifecycle(org.mockito.ArgumentMatchers.any())).thenReturn( + new RoutingLookupResult(java.util.List.of(worker), true)); + var execution = new StandardOutboundDispatch(fixture.coordinator, routing).submitToProvider(work, worker); + var session = handler(worker); + worker.expectedRequests = 4; + respond(session, worker.coordinatedRequests.getFirst(), SmppConstants.STATUS_INVDSTADR); + assertThat(worker.requestsReady.await(5, TimeUnit.SECONDS)).isTrue(); + assertThat(worker.coordinatedRequests).hasSize(4); + respond(session, worker.coordinatedRequests.get(2), SmppConstants.STATUS_OK); + respond(session, worker.coordinatedRequests.get(3), SmppConstants.STATUS_OK); + assertThat(fixture.pending.find(source)).isPresent(); + assertThat(execution.completion().toCompletableFuture()).isNotDone(); + respond(session, worker.coordinatedRequests.get(1), SmppConstants.STATUS_OK); + execution.completion().toCompletableFuture().get(5, TimeUnit.SECONDS); + assertThat(fixture.pending.find(source)).isEmpty(); + assertThat(worker.getRouterQueue().isEmpty()).isTrue(); + } + } + + @Test + void coordinatedSessionCallbacksCompleteRealLifecycleOnlyAfterEveryPart() throws Exception { + try (var fixture = new OutboundIngressFixture(1)) { + var worker = new TestSmppClientWorker(new TestConfigurationProvider(), new Queue<>(), new CapturingTracker()); + var message = messageWithNetwork(); + message.body = "a".repeat(200); + SourceId source = new SourceId(UUID.randomUUID()); + fixture.coordinator.admit(source, message); + fixture.coordinator.selectAndStage(1); + var selected = fixture.coordinator.takeForRouting(Duration.ZERO).orElseThrow(); + fixture.coordinator.route(selected, new Destination<>(worker.getFullName(), selected.message())); + var work = fixture.coordinator.takeForDestination(worker.getFullName(), Duration.ZERO).orElseThrow(); + var dispatch = new StandardOutboundDispatch(fixture.coordinator, mock(StandardRoutingManager.class)); + var execution = dispatch.submitToProvider(work, worker); + var session = handler(worker); + respond(session, worker.coordinatedRequests.getLast(), SmppConstants.STATUS_OK); + assertThat(fixture.pending.find(source)).isPresent(); + assertThat(execution.completion().toCompletableFuture()).isNotDone(); + respond(session, worker.coordinatedRequests.getFirst(), SmppConstants.STATUS_OK); + execution.completion().toCompletableFuture().join(); + assertThat(fixture.pending.find(source)).isEmpty(); + respond(session, worker.coordinatedRequests.getFirst(), SmppConstants.STATUS_OK); + assertThat(worker.coordinatedHandoffs).isEqualTo(2); + } + } + + private static void respond(SmppClientSessionHandler handler, SubmitSm request, int status) { + var response = request.createResponse(); + response.setCommandStatus(status); + response.setMessageId("provider-id"); + var async = mock(PduAsyncResponse.class); + when(async.getRequest()).thenReturn(request); + when(async.getResponse()).thenReturn(response); + handler.fireExpectedPduResponseReceived(async); + } + + @Test + void coordinatedMultipartWaitsForAllResponsesAndIgnoresDuplicateCallbacks() throws Exception { + var worker = new TestSmppClientWorker(new TestConfigurationProvider(), new Queue<>(), new CapturingTracker()); + var message = messageWithNetwork(); + message.body = "a".repeat(200); + var submission = worker.submitCoordinated(message, (payload, policy) -> { + throw new AssertionError("No retry expected"); + }); + assertThat(worker.coordinatedRequests).hasSize(2); + Object first = worker.coordinatedRequests.getFirst().getReferenceObject(); + Object last = worker.coordinatedRequests.getLast().getReferenceObject(); + worker.handleCoordinatedResponse(last, SmppConstants.STATUS_OK, "second"); + assertThat(submission.completion().toCompletableFuture()).isNotDone(); + worker.handleCoordinatedResponse(last, SmppConstants.STATUS_OK, "duplicate"); + worker.handleCoordinatedResponse(first, SmppConstants.STATUS_OK, "first"); + submission.completion().toCompletableFuture().join(); + assertThat(worker.coordinatedHandoffs).isEqualTo(2); + assertThat(worker.success).isEmpty(); + assertThat(worker.failures).isEmpty(); + } + + @Test + void coordinatedRouterRetryPreservesFirstWholeMessageAndLaterPartPayloads() throws Exception { + var worker = new TestSmppClientWorker(new TestConfigurationProvider(Map.of( + "status.retry.router", Integer.toString(SmppConstants.STATUS_INVDSTADR))), new Queue<>(), new CapturingTracker()); + var message = messageWithNetwork(); + message.body = "a".repeat(200); + var retries = new java.util.ArrayList(); + var replacement = new CompletableFuture(); + var submission = worker.submitCoordinated(message, (payload, policy) -> { + assertThat(policy).isEqualTo(SmppClientWorker.NackHandlePolicy.RETRY_ROUTER); + retries.add(payload); + return replacement; + }); + worker.coordinatedRequests.forEach(request -> worker.handleCoordinatedResponse( + request.getReferenceObject(), SmppConstants.STATUS_INVDSTADR, null)); + assertThat(retries).hasSize(2); + assertThat(retries.getFirst()).isSameAs(message); + assertThat(retries.getFirst().body).hasSize(200); + assertThat(retries.getLast()).isNotSameAs(message); + assertThat(retries.getLast().binheader).isNotBlank(); + assertThat(retries.getLast().body.length()).isLessThan(200); + assertThat(submission.completion().toCompletableFuture()).isNotDone(); + assertThat(worker.failures).isEmpty(); + replacement.complete(null); + submission.completion().toCompletableFuture().join(); + } + + @Test + void coordinatedTrackingFailureRetriesOnlyHandoffWithoutResubmittingProvider() throws Exception { + var worker = new TestSmppClientWorker(new TestConfigurationProvider(), new Queue<>(), new CapturingTracker()); + worker.failCoordinatedHandoff = true; + var message = messageWithNetwork(); + message.body = "hello"; + var submission = worker.submitCoordinated(message, (payload, policy) -> { + throw new AssertionError("No provider retry expected"); + }); + worker.handleCoordinatedResponse(worker.coordinatedRequests.getFirst().getReferenceObject(), + SmppConstants.STATUS_OK, "accepted"); + assertThat(submission.handoffFailures()).hasSize(1); + assertThat(submission.completion().toCompletableFuture()).isNotDone(); + worker.failCoordinatedHandoff = false; + submission.retryPendingHandoffs(); + submission.completion().toCompletableFuture().join(); + assertThat(worker.coordinatedRequests).hasSize(1); + assertThat(worker.coordinatedHandoffs).isEqualTo(2); + } + @Test void defaultsSensitiveDiagnosticLoggingOff() { TestConfigurationProvider config = new TestConfigurationProvider(); @@ -313,6 +526,34 @@ private static StandardMessage messageWithNetwork() { } private static class TestSmppClientWorker extends SmppClientWorker { + private final java.util.List coordinatedRequests = new java.util.concurrent.CopyOnWriteArrayList<>(); + private final CountDownLatch requestsReady = new CountDownLatch(1); + private volatile int expectedRequests = Integer.MAX_VALUE; + private int coordinatedHandoffs; + private boolean failCoordinatedHandoff; + private boolean realCoordinatedHandoff; + private int failSendNumber; + + @Override + protected void sendCoordinatedRequest(SubmitSm request) throws java.io.IOException { + coordinatedRequests.add(request); + if (coordinatedRequests.size() == failSendNumber) { + throw new java.io.IOException("submit unavailable"); + } + if (coordinatedRequests.size() >= expectedRequests) { + requestsReady.countDown(); + } + } + + @Override + protected CompletionStage coordinatedProviderHandoff(StandardMessage message, int status, String providerId) { + if (realCoordinatedHandoff) { + return super.coordinatedProviderHandoff(message, status, providerId); + } + coordinatedHandoffs++; + return failCoordinatedHandoff ? CompletableFuture.failedStage(new DlrStorageException("unavailable")) : + CompletableFuture.completedStage(null); + } private final java.util.List success = new java.util.ArrayList<>(); private final java.util.List temporaryFailures = new java.util.ArrayList<>(); private final java.util.List failures = new java.util.ArrayList<>(); diff --git a/sendium-core/src/test/java/gr/cytech/sendium/core/worker/StandardMessageTrackerTest.java b/sendium-core/src/test/java/gr/cytech/sendium/core/worker/StandardMessageTrackerTest.java index eab26d3c..d84b0a71 100644 --- a/sendium-core/src/test/java/gr/cytech/sendium/core/worker/StandardMessageTrackerTest.java +++ b/sendium-core/src/test/java/gr/cytech/sendium/core/worker/StandardMessageTrackerTest.java @@ -149,6 +149,25 @@ void submissionFailure_PersistsAndEnqueuesSmppDlr() throws InterruptedException assertEquals("22", captor.getValue().errcode); } + @Test + void coordinatedSubmissionFailureDoesNotTreatInterruptedDlrHandoffAsSuccess() throws InterruptedException { + var message = new StandardMessage(); + message.serial = "gw-123"; + message.dlrReturnMetadata = DlrReturnMetadata.smpp("account", "system", "from", "to"); + var rejected = new MessageState("gw-123", "account", "system", "from", "to", null); + rejected.setDeliveryChannel(MessageState.DeliveryChannel.SMPP); + when(dlrService.recordProviderRejected(any(), any(), any(), anyInt(), any())) + .thenReturn(java.util.Optional.of(rejected)); + doThrow(new InterruptedException("interrupted")).when(outWorker).enqueueToRouter(any()); + try { + assertThrows(DlrStorageException.class, () -> tracker.handoffSubmissionFailure( + message, "provider", "hash", "5", StandardMessage.DLR_STAT_FAILED, "22")); + assertTrue(Thread.currentThread().isInterrupted()); + } finally { + Thread.interrupted(); + } + } + @Test void submissionFailure_PersistsHttpDlrWithoutRouterEnqueue() throws InterruptedException { StandardMessage message = new StandardMessage(); diff --git a/sendium-core/src/test/java/gr/cytech/sendium/routing/StandardOutboundDispatchTest.java b/sendium-core/src/test/java/gr/cytech/sendium/routing/StandardOutboundDispatchTest.java new file mode 100644 index 00000000..95470802 --- /dev/null +++ b/sendium-core/src/test/java/gr/cytech/sendium/routing/StandardOutboundDispatchTest.java @@ -0,0 +1,156 @@ +package gr.cytech.sendium.routing; + +import gr.cytech.sendium.core.AbstractOutWorker; +import gr.cytech.sendium.core.message.StandardMessage; +import gr.cytech.sendium.core.outbound.OutboundWork.Routed; +import gr.cytech.sendium.core.outbound.OutboundWork.SourceId; +import gr.cytech.sendium.core.storage.OutboundStorageException; +import org.junit.jupiter.api.Test; +import utils.OutboundIngressFixture; + +import java.time.Duration; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.UUID; +import java.util.concurrent.CompletableFuture; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +class StandardOutboundDispatchTest { + @Test + void routesToOneRecordedDestinationAndRetainsSourcesThroughAllProviderPartHandoffs() throws Exception { + try (var fixture = new OutboundIngressFixture(2)) { + var worker = worker("smpp.provider", "provider"); + var routing = routing(List.of(worker), "provider::default:"); + var dispatch = new StandardOutboundDispatch(fixture.coordinator, routing); + SourceId source = new SourceId(UUID.randomUUID()); + fixture.coordinator.admit(source, message("hello")); + + var routed = routeNext(dispatch, fixture).orElseThrow(); + assertThat(routed.destination()).isEqualTo(worker.getFullName()); + assertThat(dispatch.takeForProvider("other", Duration.ZERO)).isEmpty(); + var taken = dispatch.takeForProvider(worker.getFullName(), Duration.ZERO).orElseThrow(); + assertThat(taken.id()).isEqualTo(routed.id()); + assertThat(fixture.pending.find(source)).isPresent(); + + var firstPartHandoff = new CompletableFuture(); + var secondPartHandoff = new CompletableFuture(); + var completed = dispatch.finishProviderParts(taken, List.of(firstPartHandoff, secondPartHandoff)); + firstPartHandoff.complete(null); + assertThat(completed.toCompletableFuture()).isNotDone(); + assertThat(fixture.pending.find(source)).isPresent(); + secondPartHandoff.complete(null); + completed.toCompletableFuture().join(); + assertThat(fixture.pending.find(source)).isEmpty(); + assertThat(dispatch.takeForProvider(worker.getFullName(), Duration.ZERO)).isEmpty(); + dispatch.finishProviderParts(taken, List.of(CompletableFuture.completedStage(null))) + .toCompletableFuture().join(); + } + } + + @Test + void failedPartHandoffRetainsWorkForCompletionOnlyRetryWithoutRedispatch() throws Exception { + try (var fixture = new OutboundIngressFixture(1)) { + var dispatch = new StandardOutboundDispatch(fixture.coordinator, + routing(List.of(worker("smpp.provider", "provider")), "provider::default:")); + SourceId source = new SourceId(UUID.randomUUID()); + fixture.coordinator.admit(source, message("hello")); + routeNext(dispatch, fixture).orElseThrow(); + var taken = dispatch.takeForProvider("smpp.provider", Duration.ZERO).orElseThrow(); + var handoff = new CompletableFuture(); + var completion = dispatch.finishProviderParts(taken, List.of(CompletableFuture.completedStage(null), handoff)); + handoff.completeExceptionally(new IllegalStateException("handoff unavailable")); + assertThatThrownBy(() -> completion.toCompletableFuture().join()).hasCauseInstanceOf(IllegalStateException.class); + assertThat(fixture.pending.find(source)).isPresent(); + assertThat(dispatch.takeForProvider("smpp.provider", Duration.ZERO)).isEmpty(); + + dispatch.finishProviderParts(taken, List.of(CompletableFuture.completedStage(null), + CompletableFuture.completedStage(null))).toCompletableFuture().join(); + assertThat(fixture.pending.find(source)).isEmpty(); + } + } + + @Test + void copiedRouteIsRejectedBeforeRecordingWorkEvenWhenItFindsOnlyOneDestination() throws Exception { + try (var fixture = new OutboundIngressFixture(1)) { + var provider = worker("smpp.provider", "provider"); + var routing = routing(List.of(provider), "+provider::default:"); + var dispatch = new StandardOutboundDispatch(fixture.coordinator, routing); + SourceId source = new SourceId(UUID.randomUUID()); + fixture.coordinator.admit(source, message("hello")); + + assertThat(routing.lookupRoutingForMessage(message("legacy"), routing.getTargets().defaultTable) + .getDestinations()).containsExactly(provider); + assertThatThrownBy(() -> routeNext(dispatch, fixture)) + .isInstanceOfSatisfying(OutboundStorageException.class, failure -> + assertThat(failure.reason()).isEqualTo(OutboundStorageException.Reason.UNSUPPORTED)); + assertThat(fixture.pending.find(source)).isPresent(); + assertThat(dispatch.takeForProvider("smpp.provider", Duration.ZERO)).isEmpty(); + + routing.parseNewRoutingTable(RoutingFileParser.parseRoutingTable(List.of("provider::default:")), List.of(provider)); + assertThat(routeNext(dispatch, fixture)).isPresent(); + assertThat(dispatch.takeForProvider("smpp.provider", Duration.ZERO)).isPresent(); + } + } + + @Test + void copiedRouteInsideNestedTableIsRejectedBeforeAssignment() throws Exception { + try (var fixture = new OutboundIngressFixture(1)) { + var provider = worker("smpp.provider", "provider"); + var routing = routing(List.of(provider), "second::default:", "[second]", "+provider::default:"); + var dispatch = new StandardOutboundDispatch(fixture.coordinator, routing); + fixture.coordinator.admit(new SourceId(UUID.randomUUID()), message("hello")); + assertThatThrownBy(() -> routeNext(dispatch, fixture)) + .isInstanceOfSatisfying(OutboundStorageException.class, failure -> + assertThat(failure.reason()).isEqualTo(OutboundStorageException.Reason.UNSUPPORTED)); + assertThat(dispatch.takeForProvider("smpp.provider", Duration.ZERO)).isEmpty(); + } + } + + @Test + void routingMissReturnsTheUnassignedSelectionForLaterRetry() throws Exception { + try (var fixture = new OutboundIngressFixture(1)) { + var provider = worker("smpp.provider", "provider"); + var routing = routing(List.of(provider), "unknown::default:"); + var dispatch = new StandardOutboundDispatch(fixture.coordinator, routing); + SourceId source = new SourceId(UUID.randomUUID()); + fixture.coordinator.admit(source, message("hello")); + assertThat(routeNext(dispatch, fixture)).isEmpty(); + assertThat(fixture.pending.find(source)).isPresent(); + routing.parseNewRoutingTable(RoutingFileParser.parseRoutingTable(List.of("provider::default:")), List.of(provider)); + assertThat(routeNext(dispatch, fixture)).isPresent(); + } + } + + private static StandardMessage message(String body) { + var message = new StandardMessage(); + message.body = body; + return message; + } + + private static Optional> routeNext(StandardOutboundDispatch dispatch, + OutboundIngressFixture fixture) throws Exception { + fixture.coordinator.selectAndStage(1); + return dispatch.routeSelected(fixture.coordinator.takeForRouting(Duration.ZERO).orElseThrow()); + } + + private static StandardRoutingManager routing(List> workers, String... rules) { + var routing = new StandardRoutingManager(); + Map tables = RoutingFileParser.parseRoutingTable(List.of(rules)); + routing.parseNewRoutingTable(tables, List.copyOf(workers)); + return routing; + } + + @SuppressWarnings("unchecked") + private static AbstractOutWorker worker(String fullName, String instanceName) { + AbstractOutWorker worker = mock(AbstractOutWorker.class); + when(worker.getFullName()).thenReturn(fullName); + when(worker.getInstanceName()).thenReturn(instanceName); + when(worker.acceptsMessages()).thenReturn(true); + return worker; + } +} From 7a2612f029ac8ee241f9eb3a64b57629ce24ec9c Mon Sep 17 00:00:00 2001 From: pavlos Date: Thu, 1 Oct 2026 17:36:59 +0300 Subject: [PATCH 08/20] feat(storage): schedule parent-owned provider retries Signed-off-by: pavlos --- docs/14-outbound-storage.md | 30 +- .../sendium/core/AbstractOutWorker.java | 25 +- .../core/smpp/client/SmppClientWorker.java | 46 ++++ .../routing/StandardOutboundDispatch.java | 203 ++++++++++++-- .../smpp/client/SmppClientWorkerTest.java | 257 ++++++++++++++++++ 5 files changed, 523 insertions(+), 38 deletions(-) diff --git a/docs/14-outbound-storage.md b/docs/14-outbound-storage.md index 00bef252..e86f3b8f 100644 --- a/docs/14-outbound-storage.md +++ b/docs/14-outbound-storage.md @@ -74,8 +74,10 @@ callbacks do not release another source. A successful stage represents a part re ### Coordinated SMPP provider processing -`submitToProvider` connects an explicitly taken work item to `SmppClientWorker.submitCoordinated`. -It registers every request before sending and uses typed request references for provider callbacks. +`submitToProvider` schedules an explicitly taken work item through +`SmppClientWorker.submitPreparedCoordinated`, which applies rate limiting, character mapping and +before-processing filters before `submitCoordinated` generates provider requests. It registers every +request before sending and uses typed request references for provider callbacks. The session handler recognizes these references for submit responses, request expiry and recoverable PDU errors before applying its legacy message casts. Ordinary message references retain their existing behavior. @@ -99,9 +101,29 @@ same-worker retries reuse that worker. The scope is removed before replacement w not change scheduling for ordinary legacy messages. Parent sources complete only once the original outstanding requests, their replacement attempts, and required tracking handoffs all finish. +The parent's recorded destination is its **initial assignment**, not an assertion that every later +attempt uses that provider. `ProviderExecution.attempts()` exposes ordered runtime attempt snapshots +with their resolved destination, state and preparation/routing failure. Each attempt is scheduled +explicitly, including same-worker replacements. Router retries resolve their own destination without +overwriting the parent record or re-admitting its sources. This lets a replacement finish at provider B +while an original multipart request remains outstanding at provider A. + +Paused or disconnected workers leave attempts queued; stopped workers and failed routing/preparation +leave attempts failed but unfinished. `retryPendingHandoffs()` also reschedules those failed attempts, +without re-running successfully submitted attempts. Provider tracking failures remain inside the +submission's part-handoff bookkeeping. Concurrent preparation/submission is limited by the worker's +configured thread count when its scheduling slots are first established; required tracking completion +does not hold a sending slot. Before-processing filter drops finish without sending; retry/re-enqueue +outcomes use the existing end-retry policy, including the same-worker fallback when no router exists. + +The default dispatcher owns a two-thread scheduled executor. Assembly may instead supply a +`ScheduledExecutorService`; the dispatcher does not close that application-owned executor. +Close an owned dispatcher only after its provider executions drain. This executor lifecycle boundary +does not itself quiesce ingress, return stage ownership, or drain outstanding provider requests. + This provider boundary is explicitly assembled and is not automatically wired into the standalone -router/worker loops. Worker-loop preparation, application lifecycle activation and coordinated shutdown -remain subsequent integration work. The new runtime attempt bookkeeping is non-durable; it does not +router/worker loops. Application lifecycle activation and coordinated shutdown remain subsequent +integration work. Runtime attempt bookkeeping is non-durable; it does not provide provider-outcome checkpoints or restart recovery. ## Components diff --git a/sendium-core/src/main/java/gr/cytech/sendium/core/AbstractOutWorker.java b/sendium-core/src/main/java/gr/cytech/sendium/core/AbstractOutWorker.java index fd64dbab..6472b243 100644 --- a/sendium-core/src/main/java/gr/cytech/sendium/core/AbstractOutWorker.java +++ b/sendium-core/src/main/java/gr/cytech/sendium/core/AbstractOutWorker.java @@ -1150,13 +1150,24 @@ protected void onMessageFailed(M m, boolean shouldAttemptWorkerRetry) { /** Applies the production failure policy while replacing its runtime queue scheduling boundary. */ protected CompletionStage coordinateFailure(M message, boolean workerRetry, BiFunction> retry) { + return captureRetryScheduling(() -> onMessageFailed(message, workerRetry), retry); + } + + /** Preserves the worker loop's before-processing filter rejection policy. */ + protected CompletionStage coordinatePreparationFailure(M message, boolean enqueueInstead, + BiFunction> retry) { + return captureRetryScheduling(() -> handleMessageFailInWorker("coordinated", message, enqueueInstead), retry); + } + + private CompletionStage captureRetryScheduling(Runnable failurePolicy, + BiFunction> retry) { if (coordinatedRetryScheduling.get() != null) { throw new IllegalStateException("Nested failure policy capture is not supported"); } var scheduling = new RetryScheduling(retry); coordinatedRetryScheduling.set(scheduling); try { - onMessageFailed(message, workerRetry); + failurePolicy.run(); } finally { coordinatedRetryScheduling.remove(); } @@ -1211,7 +1222,7 @@ protected void handleMessageFailInWorker(String id, M m, boolean enqueueInstead) if (enqueueInstead) { if (routerQueue == null) { logger.warn("Worker {} does not have a reference to router", getFullName()); - msgQ.enqueue(m); + enqueueFallback(m); } else { enqueueToRouter(m); } @@ -1233,6 +1244,12 @@ protected void handleMessageFailInWorker(String id, M m, boolean enqueueInstead) } } + private void enqueueFallback(M message) throws InterruptedException { + if (!scheduleCoordinatedRetry(message, false, 0)) { + msgQ.enqueue(message); + } + } + protected M doFailDelayWorkerRetryPolicyAction(M m, int tries) { //Use a policy on what to do, allowing other workers to override it and make it better fit their needs FailDelayPolicyAction action = failDelayPolicy.getActionForMessage(m, FailDelayPolicy.Stage.WORKER_RETRY, tries); @@ -1293,7 +1310,7 @@ protected void doFailDelayWorkerEndRetryPolicyAction(M m) throws InterruptedExce //Default action is to enqueue to router, otherwise it will be just dropped if (routerQueue == null) { logger.warn("Worker {} does not have a reference to router", getFullName()); - msgQ.enqueue(m); + enqueueFallback(m); } else { enqueueToRouter(m); } @@ -1309,7 +1326,7 @@ protected void doFailDelayWorkerEndRetryPolicyAction(M m) throws InterruptedExce case RE_ENQUEUE_ROUTER: if (routerQueue == null) { logger.warn("Worker {} does not have a reference to router", getFullName()); - msgQ.enqueue(m); + enqueueFallback(m); } else { enqueueToRouter(m); } diff --git a/sendium-core/src/main/java/gr/cytech/sendium/core/smpp/client/SmppClientWorker.java b/sendium-core/src/main/java/gr/cytech/sendium/core/smpp/client/SmppClientWorker.java index 7692c4fe..0031f79a 100644 --- a/sendium-core/src/main/java/gr/cytech/sendium/core/smpp/client/SmppClientWorker.java +++ b/sendium-core/src/main/java/gr/cytech/sendium/core/smpp/client/SmppClientWorker.java @@ -39,6 +39,8 @@ import gr.cytech.sendium.core.worker.WorkerType; import gr.cytech.sendium.external.HealthCheckReport; import gr.cytech.sendium.external.WorkerResourceProvider; +import gr.cytech.sendium.external.filter.FilterException; +import gr.cytech.sendium.external.filter.FilterStatusCodes; import gr.cytech.sendium.util.MessageFlexValue; import gr.cytech.sendium.util.MessageTrace; import gr.cytech.sendium.util.SecurityUtils; @@ -543,6 +545,50 @@ public M doMessage(int pThreadIndex, M pMsg) throws IOException { return null; } + /** Applies worker preparation before an initial or replacement provider attempt. */ + public CoordinatedSmppSubmission submitPreparedCoordinated(M message, CoordinatedRetry retry) + throws Exception { + if (!keepOnRunning || isPause()) { + throw new IllegalStateException("Worker is stopped or paused: " + getFullName()); + } + applyRateLimit(); + if (!keepOnRunning || isPause()) { + throw new IllegalStateException("Worker stopped or paused while awaiting rate limit: " + getFullName()); + } + String stat = stats.checkGetStats(); + if (stat != null) { + logger.info("{}", stat); + } + message.outgateway = getFullName(); + String originalBody = doCharMap(message); + try { + if (printMsgs) { + logger.info("{}", message); + } + checkBeforeDoMessageFilters(message); + return submitCoordinated(message, retry); + } catch (FilterException failure) { + message.body = originalBody; + var submission = new CoordinatedSmppSubmission(); + var part = submission.add(message); + submission.seal(); + part.claimResponse(); + if (failure.getStatusCode() == FilterStatusCodes.DROP) { + part.finish(() -> CompletableFuture.completedStage(null)); + } else { + M replacement = (M) Objects.requireNonNull(failure.getMessageObj(), "filter replacement"); + replacement.body = originalBody; + boolean enqueueInstead = failure.getStatusCode() != FilterStatusCodes.RETRY; + part.finish(() -> coordinatePreparationFailure(replacement, enqueueInstead, (payload, router) -> + retry.retry(payload, router ? NackHandlePolicy.RETRY_ROUTER : NackHandlePolicy.RETRY_WORKER))); + } + return submission; + } catch (Exception failure) { + message.body = originalBody; + throw failure; + } + } + /** * Submits lifecycle-owned work explicitly. Retry callbacks must retain the parent's ownership * and return a stage covering the replacement attempt, not merely its queue insertion. diff --git a/sendium-core/src/main/java/gr/cytech/sendium/routing/StandardOutboundDispatch.java b/sendium-core/src/main/java/gr/cytech/sendium/routing/StandardOutboundDispatch.java index 17801689..75ba137a 100644 --- a/sendium-core/src/main/java/gr/cytech/sendium/routing/StandardOutboundDispatch.java +++ b/sendium-core/src/main/java/gr/cytech/sendium/routing/StandardOutboundDispatch.java @@ -17,16 +17,43 @@ import java.util.Optional; import java.util.concurrent.CompletableFuture; import java.util.concurrent.CompletionStage; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; import java.util.concurrent.CopyOnWriteArrayList; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.Semaphore; +import java.util.concurrent.TimeUnit; +import java.util.function.Supplier; /** Explicitly assembled routing and provider-completion boundary for the single-destination lifecycle. */ -public final class StandardOutboundDispatch { +public final class StandardOutboundDispatch implements AutoCloseable { private final OutboundCoordinator coordinator; private final StandardRoutingManager routing; + private final ScheduledExecutorService scheduler; + private final boolean ownsScheduler; + private final ConcurrentMap, Semaphore> workerSlots = new ConcurrentHashMap<>(); public StandardOutboundDispatch(OutboundCoordinator coordinator, StandardRoutingManager routing) { + this(coordinator, routing, Executors.newScheduledThreadPool(2, task -> { + var thread = new Thread(task, "outbound-provider-attempt"); + thread.setDaemon(true); + return thread; + }), true); + } + + /** Applications may supply their execution scheduler without transferring its lifecycle ownership. */ + public StandardOutboundDispatch(OutboundCoordinator coordinator, StandardRoutingManager routing, + ScheduledExecutorService scheduler) { + this(coordinator, routing, scheduler, false); + } + + private StandardOutboundDispatch(OutboundCoordinator coordinator, StandardRoutingManager routing, + ScheduledExecutorService scheduler, boolean ownsScheduler) { this.coordinator = Objects.requireNonNull(coordinator, "coordinator"); this.routing = Objects.requireNonNull(routing, "routing"); + this.scheduler = Objects.requireNonNull(scheduler, "scheduler"); + this.ownsScheduler = ownsScheduler; } /** @@ -68,43 +95,46 @@ public ProviderExecution submitToProvider(Routed work, SmppClie } var execution = new ProviderExecution(); execution.retryCompletion = () -> coordinator.complete(work.id(), execution.attempts); - CompletionStage attempts = submitAttempt(work.message(), worker, execution); + CompletionStage attempts = scheduleAttempt(work.message(), () -> worker, execution); execution.attempts = attempts; execution.completion = coordinator.complete(work.id(), attempts); return execution; } - private CompletionStage submitAttempt(StandardMessage message, SmppClientWorker worker, - ProviderExecution execution) throws Exception { - var submission = worker.submitCoordinated(message, (retryMessage, policy) -> - CompletableFuture.supplyAsync(() -> { - try { - SmppClientWorker destination = worker; - if (policy == SmppClientWorker.NackHandlePolicy.RETRY_ROUTER) { - RoutingLookupResult result = routing.lookupForLifecycle(retryMessage); - if (result.getDestinations().size() != 1 || - !(result.getDestinations().getFirst() instanceof SmppClientWorker)) { - throw new OutboundStorageException(OutboundStage.Role.ROUTER_QUEUE, - OutboundStorageException.Reason.UNAVAILABLE, "No single SMPP destination for retry"); - } - destination = (SmppClientWorker) result.getDestinations().getFirst(); - } - return submitAttempt(retryMessage, destination, execution); - } catch (Exception failure) { - return CompletableFuture.failedStage(failure); - } - }).thenCompose(stage -> stage)); - execution.submissions.add(submission); - return submission.completion(); + private CompletionStage scheduleAttempt(StandardMessage message, + Supplier> destination, + ProviderExecution execution) { + var attempt = new ProviderAttempt(message, destination, execution); + execution.providerAttempts.add(attempt); + attempt.schedule(0); + return attempt.completed.minimalCompletionStage(); } - /** - * Called only once the provider has produced an outcome for every part. Each stage represents - * the required handoff for that part (an already successful stage if none is required). Pending - * handoffs retain ownership; failure retains the work for a completion-only retry with new stages. - */ + private SmppClientWorker retryDestination(StandardMessage message) { + try { + RoutingLookupResult result = routing.lookupForLifecycle(message); + if (result.getDestinations().size() != 1 || + !(result.getDestinations().getFirst() instanceof SmppClientWorker)) { + throw new OutboundStorageException(OutboundStage.Role.ROUTER_QUEUE, + OutboundStorageException.Reason.UNAVAILABLE, "No single SMPP destination for retry"); + } + return (SmppClientWorker) result.getDestinations().getFirst(); + } catch (IOException failure) { + throw new java.util.concurrent.CompletionException(failure); + } + } + + /** Call after provider executions have drained; this does not complete or close parent stores. */ + @Override + public void close() { + if (ownsScheduler) { + scheduler.shutdown(); + } + } + + /** Completes explicitly supplied provider outcomes once all required part handoffs succeed. */ public CompletionStage finishProviderParts(Routed work, - List> requiredHandoffs) { + List> requiredHandoffs) { Objects.requireNonNull(work, "work"); List> handoffs = List.copyOf(requiredHandoffs); if (handoffs.isEmpty()) { @@ -115,9 +145,116 @@ public CompletionStage finishProviderParts(Routed work, return coordinator.complete(work.id(), CompletableFuture.allOf(parts)); } + public enum AttemptState { + QUEUED, PREPARING, AWAITING_COMPLETION, COMPLETED, FAILED + } + + public record AttemptSnapshot(String destination, AttemptState state, Throwable failure) { + } + + private final class ProviderAttempt implements Runnable { + private final StandardMessage message; + private final Supplier> resolveDestination; + private final ProviderExecution execution; + private final CompletableFuture completed = new CompletableFuture<>(); + private volatile SmppClientWorker worker; + private AttemptState state = AttemptState.QUEUED; + private Throwable failure; + + private ProviderAttempt(StandardMessage message, Supplier> destination, + ProviderExecution execution) { + this.message = message; + this.resolveDestination = destination; + this.execution = execution; + } + + private synchronized AttemptSnapshot snapshot() { + return new AttemptSnapshot(worker == null ? null : worker.getFullName(), state, failure); + } + + private synchronized void schedule(long delay) { + try { + scheduler.schedule(this, delay, TimeUnit.MILLISECONDS); + } catch (RuntimeException error) { + state = AttemptState.FAILED; + failure = error; + } + } + + private synchronized void retry() { + if (state == AttemptState.FAILED) { + state = AttemptState.QUEUED; + failure = null; + schedule(0); + } + } + + @Override + public void run() { + synchronized (this) { + if (state != AttemptState.QUEUED) { + return; + } + state = AttemptState.PREPARING; + } + Semaphore slot = null; + try { + if (worker == null) { + worker = Objects.requireNonNull(resolveDestination.get(), "retry destination"); + } + if (!worker.isKeepOnRunning()) { + throw new IllegalStateException("Destination worker stopped: " + worker.getFullName()); + } + if (worker.isPause()) { + synchronized (this) { + state = AttemptState.QUEUED; + schedule(100); + } + return; + } + slot = workerSlots.computeIfAbsent(worker, destination -> + new Semaphore(Math.max(1, destination.getThreadCount()))); + if (!slot.tryAcquire()) { + slot = null; + synchronized (this) { + state = AttemptState.QUEUED; + schedule(100); + } + return; + } + var submission = worker.submitPreparedCoordinated(message, (payload, policy) -> + scheduleAttempt(payload, policy == SmppClientWorker.NackHandlePolicy.RETRY_ROUTER ? + () -> retryDestination(payload) : () -> worker, execution)); + execution.submissions.add(submission); + synchronized (this) { + state = AttemptState.AWAITING_COMPLETION; + } + submission.completion().whenComplete((ignored, error) -> { + synchronized (this) { + state = error == null ? AttemptState.COMPLETED : AttemptState.FAILED; + failure = error; + } + if (error == null) { + completed.complete(null); + } + }); + } catch (Exception error) { + synchronized (this) { + state = AttemptState.FAILED; + failure = error; + } + } finally { + if (slot != null) { + slot.release(); + } + } + } + } + /** Runtime attempt handles remain associated with the parent's recorded work, never new admissions. */ public static final class ProviderExecution { private final List> submissions = new CopyOnWriteArrayList<>(); + private final List providerAttempts = new CopyOnWriteArrayList<>(); private CompletionStage completion; private CompletionStage attempts; private java.util.function.Supplier> retryCompletion; @@ -128,6 +265,12 @@ public CompletionStage completion() { public void retryPendingHandoffs() { List.copyOf(submissions).forEach(CoordinatedSmppSubmission::retryPendingHandoffs); + List.copyOf(providerAttempts).forEach(ProviderAttempt::retry); + } + + /** Ordered runtime attempts; the parent's recorded destination remains its initial assignment. */ + public List attempts() { + return providerAttempts.stream().map(ProviderAttempt::snapshot).toList(); } /** Retries store cleanup after successful handoffs without sending the message again. */ diff --git a/sendium-core/src/test/java/gr/cytech/sendium/core/smpp/client/SmppClientWorkerTest.java b/sendium-core/src/test/java/gr/cytech/sendium/core/smpp/client/SmppClientWorkerTest.java index 05628c4c..f2ac5d2f 100644 --- a/sendium-core/src/test/java/gr/cytech/sendium/core/smpp/client/SmppClientWorkerTest.java +++ b/sendium-core/src/test/java/gr/cytech/sendium/core/smpp/client/SmppClientWorkerTest.java @@ -37,10 +37,211 @@ import static org.assertj.core.api.Assertions.assertThat; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.spy; import static org.mockito.Mockito.when; class SmppClientWorkerTest { + @Test + void workerSlotAndRateGatePreventConcurrentPreparationAndEarlySending() throws Exception { + var scheduler = new ScheduledThreadPoolExecutor(2); + var rateGate = new CompletableFuture(); + try (var fixture = new OutboundIngressFixture(2); + var dispatch = new StandardOutboundDispatch(fixture.coordinator, mock(StandardRoutingManager.class), scheduler)) { + var worker = new TestSmppClientWorker(new TestConfigurationProvider(), new Queue<>(), new CapturingTracker()); + worker.rateGate = rateGate; + var executions = new java.util.ArrayList(); + for (int index = 0; index < 2; index++) { + var message = messageWithNetwork(); + message.body = "hello"; + fixture.coordinator.admit(new SourceId(UUID.randomUUID()), message); + } + fixture.coordinator.selectAndStage(2); + for (int index = 0; index < 2; index++) { + var selected = fixture.coordinator.takeForRouting(Duration.ZERO).orElseThrow(); + fixture.coordinator.route(selected, new Destination<>(worker.getFullName(), selected.message())); + executions.add(dispatch.submitToProvider( + fixture.coordinator.takeForDestination(worker.getFullName(), Duration.ZERO).orElseThrow(), worker)); + } + assertThat(worker.rateEntered.await(5, TimeUnit.SECONDS)).isTrue(); + scheduler.submit(() -> { }).get(5, TimeUnit.SECONDS); + assertThat(worker.rateChecks).isEqualTo(1); + assertThat(worker.preparations).isZero(); + assertThat(worker.coordinatedRequests).isEmpty(); + rateGate.complete(null); + worker.awaitRequests(2); + worker.coordinatedRequests.forEach(request -> respond(handler(worker), request, SmppConstants.STATUS_OK)); + for (var execution : executions) { + execution.completion().toCompletableFuture().get(5, TimeUnit.SECONDS); + } + assertThat(worker.rateChecks).isEqualTo(2); + assertThat(worker.preparations).isEqualTo(2); + } finally { + rateGate.complete(null); + scheduler.shutdownNow(); + } + } + + @Test + void failedRetryLookupCanResumeWithoutLosingParentOrRepeatingOriginalSubmission() throws Exception { + var scheduler = new ScheduledThreadPoolExecutor(1); + try (var fixture = new OutboundIngressFixture(1)) { + var worker = new TestSmppClientWorker(new TestConfigurationProvider(Map.of( + "status.retry.router", Integer.toString(SmppConstants.STATUS_INVDSTADR))), new Queue<>(), new CapturingTracker()); + var routing = mock(StandardRoutingManager.class); + var lookupTried = new CountDownLatch(1); + when(routing.lookupForLifecycle(org.mockito.ArgumentMatchers.any())).thenAnswer(invocation -> { + lookupTried.countDown(); + throw new IllegalStateException("routing temporarily unavailable"); + }); + try (var dispatch = new StandardOutboundDispatch(fixture.coordinator, routing, scheduler)) { + var message = messageWithNetwork(); + message.body = "hello"; + SourceId source = new SourceId(UUID.randomUUID()); + fixture.coordinator.admit(source, message); + fixture.coordinator.selectAndStage(1); + var selected = fixture.coordinator.takeForRouting(Duration.ZERO).orElseThrow(); + fixture.coordinator.route(selected, new Destination<>(worker.getFullName(), selected.message())); + var work = fixture.coordinator.takeForDestination(worker.getFullName(), Duration.ZERO).orElseThrow(); + var execution = dispatch.submitToProvider(work, worker); + worker.awaitRequests(1); + respond(handler(worker), worker.coordinatedRequests.getFirst(), SmppConstants.STATUS_INVDSTADR); + assertThat(lookupTried.await(5, TimeUnit.SECONDS)).isTrue(); + scheduler.submit(() -> { }).get(5, TimeUnit.SECONDS); + assertThat(execution.attempts().getLast().state()).isEqualTo(StandardOutboundDispatch.AttemptState.FAILED); + assertThat(execution.attempts().getLast().failure()).isInstanceOf(IllegalStateException.class); + assertThat(execution.completion().toCompletableFuture()).isNotDone(); + assertThat(fixture.pending.find(source)).isPresent(); + org.mockito.Mockito.doReturn(new RoutingLookupResult(java.util.List.of(worker), true)) + .when(routing).lookupForLifecycle(org.mockito.ArgumentMatchers.any()); + execution.retryPendingHandoffs(); + worker.awaitRequests(2); + respond(handler(worker), worker.coordinatedRequests.getLast(), SmppConstants.STATUS_OK); + execution.completion().toCompletableFuture().get(5, TimeUnit.SECONDS); + assertThat(worker.coordinatedRequests).hasSize(2); + assertThat(fixture.pending.find(source)).isEmpty(); + } + assertThat(scheduler.isShutdown()).isFalse(); + } finally { + scheduler.shutdownNow(); + } + } + + @Test + void preparedSubmissionMapsCharactersBeforeProviderRequestGeneration() throws Exception { + var worker = new TestSmppClientWorker(new TestConfigurationProvider(), new Queue<>(), new CapturingTracker()); + worker.mapCharacters = true; + var message = messageWithNetwork(); + message.body = "aaa"; + var submission = worker.submitPreparedCoordinated(message, (payload, policy) -> { + throw new AssertionError("No retry expected"); + }); + assertThat(CharsetUtil.decode(worker.coordinatedRequests.getFirst().getShortMessage(), CharsetUtil.NAME_GSM)) + .isEqualTo("bbb"); + respond(handler(worker), worker.coordinatedRequests.getFirst(), SmppConstants.STATUS_OK); + submission.completion().toCompletableFuture().join(); + } + + @Test + void beforeProcessingDropCompletesWithoutSendingToProvider() throws Exception { + var worker = new TestSmppClientWorker(new TestConfigurationProvider(), new Queue<>(), new CapturingTracker()); + worker.preparationStatus = gr.cytech.sendium.external.filter.FilterStatusCodes.DROP; + var submission = worker.submitPreparedCoordinated(messageWithNetwork(), (payload, policy) -> { + throw new AssertionError("Dropped message must not retry"); + }); + submission.completion().toCompletableFuture().join(); + assertThat(worker.coordinatedRequests).isEmpty(); + assertThat(worker.coordinatedHandoffs).isZero(); + } + + @Test + void preparationReenqueueWithoutRouterRetainsOwnershipInSameWorkerReplacement() throws Exception { + var worker = new TestSmppClientWorker(new TestConfigurationProvider(), null, new CapturingTracker()); + worker.preparationStatus = gr.cytech.sendium.external.filter.FilterStatusCodes.REENQUEUE; + var replacement = new CompletableFuture(); + var retried = new java.util.ArrayList(); + var original = messageWithNetwork(); + var submission = worker.submitPreparedCoordinated(original, (payload, policy) -> { + assertThat(policy).isEqualTo(SmppClientWorker.NackHandlePolicy.RETRY_WORKER); + retried.add(payload); + return replacement; + }); + assertThat(retried).containsExactly(original); + assertThat(worker.coordinatedRequests).isEmpty(); + assertThat(submission.completion().toCompletableFuture()).isNotDone(); + replacement.complete(null); + submission.completion().toCompletableFuture().join(); + } + + @Test + void routerRetryTracksAnotherProviderWhileOriginalMultipartRequestIsOutstanding() throws Exception { + try (var fixture = new OutboundIngressFixture(1)) { + var first = new TestSmppClientWorker(new TestConfigurationProvider(Map.of( + "status.retry.router", Integer.toString(SmppConstants.STATUS_INVDSTADR))), new Queue<>(), new CapturingTracker()); + var second = spy(new TestSmppClientWorker(new TestConfigurationProvider(), new Queue<>(), new CapturingTracker())); + when(second.getFullName()).thenReturn("smppclient.second"); + var routing = mock(StandardRoutingManager.class); + when(routing.lookupForLifecycle(org.mockito.ArgumentMatchers.any())).thenReturn( + new RoutingLookupResult(java.util.List.of(second), true)); + try (var retryDispatch = new StandardOutboundDispatch(fixture.coordinator, routing)) { + SourceId source = new SourceId(UUID.randomUUID()); + var message = messageWithNetwork(); + message.body = "a".repeat(200); + fixture.coordinator.admit(source, message); + fixture.coordinator.selectAndStage(1); + var selected = fixture.coordinator.takeForRouting(Duration.ZERO).orElseThrow(); + fixture.coordinator.route(selected, new Destination<>(first.getFullName(), selected.message())); + var parent = fixture.coordinator.takeForDestination(first.getFullName(), Duration.ZERO).orElseThrow(); + var execution = retryDispatch.submitToProvider(parent, first); + first.awaitRequests(2); + respond(handler(first), first.coordinatedRequests.getFirst(), SmppConstants.STATUS_INVDSTADR); + second.awaitRequests(2); + assertThat(parent.destination()).isEqualTo(first.getFullName()); + assertThat(execution.attempts()).extracting(StandardOutboundDispatch.AttemptSnapshot::destination) + .containsExactly(first.getFullName(), second.getFullName()); + second.coordinatedRequests.forEach(request -> respond(handler(second), request, SmppConstants.STATUS_OK)); + assertThat(execution.completion().toCompletableFuture()).isNotDone(); + assertThat(fixture.pending.find(source)).isPresent(); + respond(handler(first), first.coordinatedRequests.getLast(), SmppConstants.STATUS_OK); + execution.completion().toCompletableFuture().get(5, TimeUnit.SECONDS); + assertThat(fixture.pending.find(source)).isEmpty(); + assertThat(first.preparations).isEqualTo(1); + assertThat(second.preparations).isEqualTo(1); + assertThat(first.rateChecks).isEqualTo(1); + assertThat(second.rateChecks).isEqualTo(1); + assertThat(first.getRouterQueue().isEmpty()).isTrue(); + assertThat(second.getRouterQueue().isEmpty()).isTrue(); + } + } + } + + @Test + void pausedAttemptWaitsWithoutSendingAndPreparesAfterResume() throws Exception { + try (var fixture = new OutboundIngressFixture(1); + var dispatch = new StandardOutboundDispatch(fixture.coordinator, mock(StandardRoutingManager.class))) { + var worker = new TestSmppClientWorker(new TestConfigurationProvider(), new Queue<>(), new CapturingTracker()); + worker.testPaused = true; + SourceId source = new SourceId(UUID.randomUUID()); + fixture.coordinator.admit(source, messageWithNetwork()); + fixture.coordinator.selectAndStage(1); + var selected = fixture.coordinator.takeForRouting(Duration.ZERO).orElseThrow(); + selected.message().body = "hello"; + fixture.coordinator.route(selected, new Destination<>(worker.getFullName(), selected.message())); + var work = fixture.coordinator.takeForDestination(worker.getFullName(), Duration.ZERO).orElseThrow(); + var execution = dispatch.submitToProvider(work, worker); + assertThat(worker.pauseChecked.await(5, TimeUnit.SECONDS)).isTrue(); + assertThat(worker.coordinatedRequests).isEmpty(); + assertThat(worker.preparations).isZero(); + assertThat(fixture.pending.find(source)).isPresent(); + worker.testPaused = false; + worker.awaitRequests(1); + respond(handler(worker), worker.coordinatedRequests.getFirst(), SmppConstants.STATUS_OK); + execution.completion().toCompletableFuture().get(5, TimeUnit.SECONDS); + assertThat(worker.preparations).isEqualTo(1); + assertThat(worker.rateChecks).isEqualTo(1); + } + } + @Test void firstSendFailureAbortsRemainingSubmitsAndRetriesOriginalPayload() throws Exception { var worker = new TestSmppClientWorker(new TestConfigurationProvider(), new Queue<>(), new CapturingTracker()); @@ -119,6 +320,7 @@ void firstPartRouterRetryKeepsParentSourcesThroughOriginalAndReplacementParts() when(routing.lookupForLifecycle(org.mockito.ArgumentMatchers.any())).thenReturn( new RoutingLookupResult(java.util.List.of(worker), true)); var execution = new StandardOutboundDispatch(fixture.coordinator, routing).submitToProvider(work, worker); + worker.awaitRequests(2); var session = handler(worker); worker.expectedRequests = 4; respond(session, worker.coordinatedRequests.getFirst(), SmppConstants.STATUS_INVDSTADR); @@ -149,6 +351,7 @@ void coordinatedSessionCallbacksCompleteRealLifecycleOnlyAfterEveryPart() throws var work = fixture.coordinator.takeForDestination(worker.getFullName(), Duration.ZERO).orElseThrow(); var dispatch = new StandardOutboundDispatch(fixture.coordinator, mock(StandardRoutingManager.class)); var execution = dispatch.submitToProvider(work, worker); + worker.awaitRequests(2); var session = handler(worker); respond(session, worker.coordinatedRequests.getLast(), SmppConstants.STATUS_OK); assertThat(fixture.pending.find(source)).isPresent(); @@ -526,6 +729,58 @@ private static StandardMessage messageWithNetwork() { } private static class TestSmppClientWorker extends SmppClientWorker { + private volatile boolean testPaused; + private volatile int preparations; + private volatile int rateChecks; + private CompletableFuture rateGate; + private final CountDownLatch rateEntered = new CountDownLatch(1); + private boolean mapCharacters; + private gr.cytech.sendium.external.filter.FilterStatusCodes preparationStatus; + private final CountDownLatch pauseChecked = new CountDownLatch(1); + + @Override + public boolean isPause() { + pauseChecked.countDown(); + return testPaused || super.isPause(); + } + + @Override + protected void applyRateLimit() { + rateChecks++; + if (rateGate != null) { + rateEntered.countDown(); + rateGate.join(); + } + super.applyRateLimit(); + } + + @Override + protected void checkBeforeDoMessageFilters(StandardMessage message) throws java.io.IOException { + preparations++; + if (preparationStatus != null) { + var status = preparationStatus; + preparationStatus = null; + throw new gr.cytech.sendium.external.filter.FilterException(null, status, message, "test filter"); + } + super.checkBeforeDoMessageFilters(message); + } + private final java.util.concurrent.Semaphore requestsSeen = new java.util.concurrent.Semaphore(0); + + private void awaitRequests(int count) throws InterruptedException { + while (coordinatedRequests.size() < count) { + assertThat(requestsSeen.tryAcquire(5, TimeUnit.SECONDS)).isTrue(); + } + } + + @Override + protected String charMap(String body) { + return mapCharacters ? body.replace('a', 'b') : body; + } + + @Override + public boolean verifyConnectivity() { + return true; + } private final java.util.List coordinatedRequests = new java.util.concurrent.CopyOnWriteArrayList<>(); private final CountDownLatch requestsReady = new CountDownLatch(1); private volatile int expectedRequests = Integer.MAX_VALUE; @@ -537,6 +792,7 @@ private static class TestSmppClientWorker extends SmppClientWorker coordinatedProviderHandoff(StandardMessage messa Tracker tracker) { super(configurationProvider, routerQueue, new ScheduledThreadPoolExecutor(1)); this.messageTracker = tracker; + this.suspendAuto = false; } TestSmppClientWorker(SendiumConfigurationProvider configurationProvider, From 2b1f679a43cbe5877209fa25e477b542e32d798d Mon Sep 17 00:00:00 2001 From: pavlos Date: Thu, 1 Oct 2026 17:37:20 +0300 Subject: [PATCH 09/20] test(storage): preserve startup logs under CI log levels Signed-off-by: pavlos --- .../cytech/sendium/app/storage/SmsStorageStartupSupport.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/sendium-app/src/test/java/gr/cytech/sendium/app/storage/SmsStorageStartupSupport.java b/sendium-app/src/test/java/gr/cytech/sendium/app/storage/SmsStorageStartupSupport.java index 5b567c7c..d8206150 100644 --- a/sendium-app/src/test/java/gr/cytech/sendium/app/storage/SmsStorageStartupSupport.java +++ b/sendium-app/src/test/java/gr/cytech/sendium/app/storage/SmsStorageStartupSupport.java @@ -23,6 +23,9 @@ static QuarkusExtensionTest application() { .overrideConfigKey("quarkus.arc.exclude-dependency.core.artifact-id", "sendium-core") .overrideConfigKey("sendium.dlr.persistence.enabled", "false") .overrideConfigKey("quarkus.http.test-port", "0") + // These assertions must remain valid when CI sets the root log level to ERROR. + .overrideConfigKey("quarkus.log.category.\"gr.cytech.sendium.app.storage.StandaloneSmsStorage\".level", "INFO") + .overrideConfigKey("quarkus.log.category.\"gr.cytech.sendium.app.storage.SmsAdmissionProbe\".level", "INFO") .setLogRecordPredicate(record -> StandaloneSmsStorage.class.getName().equals(record.getLoggerName()) || SmsAdmissionProbe.class.getName().equals(record.getLoggerName())); } From 67d514d2fb75ee047debb30005b82262771ef565 Mon Sep 17 00:00:00 2001 From: pavlos Date: Mon, 5 Oct 2026 13:43:58 +0300 Subject: [PATCH 10/20] feat(storage): drain outbound executions before graceful shutdown Signed-off-by: pavlos --- docs/14-outbound-storage.md | 25 +++++- .../routing/StandardOutboundDispatch.java | 83 +++++++++++++++++-- .../StandardOutgoingWorkerHandler.java | 10 ++- .../smpp/client/SmppClientWorkerTest.java | 5 ++ .../routing/StandardOutboundDispatchTest.java | 49 +++++++++++ .../routing/StandardWorkerShutdownTest.java | 46 ++++++++++ 6 files changed, 205 insertions(+), 13 deletions(-) create mode 100644 sendium-core/src/test/java/gr/cytech/sendium/routing/StandardWorkerShutdownTest.java diff --git a/docs/14-outbound-storage.md b/docs/14-outbound-storage.md index e86f3b8f..9b6af370 100644 --- a/docs/14-outbound-storage.md +++ b/docs/14-outbound-storage.md @@ -118,11 +118,30 @@ outcomes use the existing end-retry policy, including the same-worker fallback w The default dispatcher owns a two-thread scheduled executor. Assembly may instead supply a `ScheduledExecutorService`; the dispatcher does not close that application-owned executor. -Close an owned dispatcher only after its provider executions drain. This executor lifecycle boundary -does not itself quiesce ingress, return stage ownership, or drain outstanding provider requests. +Graceful shutdown uses the following application-owned ordering: + +1. Call dispatcher `quiesce()`. It quiesces the coordinator, stopping admission, selection and takes, + and prevents new provider submissions. An already-taken unassigned selection presented to the + dispatcher is returned to routing without recording a destination. +2. Keep destination workers, provider callbacks and the execution scheduler running. Call + `awaitProviderDrain(timeout)` to await active provider executions, including replacement attempts, + required handoffs and terminal cleanup. A timeout returns `false`; a failed completion throws an + actionable exception. Neither result releases ownership or stops the scheduler. Retry pending + handoffs/attempts or terminal cleanup through the existing execution handles and await again. +3. After a successful drain, stop/join application routing and destination execution loops and workers. + Application-wide worker stop retains chosen worker queues; runtime worker removal still uses its + existing return-to-router policy. +4. Close the dispatcher, then the coordinator. Dispatcher close refuses outstanding/failed executions + and never shuts down an application-owned scheduler. Coordinator close returns unsubmitted taken + work to its recorded destination and taken unassigned work to routing before closing stages. + +Stopping provider workers before drain can prevent callbacks or replacement attempts from finishing. +Unresolved shutdown must retain the live ownership and report failure rather than requeue a parent +while multipart callbacks are active. Memory closure remains non-durable and clears memory state; +returning ownership during shutdown does not promise restart recovery. This provider boundary is explicitly assembled and is not automatically wired into the standalone -router/worker loops. Application lifecycle activation and coordinated shutdown remain subsequent +router/worker loops. Standalone activation of this shutdown ordering remains subsequent integration work. Runtime attempt bookkeeping is non-durable; it does not provide provider-outcome checkpoints or restart recovery. diff --git a/sendium-core/src/main/java/gr/cytech/sendium/routing/StandardOutboundDispatch.java b/sendium-core/src/main/java/gr/cytech/sendium/routing/StandardOutboundDispatch.java index 75ba137a..3ccace3f 100644 --- a/sendium-core/src/main/java/gr/cytech/sendium/routing/StandardOutboundDispatch.java +++ b/sendium-core/src/main/java/gr/cytech/sendium/routing/StandardOutboundDispatch.java @@ -5,6 +5,7 @@ import gr.cytech.sendium.core.outbound.OutboundWork.Destination; import gr.cytech.sendium.core.outbound.OutboundWork.Routed; import gr.cytech.sendium.core.outbound.OutboundWork.Selected; +import gr.cytech.sendium.core.outbound.OutboundWork.WorkId; import gr.cytech.sendium.core.smpp.client.CoordinatedSmppSubmission; import gr.cytech.sendium.core.smpp.client.SmppClientWorker; import gr.cytech.sendium.core.storage.OutboundStage; @@ -33,6 +34,8 @@ public final class StandardOutboundDispatch implements AutoCloseable { private final ScheduledExecutorService scheduler; private final boolean ownsScheduler; private final ConcurrentMap, Semaphore> workerSlots = new ConcurrentHashMap<>(); + private final ConcurrentMap> activeExecutions = new ConcurrentHashMap<>(); + private boolean quiescing; public StandardOutboundDispatch(OutboundCoordinator coordinator, StandardRoutingManager routing) { this(coordinator, routing, Executors.newScheduledThreadPool(2, task -> { @@ -60,8 +63,12 @@ private StandardOutboundDispatch(OutboundCoordinator coordinato * Routes one owned selection. A miss or lookup failure returns it to routing. If recording the * destination fails, the caller retains the selection handle to retry the same routing intent. */ - public Optional> routeSelected(Selected selected) throws IOException { + public synchronized Optional> routeSelected(Selected selected) throws IOException { Objects.requireNonNull(selected, "selected"); + if (quiescing) { + coordinator.returnToRouter(selected); + return Optional.empty(); + } RoutingLookupResult result; try { result = routing.lookupForLifecycle(selected.message()); @@ -88,16 +95,17 @@ public Optional> takeForProvider(String destination, Dur } /** Runs an explicitly taken SMPP work item and carries its ownership through replacement attempts. */ - public ProviderExecution submitToProvider(Routed work, SmppClientWorker worker) + public synchronized ProviderExecution submitToProvider(Routed work, SmppClientWorker worker) throws Exception { + requireDispatching(); if (!work.destination().equals(worker.getFullName())) { throw new IllegalArgumentException("Provider does not match the recorded destination"); } var execution = new ProviderExecution(); - execution.retryCompletion = () -> coordinator.complete(work.id(), execution.attempts); + execution.retryCompletion = () -> track(work.id(), coordinator.complete(work.id(), execution.attempts)); CompletionStage attempts = scheduleAttempt(work.message(), () -> worker, execution); execution.attempts = attempts; - execution.completion = coordinator.complete(work.id(), attempts); + execution.completion = track(work.id(), coordinator.complete(work.id(), attempts)); return execution; } @@ -124,16 +132,75 @@ private SmppClientWorker retryDestination(StandardMessage messa } } - /** Call after provider executions have drained; this does not complete or close parent stores. */ + /** Stops new admission/takes and dispatch; existing provider retries and callbacks remain enabled. */ + public synchronized void quiesce() { + coordinator.quiesce(); + quiescing = true; + } + + /** Waits for terminal handoffs and cleanup. A timeout or failed handoff leaves ownership intact. */ + public boolean awaitProviderDrain(Duration timeout) throws InterruptedException { + Objects.requireNonNull(timeout, "timeout"); + if (timeout.isNegative()) { + throw new IllegalArgumentException("Negative drain timeout"); + } + CompletableFuture[] completions; + synchronized (this) { + if (!quiescing) { + throw new IllegalStateException("Quiesce dispatch before awaiting provider drain"); + } + completions = activeExecutions.values().stream().map(CompletionStage::toCompletableFuture) + .toArray(CompletableFuture[]::new); + } + try { + CompletableFuture.allOf(completions).get(timeout.toNanos(), TimeUnit.NANOSECONDS); + removeSuccessfulExecutions(); + return true; + } catch (java.util.concurrent.TimeoutException timeoutFailure) { + return false; + } catch (java.util.concurrent.ExecutionException failure) { + throw new IllegalStateException("Resolve failed provider handoffs or terminal cleanup before shutdown", failure.getCause()); + } + } + + private CompletionStage track(WorkId id, CompletionStage completion) { + activeExecutions.put(id, completion); + completion.whenComplete((ignored, failure) -> { + if (failure == null) { + activeExecutions.remove(id, completion); + } + }); + return completion; + } + + private void requireDispatching() { + if (quiescing) { + throw new IllegalStateException("Outbound dispatch is quiescing"); + } + } + + private void removeSuccessfulExecutions() { + activeExecutions.entrySet().removeIf(entry -> { + var completion = entry.getValue().toCompletableFuture(); + return completion.isDone() && !completion.isCompletedExceptionally(); + }); + } + + /** Call after drain and execution-loop stop/join; parent stores remain application-owned. */ @Override - public void close() { + public synchronized void close() { + quiesce(); + removeSuccessfulExecutions(); + if (!activeExecutions.isEmpty()) { + throw new IllegalStateException("Drain provider executions before closing outbound dispatch"); + } if (ownsScheduler) { scheduler.shutdown(); } } /** Completes explicitly supplied provider outcomes once all required part handoffs succeed. */ - public CompletionStage finishProviderParts(Routed work, + public synchronized CompletionStage finishProviderParts(Routed work, List> requiredHandoffs) { Objects.requireNonNull(work, "work"); List> handoffs = List.copyOf(requiredHandoffs); @@ -142,7 +209,7 @@ public CompletionStage finishProviderParts(Routed work, } CompletableFuture[] parts = handoffs.stream().map(CompletionStage::toCompletableFuture) .toArray(CompletableFuture[]::new); - return coordinator.complete(work.id(), CompletableFuture.allOf(parts)); + return track(work.id(), coordinator.complete(work.id(), CompletableFuture.allOf(parts))); } public enum AttemptState { diff --git a/sendium-core/src/main/java/gr/cytech/sendium/routing/StandardOutgoingWorkerHandler.java b/sendium-core/src/main/java/gr/cytech/sendium/routing/StandardOutgoingWorkerHandler.java index 36c0c356..43dac193 100644 --- a/sendium-core/src/main/java/gr/cytech/sendium/routing/StandardOutgoingWorkerHandler.java +++ b/sendium-core/src/main/java/gr/cytech/sendium/routing/StandardOutgoingWorkerHandler.java @@ -243,7 +243,7 @@ private void stopWorkers() { List stopWorkers = new ArrayList<>(outSmsWorkers.size()); for (AbstractOutWorker worker : outSmsWorkers.values()) { stopWorkers.add(worker.getFullName()); - stopFutures.add(ex.submit(() -> stopWorker(worker))); + stopFutures.add(ex.submit(() -> stopWorker(worker, false))); } for (int i = 0; i < stopFutures.size(); i++) { @@ -274,6 +274,10 @@ private void notifyAfterStartAll() { } private void stopWorker(AbstractOutWorker worker) { + stopWorker(worker, true); + } + + private void stopWorker(AbstractOutWorker worker, boolean runtimeRemoval) { final long start = System.currentTimeMillis(); logger.debug("stopping worker {} in thread {}", worker.getFullName(), Thread.currentThread().getName()); notifyBeforeWorkerStop(worker); @@ -284,7 +288,9 @@ private void stopWorker(AbstractOutWorker worker) { logger.warn("exception stopping worker: {}", worker.getFullName(), e); } outSmsWorkers.remove(worker.getInstanceName()); - worker.dequeueAllToRouter(); + if (runtimeRemoval) { + worker.dequeueAllToRouter(); + } logger.debug("stopping worker {} in thread {} took: {}ms", worker.getFullName(), Thread.currentThread().getName(), System.currentTimeMillis() - start); } diff --git a/sendium-core/src/test/java/gr/cytech/sendium/core/smpp/client/SmppClientWorkerTest.java b/sendium-core/src/test/java/gr/cytech/sendium/core/smpp/client/SmppClientWorkerTest.java index f2ac5d2f..ecd0a353 100644 --- a/sendium-core/src/test/java/gr/cytech/sendium/core/smpp/client/SmppClientWorkerTest.java +++ b/sendium-core/src/test/java/gr/cytech/sendium/core/smpp/client/SmppClientWorkerTest.java @@ -36,6 +36,7 @@ import java.util.concurrent.TimeUnit; import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.spy; import static org.mockito.Mockito.when; @@ -105,6 +106,9 @@ void failedRetryLookupCanResumeWithoutLosingParentOrRepeatingOriginalSubmission( var work = fixture.coordinator.takeForDestination(worker.getFullName(), Duration.ZERO).orElseThrow(); var execution = dispatch.submitToProvider(work, worker); worker.awaitRequests(1); + dispatch.quiesce(); + assertThat(dispatch.awaitProviderDrain(Duration.ZERO)).isFalse(); + assertThatThrownBy(() -> dispatch.submitToProvider(work, worker)).isInstanceOf(IllegalStateException.class); respond(handler(worker), worker.coordinatedRequests.getFirst(), SmppConstants.STATUS_INVDSTADR); assertThat(lookupTried.await(5, TimeUnit.SECONDS)).isTrue(); scheduler.submit(() -> { }).get(5, TimeUnit.SECONDS); @@ -118,6 +122,7 @@ void failedRetryLookupCanResumeWithoutLosingParentOrRepeatingOriginalSubmission( worker.awaitRequests(2); respond(handler(worker), worker.coordinatedRequests.getLast(), SmppConstants.STATUS_OK); execution.completion().toCompletableFuture().get(5, TimeUnit.SECONDS); + assertThat(dispatch.awaitProviderDrain(Duration.ofSeconds(1))).isTrue(); assertThat(worker.coordinatedRequests).hasSize(2); assertThat(fixture.pending.find(source)).isEmpty(); } diff --git a/sendium-core/src/test/java/gr/cytech/sendium/routing/StandardOutboundDispatchTest.java b/sendium-core/src/test/java/gr/cytech/sendium/routing/StandardOutboundDispatchTest.java index 95470802..41692f26 100644 --- a/sendium-core/src/test/java/gr/cytech/sendium/routing/StandardOutboundDispatchTest.java +++ b/sendium-core/src/test/java/gr/cytech/sendium/routing/StandardOutboundDispatchTest.java @@ -21,6 +21,55 @@ import static org.mockito.Mockito.when; class StandardOutboundDispatchTest { + @Test + void shutdownDrainsAllHandoffsAndRetainsOwnershipOnTimeoutOrFailure() throws Exception { + try (var fixture = new OutboundIngressFixture(1)) { + var dispatch = new StandardOutboundDispatch(fixture.coordinator, + routing(List.of(worker("smpp.provider", "provider")), "provider::default:")); + SourceId source = new SourceId(UUID.randomUUID()); + fixture.coordinator.admit(source, message("hello")); + routeNext(dispatch, fixture); + var taken = dispatch.takeForProvider("smpp.provider", Duration.ZERO).orElseThrow(); + var first = new CompletableFuture(); + var second = new CompletableFuture(); + dispatch.finishProviderParts(taken, List.of(first, second)); + dispatch.quiesce(); + first.complete(null); + assertThat(dispatch.awaitProviderDrain(Duration.ZERO)).isFalse(); + assertThatThrownBy(dispatch::close).isInstanceOf(IllegalStateException.class); + assertThat(fixture.pending.find(source)).isPresent(); + second.completeExceptionally(new IllegalStateException("handoff unavailable")); + assertThatThrownBy(() -> dispatch.awaitProviderDrain(Duration.ZERO)) + .isInstanceOf(IllegalStateException.class).hasCauseInstanceOf(IllegalStateException.class); + assertThatThrownBy(dispatch::close).isInstanceOf(IllegalStateException.class); + dispatch.finishProviderParts(taken, List.of(CompletableFuture.completedStage(null))); + assertThat(dispatch.awaitProviderDrain(Duration.ZERO)).isTrue(); + assertThat(fixture.pending.find(source)).isEmpty(); + dispatch.close(); + } + } + + @Test + void shutdownReturnsUnassignedWorkAndLeavesUnsubmittedWorkAtItsRecordedDestination() throws Exception { + try (var fixture = new OutboundIngressFixture(2)) { + var dispatch = new StandardOutboundDispatch(fixture.coordinator, + routing(List.of(worker("smpp.provider", "provider")), "provider::default:")); + fixture.coordinator.admit(new SourceId(UUID.randomUUID()), message("assigned")); + routeNext(dispatch, fixture); + var assigned = dispatch.takeForProvider("smpp.provider", Duration.ZERO).orElseThrow(); + fixture.coordinator.admit(new SourceId(UUID.randomUUID()), message("unassigned")); + fixture.coordinator.selectAndStage(1); + var selected = fixture.coordinator.takeForRouting(Duration.ZERO).orElseThrow(); + dispatch.quiesce(); + assertThat(dispatch.routeSelected(selected)).isEmpty(); + assertThat(fixture.router.take(Duration.ZERO).orElseThrow().id()).isEqualTo(selected.id()); + fixture.coordinator.returnToDestination(assigned); + assertThat(fixture.routed.take("smpp.provider", Duration.ZERO).orElseThrow().id()).isEqualTo(assigned.id()); + assertThat(dispatch.awaitProviderDrain(Duration.ZERO)).isTrue(); + dispatch.close(); + } + } + @Test void routesToOneRecordedDestinationAndRetainsSourcesThroughAllProviderPartHandoffs() throws Exception { try (var fixture = new OutboundIngressFixture(2)) { diff --git a/sendium-core/src/test/java/gr/cytech/sendium/routing/StandardWorkerShutdownTest.java b/sendium-core/src/test/java/gr/cytech/sendium/routing/StandardWorkerShutdownTest.java new file mode 100644 index 00000000..37456c8a --- /dev/null +++ b/sendium-core/src/test/java/gr/cytech/sendium/routing/StandardWorkerShutdownTest.java @@ -0,0 +1,46 @@ +package gr.cytech.sendium.routing; + +import gr.cytech.sendium.core.AbstractOutWorker; +import gr.cytech.sendium.conf.PropertyChangeEvent; +import gr.cytech.sendium.conf.SendiumConfigurationHandler; +import org.junit.jupiter.api.Test; + +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +class StandardWorkerShutdownTest { + @Test + void runtimeWorkerRemovalRetainsItsReturnToRouterPolicy() { + var handler = new StandardOutgoingWorkerHandler(); + handler.init(); + handler.configurationHandler = mock(SendiumConfigurationHandler.class); + handler.routingManager = mock(StandardRoutingManager.class); + var worker = mock(AbstractOutWorker.class); + when(worker.getInstanceName()).thenReturn("provider"); + when(worker.getFullName()).thenReturn("smpp.provider"); + handler.getWorkers().put("provider", worker); + + handler.propertyChange(new PropertyChangeEvent("outSms.instance.provider.enable", "false")); + + verify(worker).stop(); + verify(worker).dequeueAllToRouter(); + verify(handler.routingManager).beforeWorkerStop(worker); + } + @Test + void applicationShutdownStopsWorkersWithoutSendingTheirQueuesBackToRouting() { + var handler = new StandardOutgoingWorkerHandler(); + handler.routingManager = mock(StandardRoutingManager.class); + var worker = mock(AbstractOutWorker.class); + when(worker.getInstanceName()).thenReturn("provider"); + when(worker.getFullName()).thenReturn("smpp.provider"); + handler.getWorkers().put("provider", worker); + + handler.stopAllWorkers(); + + verify(worker).stop(); + verify(worker, never()).dequeueAllToRouter(); + verify(handler.routingManager).beforeStopAll(); + } +} From dc33da7689345054ce863d0b36a4be7f03371b6a Mon Sep 17 00:00:00 2001 From: pavlos Date: Tue, 6 Oct 2026 12:13:21 +0300 Subject: [PATCH 11/20] feat(storage): activate standalone message outbound pipeline Signed-off-by: pavlos --- docs/04-smpp-configuration.md | 7 +- docs/06-http-api.md | 2 +- docs/09-configuration-reference.md | 23 +- docs/14-outbound-storage.md | 61 +++-- sendium-app/pom.xml | 5 + ...rofile.java => MessageStorageProfile.java} | 12 +- .../storage/StandaloneMessageSnapshot.java | 67 +++++ .../app/storage/StandaloneMessageStorage.java | 43 ++++ .../storage/StandaloneOutboundPipeline.java | 232 ++++++++++++++++++ .../app/storage/StandaloneSmsStorage.java | 37 --- .../src/main/resources/application.properties | 10 +- ...nProbe.java => MessageAdmissionProbe.java} | 10 +- ...est.java => MessageLifecycleHttpTest.java} | 4 +- ...ava => MessageStorageFileStartupTest.java} | 4 +- .../MessageStorageMemoryStartupTest.java | 33 +++ ...MessageStoragePartialFileStartupTest.java} | 4 +- ... MessageStoragePostgresqlStartupTest.java} | 4 +- ...st.java => MessageStorageProfileTest.java} | 32 +-- ...java => MessageStorageStartupSupport.java} | 24 +- .../storage/SmsStorageMemoryStartupTest.java | 33 --- .../StandaloneMessageSnapshotTest.java | 53 ++++ .../StandaloneOutboundConfigurationTest.java | 46 ++++ .../StandaloneOutboundPipelineTest.java | 154 ++++++++++++ .../StandaloneOutboundShutdownTest.java | 91 +++++++ .../routing/StandardOutboundDispatch.java | 57 ++++- .../routing/StandardOutboundDispatchTest.java | 44 ++++ 26 files changed, 937 insertions(+), 155 deletions(-) rename sendium-app/src/main/java/gr/cytech/sendium/app/storage/{SmsStorageProfile.java => MessageStorageProfile.java} (57%) create mode 100644 sendium-app/src/main/java/gr/cytech/sendium/app/storage/StandaloneMessageSnapshot.java create mode 100644 sendium-app/src/main/java/gr/cytech/sendium/app/storage/StandaloneMessageStorage.java create mode 100644 sendium-app/src/main/java/gr/cytech/sendium/app/storage/StandaloneOutboundPipeline.java delete mode 100644 sendium-app/src/main/java/gr/cytech/sendium/app/storage/StandaloneSmsStorage.java rename sendium-app/src/test/java/gr/cytech/sendium/app/storage/{SmsAdmissionProbe.java => MessageAdmissionProbe.java} (65%) rename sendium-app/src/test/java/gr/cytech/sendium/app/storage/{SmsLifecycleHttpTest.java => MessageLifecycleHttpTest.java} (98%) rename sendium-app/src/test/java/gr/cytech/sendium/app/storage/{SmsStorageFileStartupTest.java => MessageStorageFileStartupTest.java} (71%) create mode 100644 sendium-app/src/test/java/gr/cytech/sendium/app/storage/MessageStorageMemoryStartupTest.java rename sendium-app/src/test/java/gr/cytech/sendium/app/storage/{SmsStoragePartialFileStartupTest.java => MessageStoragePartialFileStartupTest.java} (70%) rename sendium-app/src/test/java/gr/cytech/sendium/app/storage/{SmsStoragePostgresqlStartupTest.java => MessageStoragePostgresqlStartupTest.java} (68%) rename sendium-app/src/test/java/gr/cytech/sendium/app/storage/{SmsStorageProfileTest.java => MessageStorageProfileTest.java} (71%) rename sendium-app/src/test/java/gr/cytech/sendium/app/storage/{SmsStorageStartupSupport.java => MessageStorageStartupSupport.java} (66%) delete mode 100644 sendium-app/src/test/java/gr/cytech/sendium/app/storage/SmsStorageMemoryStartupTest.java create mode 100644 sendium-app/src/test/java/gr/cytech/sendium/app/storage/StandaloneMessageSnapshotTest.java create mode 100644 sendium-app/src/test/java/gr/cytech/sendium/app/storage/StandaloneOutboundConfigurationTest.java create mode 100644 sendium-app/src/test/java/gr/cytech/sendium/app/storage/StandaloneOutboundPipelineTest.java create mode 100644 sendium-app/src/test/java/gr/cytech/sendium/app/storage/StandaloneOutboundShutdownTest.java diff --git a/docs/04-smpp-configuration.md b/docs/04-smpp-configuration.md index 29e4619f..b7c67e65 100644 --- a/docs/04-smpp-configuration.md +++ b/docs/04-smpp-configuration.md @@ -64,15 +64,14 @@ All properties below should be prefixed with your instance path. For example, if ### Admission and acknowledgement -The default server accepts an ingress event into its local queue before sending `submit_sm_resp STATUS_OK`. -An application explicitly binding an outbound coordinator instead admits a complete message, or retains +The standalone server uses the shared memory outbound coordinator. It admits a complete message, or retains each concatenated part as held work, before success. Parts are acknowledged individually rather than waiting for the whole message. Completed/expired reassembly work retains those original source IDs. In that coordinated path, capacity rejection returns `STATUS_THROTTLED`, unsupported input returns `STATUS_SUBMITFAIL`, and other lifecycle/storage failures return `STATUS_SYSERR`. A failure after -acknowledgement during ready publication is retried without another client response. Standalone lifecycle -activation remains deferred until dispatch and completion are connected; see [Outbound Storage Contracts](14-outbound-storage.md). +acknowledgement during ready publication is retried without another client response. Embedded workers +without a coordinator binding retain local ingress-queue admission before success. See [Outbound Storage Contracts](14-outbound-storage.md). ## 🧵 Thread Pool Configuration diff --git a/docs/06-http-api.md b/docs/06-http-api.md index d4b8fde1..5bf07e9b 100644 --- a/docs/06-http-api.md +++ b/docs/06-http-api.md @@ -90,7 +90,7 @@ For a manual installation, replace the environment variables with the HTTP `syst 123e4567-e89b-12d3-a456-426614174000 ``` -In the default runtime, `202 Accepted` means insertion into the in-memory router queue. An application explicitly supplying the new outbound coordinator receives `202` only after pending admission succeeds; a failing coordinator is never replaced by legacy queue admission. Standalone activation of that lifecycle is deferred until dispatch and completion are connected. See [Outbound Storage Contracts](14-outbound-storage.md). +In the standalone runtime, `202 Accepted` means admission into the memory-backed pending store. The source remains retained through routing, provider processing, and required DLR handoffs. Capacity rejection returns `503`; a failing coordinator is never replaced by legacy queue admission. Embedded callers without a coordinator retain the legacy in-memory queue path. The memory profile is non-durable across restart. See [Outbound Storage Contracts](14-outbound-storage.md). Neither memory admission path writes an outbound message or DLR row to PostgreSQL. Acceptance does not prove that a viable route exists, that an upstream SMSC accepted the message, or that a handset received it. Durable DLR state is created only after an upstream provider outcome. Local-only Quick Start installations have no outbound route. Check routing configuration, the SMPP client connection, message lifecycle logs, submit response, and delivery receipt for those later stages. diff --git a/docs/09-configuration-reference.md b/docs/09-configuration-reference.md index 3104bba2..943c467e 100644 --- a/docs/09-configuration-reference.md +++ b/docs/09-configuration-reference.md @@ -43,7 +43,7 @@ In the Docker image, the working directory is `/work`, so the default configurat | `QUARKUS_HTTP_ACCESS_LOG_ENABLE` | `true` | Enables HTTP access logging. | | `QUARKUS_HTTP_ACCESS_LOG_DIRECTORY` | `/work/logs` | HTTP access log directory. | -## Outbound SMS Storage Profile +## Outbound Message Storage Profile The standalone application reads these global, startup-only settings from Quarkus configuration (`application.properties`, JVM `-D` properties, or the environment). They are not hot-reloaded worker @@ -51,9 +51,12 @@ settings in `smsg.properties`. | Property | Environment variable | Default | | :--- | :--- | :--- | -| `sendium.sms.pending.backend` | `SENDIUM_SMS_PENDING_BACKEND` | `memory` | -| `sendium.sms.router-queue.backend` | `SENDIUM_SMS_ROUTER_QUEUE_BACKEND` | `memory` | -| `sendium.sms.routed-work.backend` | `SENDIUM_SMS_ROUTED_WORK_BACKEND` | `memory` | +| `sendium.message.pending.backend` | `SENDIUM_MESSAGE_PENDING_BACKEND` | `memory` | +| `sendium.message.router-queue.backend` | `SENDIUM_MESSAGE_ROUTER_QUEUE_BACKEND` | `memory` | +| `sendium.message.routed-work.backend` | `SENDIUM_MESSAGE_ROUTED_WORK_BACKEND` | `memory` | +| `sendium.message.pending.capacity` | `SENDIUM_MESSAGE_PENDING_CAPACITY` | `10000` | +| `sendium.message.router-queue.capacity` | `SENDIUM_MESSAGE_ROUTER_QUEUE_CAPACITY` | `1000` | +| `sendium.message.selection-batch-size` | `SENDIUM_MESSAGE_SELECTION_BATCH_SIZE` | `100` | Only **`memory/memory/memory`** is currently selectable. Missing settings default to `memory`; explicitly empty, unknown, or unsupported values fail startup. Values are case-sensitive. Requests @@ -64,15 +67,17 @@ fallback to a less durable backend. Validation runs before Sendium's file-watcher and router/worker startup observers. The validated profile is retained for the process lifetime. Startup logs the effective profile and a prominent `NON-DURABLE` warning: accepted messages and in-flight work can be lost on restart. The current -runtime pipeline is still memory-backed. All three memory stage stores and the default lifecycle -coordinator exist as library components. HTTP/SMPP can bind to an explicitly supplied coordinator; -standalone activation still awaits connected dispatch, completion, and retry/rerouting work in #338. Their -capacity bounds are constructor arguments at this stage, not additional operator settings. +standalone pipeline binds HTTP/SMPP to one shared coordinator and executes bounded selection, routing, +SMPP-client submission, retries and terminal handoffs. All capacity/batch settings must be positive. +Pending capacity counts retained accepted sources (including held multipart parts); router capacity +counts queued plus taken unassigned selections. Routed capacity equals pending source capacity, so a +recorded selection cannot exhaust destination-record slots while its sources fit in pending storage. +Selection and each destination take loop process at most the configured batch size per poll. See [Outbound Storage Contracts](14-outbound-storage.md). The producer and startup observer live only in `sendium-app`. Embedding `sendium-core` does not activate this validation, configure an outbound coordinator, or enforce the standalone profile list. -Embedding applications own their assembly. Outbound SMS selectors do not change the independently +Embedding applications own their assembly. Outbound message selectors do not change the independently configured DLR subsystem below. ## DLR Storage Environment Variables diff --git a/docs/14-outbound-storage.md b/docs/14-outbound-storage.md index 9b6af370..9909c5f4 100644 --- a/docs/14-outbound-storage.md +++ b/docs/14-outbound-storage.md @@ -4,8 +4,14 @@ This describes the initial contracts for [#338](https://github.com/cytechmobile/ under [#337](https://github.com/cytechmobile/sendium/issues/337). The stage contracts are library APIs, and standalone profile selection, early validation, and startup logging are implemented. All three memory stage stores and the default coordinator are available as explicitly constructed library components. -HTTP/SMPP admission can bind to an explicitly supplied coordinator. Standalone activation remains deferred -until dispatch, completion, retry/rerouting, and shutdown are connected. The default runtime remains non-durable. +The standalone application binds HTTP/SMPP admission to one shared memory coordinator and runs selection, +routing, SMPP-client provider execution, retries and terminal handoffs end to end. The default runtime +remains explicitly non-durable. Importing the core library alone does not activate this assembly. + +Shared storage configuration uses the channel-neutral `sendium.message.*` namespace, with +`MessageStorageProfile` and `StandaloneMessageStorage` as the standalone profile classes. The same +ownership boundaries can serve future OTT provider integrations; the current activated provider +execution boundary remains SMPP-client submission. ## Ownership model @@ -46,8 +52,8 @@ retrying taken work at the same destination through `returnToDestination(Routed) be used after destination assignment and is not a worker-to-router transition. Sendium's existing `AbstractOutWorker` failure policies can call `enqueueToRouter` to let routing choose -again. That production behavior remains intact. Its source-retaining lifecycle integration is planned -separately; it must be implemented before the new pipeline replaces that path. No direct worker-to-worker +again. Legacy queue behavior remains intact; lifecycle provider retries perform a new lookup under the +original parent work identity, retaining every required provider-part handoff. No direct worker-to-worker API is substituted for it. Consumer-specific dispatchers, including mCore's queue worker, remain the consumer team's responsibility and require separate approval for Sendium support. @@ -64,7 +70,8 @@ requirements previously described in #337/#338. Those issue bodies have not been `StandardOutboundDispatch`, constructed explicitly with a coordinator and Sendium's routing manager, can route one already selected item to one recorded destination and take that work for provider -processing. The caller retains the selected handle if recording the destination needs a retry. +processing. The dispatcher retains the selected handle and original destination intent if recording +needs a retry. `retryRoutingTransitions()` resumes that transition without repeating lookup or filters. A routing miss returns the selection to the router; a matched copied rule fails before assignment and likewise returns the selection. `finishProviderParts` is called only after the provider has reported an outcome for every part. It waits for each part's required DLR/tracking handoff before @@ -140,9 +147,10 @@ Unresolved shutdown must retain the live ownership and report failure rather tha while multipart callbacks are active. Memory closure remains non-durable and clears memory state; returning ownership during shutdown does not promise restart recovery. -This provider boundary is explicitly assembled and is not automatically wired into the standalone -router/worker loops. Standalone activation of this shutdown ordering remains subsequent -integration work. Runtime attempt bookkeeping is non-durable; it does not +The standalone application owns separate lifecycle processors alongside retained legacy entry points. +Its early shutdown observer applies this ordering before normal worker/connection shutdown observers. +Drain-first shutdown has no forced cutoff: unresolved callbacks or required handoffs keep shutdown +waiting, with actionable failures and periodic waiting logs. Runtime attempt bookkeeping is non-durable; it does not provide provider-outcome checkpoints or restart recovery. ## Components @@ -169,9 +177,9 @@ storage/ Memory implementations live under `storage.memory`. Standalone configuration assembly lives separately in `sendium-app`, under `gr.cytech.sendium.app.storage`: -- `SmsStorageProfile` is an immutable validated configuration value. Its constructor is the single +- `MessageStorageProfile` is an immutable validated configuration value. Its constructor is the single supported-profile validator; it accepts only `memory/memory/memory`. -- `StandaloneSmsStorage` produces that profile as a CDI singleton from runtime configuration and +- `StandaloneMessageStorage` produces that profile as a CDI singleton from runtime configuration and requires it in an early startup observer. The observer logs the effective profile and non-durable warning. These classes are absent from the `sendium-core` artifact; they do not activate inside an embedding application. @@ -355,11 +363,24 @@ is instance-local and disappears on close/restart; no new durability guarantee i Other explicitly assembled SMPP workers can use `setIngressCoordinator` with their own message type. The SMPP binding cannot be changed after startup or any submission has begun. -No production coordinator producer is added at this intermediate step. With no coordinator binding, +`StandaloneOutboundPipeline` supplies a default-qualified, started singleton coordinator in `sendium-app`. +Its static default producer validates the profile and positive memory bounds before construction. +With no coordinator binding in an embedded application, the existing memory-queue admission path remains available. When a binding exists, resolution or admission failure does not fall back to that queue. Applications must supply a long-lived, started coordinator and connect selection, dispatch, completion, and retry policies before enabling the new -pipeline. The standard message snapshot mapper and standalone activation are subsequent integration work. +pipeline. Standalone assembly uses `StandaloneMessageSnapshot` to detach mutable protocol collections, +byte arrays and nested map/list attribute values. Unknown mutable extension objects and custom message +subtypes require application-owned assembly and a suitable snapshot policy rather than shallow copying. + +The standalone processors poll every 100 milliseconds, honor the global routing pause, select a bounded +batch, route selected work, and take bounded work for each available SMPP-client destination. Misses +and lookup failures return selections; routing filter drops complete their source without provider +dispatch. Recorded transition failures retain their original intent for retry; other destination work +can continue. Provider execution handles are retained for handoff, attempt and terminal-cleanup retries. +The activated provider boundary is SMPP-client submission: copied routes and other legacy worker types +are rejected before assignment with an explicit unsupported error and retained routing ownership. +Legacy queues remain available for callers explicitly using those entry points, not as admission fallback. ### HTTP @@ -410,8 +431,8 @@ old timer cannot consume a newer group that reused the same reference. Expiry on parts already in that group, not accepted parts still waiting in the ingress queue. Group-key rules, body assembly, relative timeout settings, and first-part-wins policy otherwise remain unchanged. -These changes do not activate provider dispatch or implement worker-to-router retry. No mCore-specific -workflow or storage adapter is added. +Standalone dispatch and source-retaining provider retries are connected through the shared coordinator. +No mCore-specific workflow or storage adapter is added. ## Memory routed-work implementation @@ -538,7 +559,7 @@ resolved/retried before close can finish. A refused close leaves the coordinator stores open. A return failure also leaves ownership available for a subsequent close retry. Once draining succeeds, close attempts every store and preserves close failures; closure is idempotent and restart is forbidden. This library lifecycle does not stop application executors or provide a forced -shutdown policy. Actual ingress/worker activation and application shutdown ordering remain integration work. +shutdown policy. The standalone assembly owns its processors and applies the drain-first ordering above. ## Completion and retry boundaries @@ -600,9 +621,9 @@ cannot mark unfinished work terminal, and does not imply that memory survives a The standalone application reads these selectors once during startup: ```properties -sendium.sms.pending.backend=memory -sendium.sms.router-queue.backend=memory -sendium.sms.routed-work.backend=memory +sendium.message.pending.backend=memory +sendium.message.router-queue.backend=memory +sendium.message.routed-work.backend=memory ``` | Pending / router / routed | Availability | Restart behavior | @@ -622,8 +643,8 @@ Future durable guarantees require surviving storage and remain at least once. Missing selectors default to `memory`. Explicit blanks, unknown values, and every unimplemented combination fail startup with the requested profile and supported choice. The values are case-sensitive and are read from Quarkus runtime configuration, not hot-reloaded `smsg.properties` worker settings. -Use the corresponding environment variables `SENDIUM_SMS_PENDING_BACKEND`, -`SENDIUM_SMS_ROUTER_QUEUE_BACKEND`, and `SENDIUM_SMS_ROUTED_WORK_BACKEND`, or JVM `-D` +Use the corresponding environment variables `SENDIUM_MESSAGE_PENDING_BACKEND`, +`SENDIUM_MESSAGE_ROUTER_QUEUE_BACKEND`, and `SENDIUM_MESSAGE_ROUTED_WORK_BACKEND`, or JVM `-D` properties. Changes require a restart; they do not require rebuilding the application. The startup observer uses `Interceptor.Priority.PLATFORM_BEFORE`, ahead of the existing file watchers diff --git a/sendium-app/pom.xml b/sendium-app/pom.xml index 3de8c9a0..58d140ed 100644 --- a/sendium-app/pom.xml +++ b/sendium-app/pom.xml @@ -37,6 +37,11 @@ assertj-core test + + org.mockito + mockito-core + test + diff --git a/sendium-app/src/main/java/gr/cytech/sendium/app/storage/SmsStorageProfile.java b/sendium-app/src/main/java/gr/cytech/sendium/app/storage/MessageStorageProfile.java similarity index 57% rename from sendium-app/src/main/java/gr/cytech/sendium/app/storage/SmsStorageProfile.java rename to sendium-app/src/main/java/gr/cytech/sendium/app/storage/MessageStorageProfile.java index 5eca61d5..11e53ef5 100644 --- a/sendium-app/src/main/java/gr/cytech/sendium/app/storage/SmsStorageProfile.java +++ b/sendium-app/src/main/java/gr/cytech/sendium/app/storage/MessageStorageProfile.java @@ -1,14 +1,14 @@ package gr.cytech.sendium.app.storage; -public record SmsStorageProfile(String pending, String routerQueue, String routedWork) { - public static final String PENDING_BACKEND = "sendium.sms.pending.backend"; - public static final String ROUTER_BACKEND = "sendium.sms.router-queue.backend"; - public static final String ROUTED_BACKEND = "sendium.sms.routed-work.backend"; +public record MessageStorageProfile(String pending, String routerQueue, String routedWork) { + public static final String PENDING_BACKEND = "sendium.message.pending.backend"; + public static final String ROUTER_BACKEND = "sendium.message.router-queue.backend"; + public static final String ROUTED_BACKEND = "sendium.message.routed-work.backend"; - public SmsStorageProfile { + public MessageStorageProfile { if (!"memory".equals(pending) || !"memory".equals(routerQueue) || !"memory".equals(routedWork)) { throw new IllegalArgumentException( - "Unsupported outbound SMS storage profile '" + pending + "/" + routerQueue + "/" + routedWork + + "Unsupported outbound message storage profile '" + pending + "/" + routerQueue + "/" + routedWork + "' (pending/router-queue/routed-work). Supported profiles: memory/memory/memory. " + "Configure " + PENDING_BACKEND + ", " + ROUTER_BACKEND + ", and " + ROUTED_BACKEND + "; file and postgresql outbound backends are not implemented. No fallback is available."); diff --git a/sendium-app/src/main/java/gr/cytech/sendium/app/storage/StandaloneMessageSnapshot.java b/sendium-app/src/main/java/gr/cytech/sendium/app/storage/StandaloneMessageSnapshot.java new file mode 100644 index 00000000..80287a4c --- /dev/null +++ b/sendium-app/src/main/java/gr/cytech/sendium/app/storage/StandaloneMessageSnapshot.java @@ -0,0 +1,67 @@ +package gr.cytech.sendium.app.storage; + +import gr.cytech.sendium.core.message.StandardMessage; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** Snapshot policy for the standalone protocol payload; embedded applications supply their own mapper. */ +final class StandaloneMessageSnapshot { + private StandaloneMessageSnapshot() { + } + + static StandardMessage copy(StandardMessage message) { + if (message.getClass() != StandardMessage.class) { + throw new IllegalArgumentException("Custom messages require an application-owned outbound snapshot policy"); + } + try { + var copy = (StandardMessage) message.clone(); + copy.attrs = message.attrs == null ? null : copyMap(message.attrs); + copy.tlvs = message.tlvs == null ? null : new HashMap<>(message.tlvs); + copy.reassembledParts = message.reassembledParts == null ? null : new ArrayList<>(message.reassembledParts); + copy.field1 = value(message.field1); + copy.field2 = value(message.field2); + copy.field3 = value(message.field3); + copy.field4 = value(message.field4); + copy.field5 = value(message.field5); + copy.field6 = value(message.field6); + copy.field7 = value(message.field7); + copy.field8 = value(message.field8); + copy.field9 = value(message.field9); + copy.field10 = value(message.field10); + return copy; + } catch (CloneNotSupportedException failure) { + throw new IllegalStateException("Cannot snapshot standalone message", failure); + } + } + + private static Map copyMap(Map original) { + var copy = new HashMap(); + original.forEach((key, item) -> copy.put(key, value(item))); + return copy; + } + + private static Object value(Object value) { + if (value == null || value instanceof String || value instanceof Boolean || value instanceof Character || + value instanceof Byte || value instanceof Short || value instanceof Integer || value instanceof Long || + value instanceof Float || value instanceof Double || value instanceof Enum) { + return value; + } + if (value instanceof byte[] bytes) { + return bytes.clone(); + } + if (value instanceof List list) { + var copy = new ArrayList<>(); + list.forEach(item -> copy.add(value(item))); + return copy; + } + if (value instanceof Map map) { + var copy = new HashMap<>(); + map.forEach((key, item) -> copy.put(value(key), value(item))); + return copy; + } + throw new IllegalArgumentException("Unsupported mutable standalone message field: " + value.getClass().getName()); + } +} diff --git a/sendium-app/src/main/java/gr/cytech/sendium/app/storage/StandaloneMessageStorage.java b/sendium-app/src/main/java/gr/cytech/sendium/app/storage/StandaloneMessageStorage.java new file mode 100644 index 00000000..414f577d --- /dev/null +++ b/sendium-app/src/main/java/gr/cytech/sendium/app/storage/StandaloneMessageStorage.java @@ -0,0 +1,43 @@ +package gr.cytech.sendium.app.storage; + +import io.quarkus.runtime.StartupEvent; +import jakarta.annotation.Priority; +import jakarta.enterprise.context.ApplicationScoped; +import jakarta.enterprise.event.Observes; +import jakarta.enterprise.inject.Produces; +import jakarta.inject.Singleton; +import jakarta.interceptor.Interceptor; +import org.eclipse.microprofile.config.Config; +import org.eclipse.microprofile.config.ConfigValue; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +@ApplicationScoped +public class StandaloneMessageStorage { + private static final Logger logger = LoggerFactory.getLogger(StandaloneMessageStorage.class); + + @Produces + @Singleton + MessageStorageProfile profile(Config config) { + try { + return new MessageStorageProfile(backend(config, MessageStorageProfile.PENDING_BACKEND), + backend(config, MessageStorageProfile.ROUTER_BACKEND), + backend(config, MessageStorageProfile.ROUTED_BACKEND)); + } catch (IllegalArgumentException failure) { + // Startup can exit before asynchronous log handlers flush. Selectors contain no message payload. + System.err.println("Outbound message storage startup failed: " + failure.getMessage()); + throw failure; + } + } + + void start(@Observes @Priority(Interceptor.Priority.PLATFORM_BEFORE) StartupEvent event, MessageStorageProfile profile) { + logger.info("Outbound message storage profile (pending/router-queue/routed-work): {}", profile); + logger.warn("NON-DURABLE outbound message storage: accepted messages and in-flight work can be lost on process restart."); + } + + private static String backend(Config config, String name) { + ConfigValue property = config.getConfigValue(name); + // An absent setting uses the default; an explicitly empty setting must fail validation. + return property.getRawValue() == null ? "memory" : property.getValue(); + } +} diff --git a/sendium-app/src/main/java/gr/cytech/sendium/app/storage/StandaloneOutboundPipeline.java b/sendium-app/src/main/java/gr/cytech/sendium/app/storage/StandaloneOutboundPipeline.java new file mode 100644 index 00000000..060080ce --- /dev/null +++ b/sendium-app/src/main/java/gr/cytech/sendium/app/storage/StandaloneOutboundPipeline.java @@ -0,0 +1,232 @@ +package gr.cytech.sendium.app.storage; + +import gr.cytech.sendium.conf.SendiumConfigurationHandler; +import gr.cytech.sendium.core.message.StandardMessage; +import gr.cytech.sendium.core.outbound.DefaultOutboundCoordinator; +import gr.cytech.sendium.core.outbound.OutboundCoordinator; +import gr.cytech.sendium.core.outbound.OutboundWork.WorkId; +import gr.cytech.sendium.core.smpp.client.SmppClientWorker; +import gr.cytech.sendium.core.storage.OutboundStage; +import gr.cytech.sendium.core.storage.OutboundStorageException; +import gr.cytech.sendium.core.storage.memory.MemoryPendingMessageStore; +import gr.cytech.sendium.core.storage.memory.MemoryRoutedWorkStore; +import gr.cytech.sendium.core.storage.memory.MemorySelectedRouterStore; +import gr.cytech.sendium.external.filter.FilterException; +import gr.cytech.sendium.routing.AbstractRoutingManager; +import gr.cytech.sendium.routing.OutgoingWorkerManager; +import gr.cytech.sendium.routing.RoutingLookupResult; +import gr.cytech.sendium.routing.StandardOutboundDispatch; +import gr.cytech.sendium.routing.StandardRoutingManager; +import io.quarkus.arc.DefaultBean; +import io.quarkus.runtime.ShutdownEvent; +import io.quarkus.runtime.StartupEvent; +import jakarta.annotation.Priority; +import jakarta.enterprise.context.ApplicationScoped; +import jakarta.enterprise.event.Observes; +import jakarta.enterprise.inject.Instance; +import jakarta.enterprise.inject.Produces; +import jakarta.enterprise.inject.Vetoed; +import jakarta.inject.Inject; +import jakarta.inject.Singleton; +import jakarta.interceptor.Interceptor; +import org.eclipse.microprofile.config.Config; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.time.Duration; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; + +/** Standalone-only assembly; importing sendium-core never starts these processors. */ +@ApplicationScoped +public class StandaloneOutboundPipeline { + private static final Logger logger = LoggerFactory.getLogger(StandaloneOutboundPipeline.class); + @Inject Instance> coordinators; + OutboundCoordinator coordinator; + @Inject StandardRoutingManager routing; + @Inject OutgoingWorkerManager workers; + @Inject Config config; + @Inject SendiumConfigurationHandler configuration; + private StandardOutboundDispatch dispatch; + private ScheduledExecutorService processors; + private int batchSize; + private final Map executions = new ConcurrentHashMap<>(); + + @Produces + @Singleton + @DefaultBean + static OutboundCoordinator coordinator(MessageStorageProfile profile, Config config) { + int sourceCapacity = positive(config, "pending.capacity", 10000); + positive(config, "selection-batch-size", 100); + var pending = new MemoryPendingMessageStore(sourceCapacity, + StandaloneMessageSnapshot::copy); + var selected = new MemorySelectedRouterStore<>(pending, positive(config, "router-queue.capacity", 1000)); + var routed = new MemoryRoutedWorkStore(sourceCapacity, + StandaloneMessageSnapshot::copy); + var result = new DefaultOutboundCoordinator<>(pending, selected, routed); + result.start(); + return result; + } + + void start(@Observes @Priority(Interceptor.Priority.APPLICATION + 100) StartupEvent event) { + if (coordinator == null) { + coordinator = coordinators.get(); + } + batchSize = positive(config, "selection-batch-size", 100); + dispatch = new StandardOutboundDispatch(coordinator, new LifecycleRouting(routing)); + processors = Executors.newSingleThreadScheduledExecutor(task -> { + var thread = new Thread(task, "standalone-outbound"); + thread.setDaemon(true); + return thread; + }); + processors.scheduleWithFixedDelay(this::process, 0, 100, TimeUnit.MILLISECONDS); + logger.info("Standalone outbound memory pipeline activated"); + } + + void process() { + try { + retryExecutions(); + if (configuration.getBlnPrpt(AbstractRoutingManager._pause)) { + return; + } + coordinator.selectAndStage(batchSize); + for (int i = 0; i < batchSize; i++) { + var selected = coordinator.takeForRouting(Duration.ZERO); + if (selected.isEmpty()) { + break; + } + try { + dispatch.routeSelected(selected.orElseThrow()); + } catch (Exception failure) { + logFailure("routing", failure); + } + } + } catch (InterruptedException interrupted) { + Thread.currentThread().interrupt(); + } catch (Exception failure) { + logFailure("selection", failure); + } + try { + dispatch.retryRoutingTransitions(); + } catch (Exception failure) { + logFailure("routing transition", failure); + } + try { + for (var worker : workers.getWorkersCopy()) { + if (!(worker instanceof SmppClientWorker client) || !worker.acceptsMessages() || worker.isPause()) { + continue; + } + for (int i = 0; i < batchSize; i++) { + var work = dispatch.takeForProvider(worker.getFullName(), Duration.ZERO); + if (work.isEmpty()) { + break; + } + try { + var execution = dispatch.submitToProvider(work.orElseThrow(), (SmppClientWorker) client); + executions.put(work.orElseThrow().id(), execution); + } catch (Exception failure) { + coordinator.returnToDestination(work.orElseThrow()); + throw failure; + } + } + } + } catch (InterruptedException interrupted) { + Thread.currentThread().interrupt(); + } catch (Exception failure) { + logFailure("provider dispatch", failure); + } + } + + private void retryExecutions() { + executions.entrySet().removeIf(entry -> { + var execution = entry.getValue(); + var completion = execution.completion().toCompletableFuture(); + if (completion.isDone() && !completion.isCompletedExceptionally()) { + return true; + } + execution.retryPendingHandoffs(); + if (completion.isCompletedExceptionally()) { + execution.retryTerminalCleanup(); + } + return false; + }); + } + + void stop(@Observes @Priority(Interceptor.Priority.PLATFORM_BEFORE) ShutdownEvent event) throws InterruptedException { + if (dispatch == null) { + return; + } + dispatch.quiesce(); + processors.shutdown(); + while (!processors.awaitTermination(1, TimeUnit.SECONDS)) { + logger.warn("Waiting for standalone outbound processors to stop before ownership restoration"); + } + boolean drained = false; + int waits = 0; + while (!drained) { + try { + retryExecutions(); + dispatch.retryRoutingTransitions(); + drained = dispatch.awaitProviderDrain(Duration.ofSeconds(1)); + } catch (RuntimeException failure) { + logFailure("shutdown drain; retrying required handoffs and cleanup", failure); + TimeUnit.SECONDS.sleep(1); + } + if (!drained && ++waits % 30 == 0) { + logger.warn("Outbound shutdown still awaiting provider completion; workers and source ownership remain active"); + } + } + routing.stop(); + if (!workers.stop()) { + throw new IllegalStateException("Worker shutdown incomplete; outbound stages remain open"); + } + dispatch.close(); + coordinator.close(); + logger.info("Standalone outbound pipeline stopped after provider drain"); + } + + private static int positive(Config config, String suffix, int defaultValue) { + String key = "sendium.message." + suffix; + int value = config.getOptionalValue(key, Integer.class).orElse(defaultValue); + if (value <= 0) { + throw new IllegalArgumentException(key + " must be positive"); + } + return value; + } + + private static void logFailure(String operation, Exception failure) { + if (failure instanceof OutboundStorageException storage) { + logger.error("Outbound {} failed; ownership retained stage={} reason={} detail={}", + operation, storage.stage(), storage.reason(), storage.getMessage()); + } else if (failure instanceof FilterException filter) { + logger.warn("Outbound {} deferred by filter; ownership retained status={}", operation, filter.getStatusCode()); + } else { + logger.error("Outbound {} failed; ownership retained failureType={}", operation, failure.getClass().getName()); + } + } + + /** The activated provider boundary supports SMPP submission, not arbitrary legacy worker methods. */ + @Vetoed + static final class LifecycleRouting extends StandardRoutingManager { + private final StandardRoutingManager delegate; + + LifecycleRouting(StandardRoutingManager delegate) { + this.delegate = delegate; + } + + @Override + public RoutingLookupResult lookupForLifecycle(StandardMessage message) throws IOException { + var result = delegate.lookupForLifecycle(message); + if (result.getDestinations().stream().anyMatch(worker -> !(worker instanceof SmppClientWorker))) { + throw new OutboundStorageException(OutboundStage.Role.ROUTER_QUEUE, + OutboundStorageException.Reason.UNSUPPORTED, + "Standalone outbound execution requires an SMPP-client destination"); + } + return result; + } + } +} diff --git a/sendium-app/src/main/java/gr/cytech/sendium/app/storage/StandaloneSmsStorage.java b/sendium-app/src/main/java/gr/cytech/sendium/app/storage/StandaloneSmsStorage.java deleted file mode 100644 index 4cd8fc41..00000000 --- a/sendium-app/src/main/java/gr/cytech/sendium/app/storage/StandaloneSmsStorage.java +++ /dev/null @@ -1,37 +0,0 @@ -package gr.cytech.sendium.app.storage; - -import io.quarkus.runtime.StartupEvent; -import jakarta.annotation.Priority; -import jakarta.enterprise.context.ApplicationScoped; -import jakarta.enterprise.event.Observes; -import jakarta.enterprise.inject.Produces; -import jakarta.inject.Singleton; -import jakarta.interceptor.Interceptor; -import org.eclipse.microprofile.config.Config; -import org.eclipse.microprofile.config.ConfigValue; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -@ApplicationScoped -public class StandaloneSmsStorage { - private static final Logger logger = LoggerFactory.getLogger(StandaloneSmsStorage.class); - - @Produces - @Singleton - SmsStorageProfile profile(Config config) { - return new SmsStorageProfile(backend(config, SmsStorageProfile.PENDING_BACKEND), - backend(config, SmsStorageProfile.ROUTER_BACKEND), - backend(config, SmsStorageProfile.ROUTED_BACKEND)); - } - - void start(@Observes @Priority(Interceptor.Priority.PLATFORM_BEFORE) StartupEvent event, SmsStorageProfile profile) { - logger.info("Outbound SMS storage profile (pending/router-queue/routed-work): {}", profile); - logger.warn("NON-DURABLE outbound SMS storage: accepted messages and in-flight work can be lost on process restart."); - } - - private static String backend(Config config, String name) { - ConfigValue property = config.getConfigValue(name); - // An absent setting uses the default; an explicitly empty setting must fail validation. - return property.getRawValue() == null ? "memory" : property.getValue(); - } -} diff --git a/sendium-app/src/main/resources/application.properties b/sendium-app/src/main/resources/application.properties index 61c5d32c..605ba5b4 100644 --- a/sendium-app/src/main/resources/application.properties +++ b/sendium-app/src/main/resources/application.properties @@ -3,9 +3,13 @@ smsg.properties.file.path=conf/smsg.properties smsg.credentials.file.path=conf/credentials.yml # Global, startup-only outbound storage profile. Only memory/memory/memory is implemented. -sendium.sms.pending.backend=memory -sendium.sms.router-queue.backend=memory -sendium.sms.routed-work.backend=memory +sendium.message.pending.backend=memory +sendium.message.router-queue.backend=memory +sendium.message.routed-work.backend=memory +# Positive, startup-only memory bounds; routed records are bounded by retained source capacity. +sendium.message.pending.capacity=10000 +sendium.message.router-queue.capacity=1000 +sendium.message.selection-batch-size=100 # DLR persistence. PostgreSQL requires valid named datasource connection settings. sendium.dlr.persistence.enabled=true diff --git a/sendium-app/src/test/java/gr/cytech/sendium/app/storage/SmsAdmissionProbe.java b/sendium-app/src/test/java/gr/cytech/sendium/app/storage/MessageAdmissionProbe.java similarity index 65% rename from sendium-app/src/test/java/gr/cytech/sendium/app/storage/SmsAdmissionProbe.java rename to sendium-app/src/test/java/gr/cytech/sendium/app/storage/MessageAdmissionProbe.java index e19f29d2..e14ae728 100644 --- a/sendium-app/src/test/java/gr/cytech/sendium/app/storage/SmsAdmissionProbe.java +++ b/sendium-app/src/test/java/gr/cytech/sendium/app/storage/MessageAdmissionProbe.java @@ -9,18 +9,18 @@ import org.slf4j.LoggerFactory; @ApplicationScoped -public class SmsAdmissionProbe { +public class MessageAdmissionProbe { @Inject - SmsStorageProfile profile; + MessageStorageProfile profile; - private SmsStorageProfile profileAtStartup; + private MessageStorageProfile profileAtStartup; void start(@Observes @Priority(Interceptor.Priority.APPLICATION) StartupEvent event) { profileAtStartup = profile; - LoggerFactory.getLogger(SmsAdmissionProbe.class).info("test admission startup reached"); + LoggerFactory.getLogger(MessageAdmissionProbe.class).info("test admission startup reached"); } - public SmsStorageProfile profileAtStartup() { + public MessageStorageProfile profileAtStartup() { return profileAtStartup; } } diff --git a/sendium-app/src/test/java/gr/cytech/sendium/app/storage/SmsLifecycleHttpTest.java b/sendium-app/src/test/java/gr/cytech/sendium/app/storage/MessageLifecycleHttpTest.java similarity index 98% rename from sendium-app/src/test/java/gr/cytech/sendium/app/storage/SmsLifecycleHttpTest.java rename to sendium-app/src/test/java/gr/cytech/sendium/app/storage/MessageLifecycleHttpTest.java index f1e88ee7..7e1fdafd 100644 --- a/sendium-app/src/test/java/gr/cytech/sendium/app/storage/SmsLifecycleHttpTest.java +++ b/sendium-app/src/test/java/gr/cytech/sendium/app/storage/MessageLifecycleHttpTest.java @@ -33,11 +33,11 @@ import static org.assertj.core.api.Assertions.assertThat; -class SmsLifecycleHttpTest { +class MessageLifecycleHttpTest { @RegisterExtension static final QuarkusExtensionTest APP = new QuarkusExtensionTest() .withApplicationRoot(archive -> archive.addClasses(KannelResource.class, Beans.class, Credentials.class, - StandaloneSmsStorage.class, SmsStorageProfile.class)) + StandaloneMessageStorage.class, MessageStorageProfile.class)) .overrideConfigKey("quarkus.arc.exclude-dependency.core.group-id", "gr.cytech") .overrideConfigKey("quarkus.arc.exclude-dependency.core.artifact-id", "sendium-core") .overrideConfigKey("sendium.dlr.persistence.enabled", "false") diff --git a/sendium-app/src/test/java/gr/cytech/sendium/app/storage/SmsStorageFileStartupTest.java b/sendium-app/src/test/java/gr/cytech/sendium/app/storage/MessageStorageFileStartupTest.java similarity index 71% rename from sendium-app/src/test/java/gr/cytech/sendium/app/storage/SmsStorageFileStartupTest.java rename to sendium-app/src/test/java/gr/cytech/sendium/app/storage/MessageStorageFileStartupTest.java index 5e3e99ff..c4f85125 100644 --- a/sendium-app/src/test/java/gr/cytech/sendium/app/storage/SmsStorageFileStartupTest.java +++ b/sendium-app/src/test/java/gr/cytech/sendium/app/storage/MessageStorageFileStartupTest.java @@ -6,9 +6,9 @@ import static org.junit.jupiter.api.Assertions.fail; -class SmsStorageFileStartupTest { +class MessageStorageFileStartupTest { @RegisterExtension - static final QuarkusExtensionTest APP = SmsStorageStartupSupport.rejected("file", "file", "memory"); + static final QuarkusExtensionTest APP = MessageStorageStartupSupport.rejected("file", "file", "memory"); @Test void startupMustFailBeforeAdmission() { diff --git a/sendium-app/src/test/java/gr/cytech/sendium/app/storage/MessageStorageMemoryStartupTest.java b/sendium-app/src/test/java/gr/cytech/sendium/app/storage/MessageStorageMemoryStartupTest.java new file mode 100644 index 00000000..6065f2a9 --- /dev/null +++ b/sendium-app/src/test/java/gr/cytech/sendium/app/storage/MessageStorageMemoryStartupTest.java @@ -0,0 +1,33 @@ +package gr.cytech.sendium.app.storage; + +import io.quarkus.test.QuarkusExtensionTest; +import jakarta.inject.Inject; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.RegisterExtension; + +import static org.assertj.core.api.Assertions.assertThat; + +class MessageStorageMemoryStartupTest { + @RegisterExtension + static final QuarkusExtensionTest APP = MessageStorageStartupSupport.application() + .overrideRuntimeConfigKey(MessageStorageProfile.PENDING_BACKEND, "memory") + .overrideRuntimeConfigKey(MessageStorageProfile.ROUTER_BACKEND, "memory") + .overrideRuntimeConfigKey(MessageStorageProfile.ROUTED_BACKEND, "memory") + .assertLogRecords(records -> assertThat(records).extracting(MessageStorageStartupSupport::message) + .containsExactly( + "Outbound message storage profile (pending/router-queue/routed-work): memory/memory/memory", + "NON-DURABLE outbound message storage: accepted messages and in-flight work can be lost on process restart.", + MessageStorageStartupSupport.ADMISSION_STARTED)); + + @Inject + MessageStorageProfile profile; + + @Inject + MessageAdmissionProbe admission; + + @Test + void validatedProfileIsAvailableBeforeAdmissionStartup() { + assertThat(profile).isEqualTo(new MessageStorageProfile("memory", "memory", "memory")); + assertThat(admission.profileAtStartup()).isSameAs(profile); + } +} diff --git a/sendium-app/src/test/java/gr/cytech/sendium/app/storage/SmsStoragePartialFileStartupTest.java b/sendium-app/src/test/java/gr/cytech/sendium/app/storage/MessageStoragePartialFileStartupTest.java similarity index 70% rename from sendium-app/src/test/java/gr/cytech/sendium/app/storage/SmsStoragePartialFileStartupTest.java rename to sendium-app/src/test/java/gr/cytech/sendium/app/storage/MessageStoragePartialFileStartupTest.java index 9759efac..dcd4523c 100644 --- a/sendium-app/src/test/java/gr/cytech/sendium/app/storage/SmsStoragePartialFileStartupTest.java +++ b/sendium-app/src/test/java/gr/cytech/sendium/app/storage/MessageStoragePartialFileStartupTest.java @@ -6,9 +6,9 @@ import static org.junit.jupiter.api.Assertions.fail; -class SmsStoragePartialFileStartupTest { +class MessageStoragePartialFileStartupTest { @RegisterExtension - static final QuarkusExtensionTest APP = SmsStorageStartupSupport.rejected("memory", "memory", "file"); + static final QuarkusExtensionTest APP = MessageStorageStartupSupport.rejected("memory", "memory", "file"); @Test void startupMustFailBeforeAdmission() { diff --git a/sendium-app/src/test/java/gr/cytech/sendium/app/storage/SmsStoragePostgresqlStartupTest.java b/sendium-app/src/test/java/gr/cytech/sendium/app/storage/MessageStoragePostgresqlStartupTest.java similarity index 68% rename from sendium-app/src/test/java/gr/cytech/sendium/app/storage/SmsStoragePostgresqlStartupTest.java rename to sendium-app/src/test/java/gr/cytech/sendium/app/storage/MessageStoragePostgresqlStartupTest.java index 2f5c045e..69c81deb 100644 --- a/sendium-app/src/test/java/gr/cytech/sendium/app/storage/SmsStoragePostgresqlStartupTest.java +++ b/sendium-app/src/test/java/gr/cytech/sendium/app/storage/MessageStoragePostgresqlStartupTest.java @@ -6,9 +6,9 @@ import static org.junit.jupiter.api.Assertions.fail; -class SmsStoragePostgresqlStartupTest { +class MessageStoragePostgresqlStartupTest { @RegisterExtension - static final QuarkusExtensionTest APP = SmsStorageStartupSupport.rejected("postgresql", "postgresql", "postgresql"); + static final QuarkusExtensionTest APP = MessageStorageStartupSupport.rejected("postgresql", "postgresql", "postgresql"); @Test void startupMustFailBeforeAdmission() { diff --git a/sendium-app/src/test/java/gr/cytech/sendium/app/storage/SmsStorageProfileTest.java b/sendium-app/src/test/java/gr/cytech/sendium/app/storage/MessageStorageProfileTest.java similarity index 71% rename from sendium-app/src/test/java/gr/cytech/sendium/app/storage/SmsStorageProfileTest.java rename to sendium-app/src/test/java/gr/cytech/sendium/app/storage/MessageStorageProfileTest.java index 0c8bd970..52ec242e 100644 --- a/sendium-app/src/test/java/gr/cytech/sendium/app/storage/SmsStorageProfileTest.java +++ b/sendium-app/src/test/java/gr/cytech/sendium/app/storage/MessageStorageProfileTest.java @@ -16,7 +16,7 @@ import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; -class SmsStorageProfileTest { +class MessageStorageProfileTest { @Test void missingSelectorsUseMemoryDefaults() { assertThat(profile(Map.of()).toString()).isEqualTo("memory/memory/memory"); @@ -25,35 +25,35 @@ void missingSelectorsUseMemoryDefaults() { @Test void explicitMemoryIsAccepted() { assertThat(profile(Map.of( - SmsStorageProfile.PENDING_BACKEND, "memory", - SmsStorageProfile.ROUTER_BACKEND, "memory", - SmsStorageProfile.ROUTED_BACKEND, "memory"))) - .isEqualTo(new SmsStorageProfile("memory", "memory", "memory")); + MessageStorageProfile.PENDING_BACKEND, "memory", + MessageStorageProfile.ROUTER_BACKEND, "memory", + MessageStorageProfile.ROUTED_BACKEND, "memory"))) + .isEqualTo(new MessageStorageProfile("memory", "memory", "memory")); } @ParameterizedTest @MethodSource("unsupportedProfiles") void rejectsEveryUnimplementedCombination(String pending, String router, String routed) { assertThatThrownBy(() -> profile(Map.of( - SmsStorageProfile.PENDING_BACKEND, pending, - SmsStorageProfile.ROUTER_BACKEND, router, - SmsStorageProfile.ROUTED_BACKEND, routed))) + MessageStorageProfile.PENDING_BACKEND, pending, + MessageStorageProfile.ROUTER_BACKEND, router, + MessageStorageProfile.ROUTED_BACKEND, routed))) .isInstanceOf(IllegalArgumentException.class) .hasMessageContaining("'" + pending + "/" + router + "/" + routed + "'") .hasMessageContaining("Supported profiles: memory/memory/memory") - .hasMessageContaining(SmsStorageProfile.PENDING_BACKEND) - .hasMessageContaining(SmsStorageProfile.ROUTER_BACKEND) - .hasMessageContaining(SmsStorageProfile.ROUTED_BACKEND); + .hasMessageContaining(MessageStorageProfile.PENDING_BACKEND) + .hasMessageContaining(MessageStorageProfile.ROUTER_BACKEND) + .hasMessageContaining(MessageStorageProfile.ROUTED_BACKEND); } @ParameterizedTest @ValueSource(strings = {"", " ", "disk", "MEMORY"}) void explicitInvalidValuesNeverUseTheDefault(String value) { - for (String key : List.of(SmsStorageProfile.PENDING_BACKEND, - SmsStorageProfile.ROUTER_BACKEND, SmsStorageProfile.ROUTED_BACKEND)) { + for (String key : List.of(MessageStorageProfile.PENDING_BACKEND, + MessageStorageProfile.ROUTER_BACKEND, MessageStorageProfile.ROUTED_BACKEND)) { assertThatThrownBy(() -> profile(Map.of(key, value))) .isInstanceOf(IllegalArgumentException.class) - .hasMessageContaining("Unsupported outbound SMS storage profile"); + .hasMessageContaining("Unsupported outbound message storage profile"); } } @@ -64,7 +64,7 @@ private static Stream unsupportedProfiles() { .map(routed -> Arguments.of(pending, router, routed)))); } - private static SmsStorageProfile profile(Map settings) { + private static MessageStorageProfile profile(Map settings) { var config = new SmallRyeConfigBuilder().withSources(new ConfigSource() { @Override public Map getProperties() { @@ -86,6 +86,6 @@ public String getName() { return "profile-test"; } }).build(); - return new StandaloneSmsStorage().profile(config); + return new StandaloneMessageStorage().profile(config); } } diff --git a/sendium-app/src/test/java/gr/cytech/sendium/app/storage/SmsStorageStartupSupport.java b/sendium-app/src/test/java/gr/cytech/sendium/app/storage/MessageStorageStartupSupport.java similarity index 66% rename from sendium-app/src/test/java/gr/cytech/sendium/app/storage/SmsStorageStartupSupport.java rename to sendium-app/src/test/java/gr/cytech/sendium/app/storage/MessageStorageStartupSupport.java index d8206150..fd29b541 100644 --- a/sendium-app/src/test/java/gr/cytech/sendium/app/storage/SmsStorageStartupSupport.java +++ b/sendium-app/src/test/java/gr/cytech/sendium/app/storage/MessageStorageStartupSupport.java @@ -7,36 +7,36 @@ import static org.assertj.core.api.Assertions.assertThat; -final class SmsStorageStartupSupport { +final class MessageStorageStartupSupport { static final String ADMISSION_STARTED = "test admission startup reached"; - private SmsStorageStartupSupport() { + private MessageStorageStartupSupport() { } static QuarkusExtensionTest application() { return new QuarkusExtensionTest() .withApplicationRoot(archive -> archive.addClasses( - StandaloneSmsStorage.class, SmsStorageProfile.class, SmsAdmissionProbe.class, - SmsStorageStartupSupport.class)) + StandaloneMessageStorage.class, MessageStorageProfile.class, MessageAdmissionProbe.class, + MessageStorageStartupSupport.class)) // Exercise the standalone assembly independently of protocol worker beans. .overrideConfigKey("quarkus.arc.exclude-dependency.core.group-id", "gr.cytech") .overrideConfigKey("quarkus.arc.exclude-dependency.core.artifact-id", "sendium-core") .overrideConfigKey("sendium.dlr.persistence.enabled", "false") .overrideConfigKey("quarkus.http.test-port", "0") // These assertions must remain valid when CI sets the root log level to ERROR. - .overrideConfigKey("quarkus.log.category.\"gr.cytech.sendium.app.storage.StandaloneSmsStorage\".level", "INFO") - .overrideConfigKey("quarkus.log.category.\"gr.cytech.sendium.app.storage.SmsAdmissionProbe\".level", "INFO") - .setLogRecordPredicate(record -> StandaloneSmsStorage.class.getName().equals(record.getLoggerName()) || - SmsAdmissionProbe.class.getName().equals(record.getLoggerName())); + .overrideConfigKey("quarkus.log.category.\"gr.cytech.sendium.app.storage.StandaloneMessageStorage\".level", "INFO") + .overrideConfigKey("quarkus.log.category.\"gr.cytech.sendium.app.storage.MessageAdmissionProbe\".level", "INFO") + .setLogRecordPredicate(record -> StandaloneMessageStorage.class.getName().equals(record.getLoggerName()) || + MessageAdmissionProbe.class.getName().equals(record.getLoggerName())); } static QuarkusExtensionTest rejected(String pending, String router, String routed) { return application() - .overrideRuntimeConfigKey(SmsStorageProfile.PENDING_BACKEND, pending) - .overrideRuntimeConfigKey(SmsStorageProfile.ROUTER_BACKEND, router) - .overrideRuntimeConfigKey(SmsStorageProfile.ROUTED_BACKEND, routed) + .overrideRuntimeConfigKey(MessageStorageProfile.PENDING_BACKEND, pending) + .overrideRuntimeConfigKey(MessageStorageProfile.ROUTER_BACKEND, router) + .overrideRuntimeConfigKey(MessageStorageProfile.ROUTED_BACKEND, routed) .assertException(failure -> assertThat(failure) - .hasStackTraceContaining("Unsupported outbound SMS storage profile '" + + .hasStackTraceContaining("Unsupported outbound message storage profile '" + pending + "/" + router + "/" + routed + "'") .hasStackTraceContaining("Supported profiles: memory/memory/memory")) .assertLogRecords(records -> assertThat(records).isEmpty()); diff --git a/sendium-app/src/test/java/gr/cytech/sendium/app/storage/SmsStorageMemoryStartupTest.java b/sendium-app/src/test/java/gr/cytech/sendium/app/storage/SmsStorageMemoryStartupTest.java deleted file mode 100644 index 40682c47..00000000 --- a/sendium-app/src/test/java/gr/cytech/sendium/app/storage/SmsStorageMemoryStartupTest.java +++ /dev/null @@ -1,33 +0,0 @@ -package gr.cytech.sendium.app.storage; - -import io.quarkus.test.QuarkusExtensionTest; -import jakarta.inject.Inject; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.RegisterExtension; - -import static org.assertj.core.api.Assertions.assertThat; - -class SmsStorageMemoryStartupTest { - @RegisterExtension - static final QuarkusExtensionTest APP = SmsStorageStartupSupport.application() - .overrideRuntimeConfigKey(SmsStorageProfile.PENDING_BACKEND, "memory") - .overrideRuntimeConfigKey(SmsStorageProfile.ROUTER_BACKEND, "memory") - .overrideRuntimeConfigKey(SmsStorageProfile.ROUTED_BACKEND, "memory") - .assertLogRecords(records -> assertThat(records).extracting(SmsStorageStartupSupport::message) - .containsExactly( - "Outbound SMS storage profile (pending/router-queue/routed-work): memory/memory/memory", - "NON-DURABLE outbound SMS storage: accepted messages and in-flight work can be lost on process restart.", - SmsStorageStartupSupport.ADMISSION_STARTED)); - - @Inject - SmsStorageProfile profile; - - @Inject - SmsAdmissionProbe admission; - - @Test - void validatedProfileIsAvailableBeforeAdmissionStartup() { - assertThat(profile).isEqualTo(new SmsStorageProfile("memory", "memory", "memory")); - assertThat(admission.profileAtStartup()).isSameAs(profile); - } -} diff --git a/sendium-app/src/test/java/gr/cytech/sendium/app/storage/StandaloneMessageSnapshotTest.java b/sendium-app/src/test/java/gr/cytech/sendium/app/storage/StandaloneMessageSnapshotTest.java new file mode 100644 index 00000000..89349250 --- /dev/null +++ b/sendium-app/src/test/java/gr/cytech/sendium/app/storage/StandaloneMessageSnapshotTest.java @@ -0,0 +1,53 @@ +package gr.cytech.sendium.app.storage; + +import gr.cytech.sendium.core.message.StandardMessage; +import org.junit.jupiter.api.Test; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +class StandaloneMessageSnapshotTest { + @Test + void snapshotDetachesProtocolCollectionsAndMutableAttributeValues() { + var message = new StandardMessage(); + var bytes = new byte[]{1, 2}; + var nested = new ArrayList<>(List.of("original")); + message.attrs = new HashMap<>(); + message.attrs.put("bytes", bytes); + message.attrs.put("nested", nested); + message.field1 = new ArrayList<>(List.of("field")); + message.tlvs = new HashMap<>(); + message.tlvs.put("1400", "original"); + message.reassembledParts = new ArrayList<>(List.of("part")); + var snapshot = StandaloneMessageSnapshot.copy(message); + + bytes[0] = 9; + nested.clear(); + ((List) message.field1).clear(); + message.tlvs.clear(); + message.reassembledParts.clear(); + + assertThat((byte[]) snapshot.attrs.get("bytes")).containsExactly((byte) 1, (byte) 2); + assertThat((List) snapshot.attrs.get("nested")).singleElement().isEqualTo("original"); + assertThat((List) snapshot.field1).singleElement().isEqualTo("field"); + assertThat(snapshot.tlvs).containsEntry("1400", "original"); + assertThat(snapshot.reassembledParts).containsExactly("part"); + } + + @Test + void unsupportedMutableFieldsAndCustomMessageTypesRequireExplicitApplicationAssembly() { + var message = new StandardMessage(); + message.field1 = new StringBuilder("mutable"); + assertThatThrownBy(() -> StandaloneMessageSnapshot.copy(message)) + .isInstanceOf(IllegalArgumentException.class).hasMessageContaining("Unsupported mutable"); + assertThatThrownBy(() -> StandaloneMessageSnapshot.copy(new CustomMessage())) + .isInstanceOf(IllegalArgumentException.class).hasMessageContaining("application-owned"); + } + + static class CustomMessage extends StandardMessage { + } +} diff --git a/sendium-app/src/test/java/gr/cytech/sendium/app/storage/StandaloneOutboundConfigurationTest.java b/sendium-app/src/test/java/gr/cytech/sendium/app/storage/StandaloneOutboundConfigurationTest.java new file mode 100644 index 00000000..398da77f --- /dev/null +++ b/sendium-app/src/test/java/gr/cytech/sendium/app/storage/StandaloneOutboundConfigurationTest.java @@ -0,0 +1,46 @@ +package gr.cytech.sendium.app.storage; + +import gr.cytech.sendium.core.AbstractOutWorker; +import gr.cytech.sendium.core.message.StandardMessage; +import gr.cytech.sendium.core.storage.OutboundStorageException; +import gr.cytech.sendium.routing.RoutingLookupResult; +import gr.cytech.sendium.routing.StandardRoutingManager; +import org.eclipse.microprofile.config.Config; +import org.junit.jupiter.api.Test; + +import java.util.List; +import java.util.Optional; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +class StandaloneOutboundConfigurationTest { + @Test + void invalidMemoryBoundsFailBeforeReturningAnAdmissionCoordinator() { + for (String setting : List.of("pending.capacity", "router-queue.capacity", "selection-batch-size")) { + var config = mock(Config.class); + when(config.getOptionalValue(anyString(), eq(Integer.class))).thenReturn(Optional.empty()); + String key = "sendium.message." + setting; + when(config.getOptionalValue(key, Integer.class)).thenReturn(Optional.of(0)); + assertThatThrownBy(() -> StandaloneOutboundPipeline.coordinator( + new MessageStorageProfile("memory", "memory", "memory"), config)) + .isInstanceOf(IllegalArgumentException.class).hasMessageContaining(key).hasMessageContaining("positive"); + } + } + + @Test + void unsupportedLegacyDestinationIsRejectedRatherThanMistakenForTerminalExecution() throws Exception { + var routing = mock(StandardRoutingManager.class); + var unsupported = mock(AbstractOutWorker.class); + when(routing.lookupForLifecycle(any())).thenReturn(new RoutingLookupResult(List.of(unsupported), true)); + var boundary = new StandaloneOutboundPipeline.LifecycleRouting(routing); + assertThatThrownBy(() -> boundary.lookupForLifecycle(new StandardMessage())) + .isInstanceOfSatisfying(OutboundStorageException.class, failure -> + assertThat(failure.reason()).isEqualTo(OutboundStorageException.Reason.UNSUPPORTED)); + } +} diff --git a/sendium-app/src/test/java/gr/cytech/sendium/app/storage/StandaloneOutboundPipelineTest.java b/sendium-app/src/test/java/gr/cytech/sendium/app/storage/StandaloneOutboundPipelineTest.java new file mode 100644 index 00000000..8d913a8f --- /dev/null +++ b/sendium-app/src/test/java/gr/cytech/sendium/app/storage/StandaloneOutboundPipelineTest.java @@ -0,0 +1,154 @@ +package gr.cytech.sendium.app.storage; + +import gr.cytech.sendium.auth.CredentialFileWatcher; +import gr.cytech.sendium.conf.SendiumConfigurationHandler; +import gr.cytech.sendium.core.AbstractOutWorker; +import gr.cytech.sendium.core.http.KannelResource; +import gr.cytech.sendium.core.message.StandardMessage; +import gr.cytech.sendium.core.outbound.OutboundCoordinator; +import gr.cytech.sendium.core.queue.InMemoryQueueProvider; +import gr.cytech.sendium.core.smpp.client.CoordinatedSmppSubmission; +import gr.cytech.sendium.core.smpp.client.SmppClientWorker; +import gr.cytech.sendium.routing.OutgoingWorkerManager; +import gr.cytech.sendium.routing.RoutingLookupResult; +import gr.cytech.sendium.routing.StandardRoutingManager; +import io.quarkus.test.QuarkusExtensionTest; +import io.quarkus.test.common.http.TestHTTPResource; +import jakarta.enterprise.inject.Produces; +import jakarta.inject.Inject; +import jakarta.inject.Singleton; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.RegisterExtension; + +import java.net.URI; +import java.net.http.HttpClient; +import java.net.http.HttpRequest; +import java.net.http.HttpResponse; +import java.time.Duration; +import java.util.Collection; +import java.util.List; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +class StandaloneOutboundPipelineTest { + @RegisterExtension + static final QuarkusExtensionTest APP = new QuarkusExtensionTest() + .withApplicationRoot(archive -> archive.addClasses(StandaloneOutboundPipeline.class, + StandaloneOutboundPipeline.LifecycleRouting.class, + StandaloneMessageSnapshot.class, StandaloneMessageStorage.class, MessageStorageProfile.class, + KannelResource.class, Beans.class, Providers.class, MessageLifecycleHttpTest.Credentials.class)) + .overrideConfigKey("quarkus.arc.exclude-dependency.core.group-id", "gr.cytech") + .overrideConfigKey("quarkus.arc.exclude-dependency.core.artifact-id", "sendium-core") + .overrideConfigKey("sendium.dlr.persistence.enabled", "false") + .overrideConfigKey("sendium.message.pending.capacity", "1") + .overrideConfigKey("quarkus.http.test-port", "0"); + + @TestHTTPResource("/sendsms") URI endpoint; + @Inject Providers providers; + @Inject InMemoryQueueProvider legacy; + @Inject OutboundCoordinator coordinator; + + @Test + void productionAssemblyProcessesHttpThroughProviderCompletionWithoutLegacyAdmission() throws Exception { + URI uri = URI.create(endpoint + "?username=http-user&password=secret&from=Sender&to=306900000001&text=Hello"); + try (HttpClient client = HttpClient.newHttpClient()) { + var request = HttpRequest.newBuilder(uri).timeout(Duration.ofSeconds(5)).GET().build(); + var response = client.send(request, HttpResponse.BodyHandlers.ofString()); + assertThat(response.statusCode()).isEqualTo(202); + assertThat(providers.submitted.await(5, TimeUnit.SECONDS)).isTrue(); + assertThat(providers.message.serial).isEqualTo(response.body()); + assertThat(providers.message.body).isEqualTo("Hello"); + assertThat(legacy.getRouterQueue().isEmpty()).isTrue(); + assertThat(client.send(request, HttpResponse.BodyHandlers.ofString()).statusCode()).isEqualTo(503); + assertThat(coordinator.selectAndStage(1)).isZero(); + + providers.handoff.complete(null); + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(5); + int status; + do { + status = client.send(request, HttpResponse.BodyHandlers.ofString()).statusCode(); + if (status == 503) { + TimeUnit.MILLISECONDS.sleep(20); + } + } while (status == 503 && System.nanoTime() < deadline); + assertThat(status).isEqualTo(202); + while (providers.calls.get() < 2 && System.nanoTime() < deadline) { + TimeUnit.MILLISECONDS.sleep(20); + } + assertThat(providers.calls.get()).isEqualTo(2); + assertThat(legacy.getRouterQueue().isEmpty()).isTrue(); + } finally { + providers.handoff.complete(null); + } + } + + @Singleton + public static class Providers implements OutgoingWorkerManager { + final CompletableFuture handoff = new CompletableFuture<>(); + final CountDownLatch submitted = new CountDownLatch(1); + final AtomicInteger calls = new AtomicInteger(); + volatile StandardMessage message; + final SmppClientWorker worker; + + @SuppressWarnings("unchecked") + public Providers() throws Exception { + worker = mock(SmppClientWorker.class); + when(worker.getFullName()).thenReturn("smppclient.provider"); + when(worker.isKeepOnRunning()).thenReturn(true); + when(worker.acceptsMessages()).thenReturn(true); + when(worker.getThreadCount()).thenReturn(1); + when(worker.submitPreparedCoordinated(any(), any())).thenAnswer(invocation -> { + message = invocation.getArgument(0); + calls.incrementAndGet(); + var submission = mock(CoordinatedSmppSubmission.class); + when(submission.completion()).thenReturn(handoff); + submitted.countDown(); + return submission; + }); + } + + @Override + public boolean stop() { + return true; + } + + @Override + public Collection getWorkersCopy() { + return List.of(worker); + } + } + + @Singleton + public static class Beans { + @Produces @Singleton + StandardRoutingManager routing(Providers providers) throws Exception { + var routing = mock(StandardRoutingManager.class); + when(routing.lookupForLifecycle(any())).thenReturn(new RoutingLookupResult(List.of(providers.worker), true)); + return routing; + } + + @Produces @Singleton + CredentialFileWatcher credentials() { + return new MessageLifecycleHttpTest.Credentials(); + } + + @Produces @Singleton + SendiumConfigurationHandler configuration() { + var configuration = new SendiumConfigurationHandler(); + configuration.init(); + return configuration; + } + + @Produces @Singleton + InMemoryQueueProvider legacyQueues() { + return new InMemoryQueueProvider(); + } + } +} diff --git a/sendium-app/src/test/java/gr/cytech/sendium/app/storage/StandaloneOutboundShutdownTest.java b/sendium-app/src/test/java/gr/cytech/sendium/app/storage/StandaloneOutboundShutdownTest.java new file mode 100644 index 00000000..d0f34770 --- /dev/null +++ b/sendium-app/src/test/java/gr/cytech/sendium/app/storage/StandaloneOutboundShutdownTest.java @@ -0,0 +1,91 @@ +package gr.cytech.sendium.app.storage; + +import gr.cytech.sendium.conf.SendiumConfigurationHandler; +import gr.cytech.sendium.core.message.StandardMessage; +import gr.cytech.sendium.core.outbound.OutboundWork.SourceId; +import gr.cytech.sendium.core.storage.OutboundStorageException; +import gr.cytech.sendium.routing.OutgoingWorkerManager; +import gr.cytech.sendium.routing.RoutingLookupResult; +import gr.cytech.sendium.routing.StandardRoutingManager; +import org.eclipse.microprofile.config.Config; +import org.junit.jupiter.api.Test; + +import java.util.List; +import java.util.Optional; +import java.util.UUID; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.inOrder; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +class StandaloneOutboundShutdownTest { + @Test + void standaloneShutdownDrainsProvidersAndRefusesStageClosureUntilWorkersStop() throws Exception { + var config = mock(Config.class); + when(config.getOptionalValue(anyString(), eq(Integer.class))).thenReturn(Optional.empty()); + var coordinator = spy(StandaloneOutboundPipeline.coordinator(new MessageStorageProfile("memory", "memory", "memory"), config)); + var quiesced = new CountDownLatch(1); + doAnswer(invocation -> { + invocation.callRealMethod(); + quiesced.countDown(); + return null; + }).when(coordinator).quiesce(); + var provider = new StandaloneOutboundPipelineTest.Providers(); + var routing = mock(StandardRoutingManager.class); + when(routing.lookupForLifecycle(any())).thenReturn(new RoutingLookupResult(List.of(provider.worker), true)); + var workers = mock(OutgoingWorkerManager.class); + when(workers.getWorkersCopy()).thenReturn(List.of(provider.worker)); + when(workers.stop()).thenReturn(false, true); + var pipeline = new StandaloneOutboundPipeline(); + pipeline.coordinator = coordinator; + pipeline.routing = routing; + pipeline.workers = workers; + pipeline.config = config; + pipeline.configuration = mock(SendiumConfigurationHandler.class); + pipeline.start(null); + + var message = new StandardMessage(); + message.body = "hello"; + coordinator.admit(new SourceId(UUID.randomUUID()), message); + try (var stopping = Executors.newSingleThreadExecutor()) { + try { + assertThat(provider.submitted.await(5, TimeUnit.SECONDS)).isTrue(); + var stopped = stopping.submit(() -> { + pipeline.stop(null); + return null; + }); + assertThat(quiesced.await(5, TimeUnit.SECONDS)).isTrue(); + assertThat(stopped.isDone()).isFalse(); + verify(workers, never()).stop(); + verify(coordinator, never()).close(); + assertThatThrownBy(() -> coordinator.admit(new SourceId(UUID.randomUUID()), message)) + .isInstanceOf(OutboundStorageException.class); + provider.handoff.complete(null); + assertThatThrownBy(() -> stopped.get(5, TimeUnit.SECONDS)).hasCauseInstanceOf(IllegalStateException.class); + verify(coordinator, never()).close(); + pipeline.stop(null); + var order = inOrder(coordinator, routing, workers); + order.verify(coordinator).quiesce(); + order.verify(routing).stop(); + order.verify(workers).stop(); + order.verify(coordinator).close(); + } finally { + provider.handoff.complete(null); + } + } finally { + pipeline.stop(null); + } + } +} diff --git a/sendium-core/src/main/java/gr/cytech/sendium/routing/StandardOutboundDispatch.java b/sendium-core/src/main/java/gr/cytech/sendium/routing/StandardOutboundDispatch.java index 3ccace3f..57a63804 100644 --- a/sendium-core/src/main/java/gr/cytech/sendium/routing/StandardOutboundDispatch.java +++ b/sendium-core/src/main/java/gr/cytech/sendium/routing/StandardOutboundDispatch.java @@ -5,11 +5,14 @@ import gr.cytech.sendium.core.outbound.OutboundWork.Destination; import gr.cytech.sendium.core.outbound.OutboundWork.Routed; import gr.cytech.sendium.core.outbound.OutboundWork.Selected; +import gr.cytech.sendium.core.outbound.OutboundWork.SelectionId; import gr.cytech.sendium.core.outbound.OutboundWork.WorkId; import gr.cytech.sendium.core.smpp.client.CoordinatedSmppSubmission; import gr.cytech.sendium.core.smpp.client.SmppClientWorker; import gr.cytech.sendium.core.storage.OutboundStage; import gr.cytech.sendium.core.storage.OutboundStorageException; +import gr.cytech.sendium.external.filter.FilterException; +import gr.cytech.sendium.external.filter.FilterStatusCodes; import java.io.IOException; import java.time.Duration; @@ -36,6 +39,7 @@ public final class StandardOutboundDispatch implements AutoCloseable { private final ConcurrentMap, Semaphore> workerSlots = new ConcurrentHashMap<>(); private final ConcurrentMap> activeExecutions = new ConcurrentHashMap<>(); private boolean quiescing; + private final ConcurrentMap routingTransitions = new ConcurrentHashMap<>(); public StandardOutboundDispatch(OutboundCoordinator coordinator, StandardRoutingManager routing) { this(coordinator, routing, Executors.newScheduledThreadPool(2, task -> { @@ -65,6 +69,13 @@ private StandardOutboundDispatch(OutboundCoordinator coordinato */ public synchronized Optional> routeSelected(Selected selected) throws IOException { Objects.requireNonNull(selected, "selected"); + var pending = routingTransitions.get(selected.id()); + if (pending != null) { + if (pending.selected() != selected) { + throw new IllegalArgumentException("Routing retry requires its original selection handle"); + } + return Optional.of(recordRouting(pending)); + } if (quiescing) { coordinator.returnToRouter(selected); return Optional.empty(); @@ -72,6 +83,16 @@ public synchronized Optional> routeSelected(Selected> routeSelected(Selected(destination, selected.message()))); + var transition = new PendingRouting(selected, new Destination<>(destination, selected.message())); + routingTransitions.putIfAbsent(selected.id(), transition); + var recorded = recordRouting(routingTransitions.get(selected.id())); + return Optional.of(recorded); + } + + /** Retries recording the original routing intent without rerunning lookup or filters. */ + public synchronized void retryRoutingTransitions() { + RuntimeException failure = null; + for (var transition : List.copyOf(routingTransitions.values())) { + try { + recordRouting(transition); + } catch (RuntimeException error) { + if (failure == null) { + failure = error; + } else if (failure != error) { + failure.addSuppressed(error); + } + } + } + if (failure != null) { + throw failure; + } + } + + private Routed recordRouting(PendingRouting transition) { + var recorded = coordinator.route(transition.selected(), transition.destination()); + routingTransitions.remove(transition.selected().id(), transition); + return recorded; } /** Takes recorded work for provider processing; taking or enqueuing it is not terminal completion. */ @@ -194,6 +243,9 @@ public synchronized void close() { if (!activeExecutions.isEmpty()) { throw new IllegalStateException("Drain provider executions before closing outbound dispatch"); } + if (!routingTransitions.isEmpty()) { + throw new IllegalStateException("Resolve unfinished routing transitions before closing outbound dispatch"); + } if (ownsScheduler) { scheduler.shutdown(); } @@ -216,6 +268,9 @@ public enum AttemptState { QUEUED, PREPARING, AWAITING_COMPLETION, COMPLETED, FAILED } + private record PendingRouting(Selected selected, Destination destination) { + } + public record AttemptSnapshot(String destination, AttemptState state, Throwable failure) { } diff --git a/sendium-core/src/test/java/gr/cytech/sendium/routing/StandardOutboundDispatchTest.java b/sendium-core/src/test/java/gr/cytech/sendium/routing/StandardOutboundDispatchTest.java index 41692f26..de1ceda8 100644 --- a/sendium-core/src/test/java/gr/cytech/sendium/routing/StandardOutboundDispatchTest.java +++ b/sendium-core/src/test/java/gr/cytech/sendium/routing/StandardOutboundDispatchTest.java @@ -5,6 +5,8 @@ import gr.cytech.sendium.core.outbound.OutboundWork.Routed; import gr.cytech.sendium.core.outbound.OutboundWork.SourceId; import gr.cytech.sendium.core.storage.OutboundStorageException; +import gr.cytech.sendium.external.filter.FilterException; +import gr.cytech.sendium.external.filter.FilterStatusCodes; import org.junit.jupiter.api.Test; import utils.OutboundIngressFixture; @@ -17,10 +19,52 @@ import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; class StandardOutboundDispatchTest { + @Test + void interruptedDestinationRecordingRetriesTheOriginalIntentWithoutRepeatingLookup() throws Exception { + try (var fixture = new OutboundIngressFixture(1)) { + var provider = worker("smpp.provider", "provider"); + var routing = mock(StandardRoutingManager.class); + when(routing.lookupForLifecycle(any())).thenReturn(new RoutingLookupResult(List.of(provider), true)); + var dispatch = new StandardOutboundDispatch(fixture.coordinator, routing); + fixture.coordinator.admit(new SourceId(UUID.randomUUID()), message("hello")); + fixture.coordinator.selectAndStage(1); + var selected = fixture.coordinator.takeForRouting(Duration.ZERO).orElseThrow(); + doThrow(new IllegalStateException("recording interrupted")).doCallRealMethod() + .when(fixture.coordinator).route(any(), any()); + assertThatThrownBy(() -> dispatch.routeSelected(selected)).isInstanceOf(IllegalStateException.class); + dispatch.retryRoutingTransitions(); + verify(routing, times(1)).lookupForLifecycle(any()); + assertThat(dispatch.takeForProvider("smpp.provider", Duration.ZERO)).isPresent(); + dispatch.close(); + } + } + + @Test + void routingFilterDropCompletesTheSourceWithoutProviderDispatch() throws Exception { + try (var fixture = new OutboundIngressFixture(1)) { + var routing = mock(StandardRoutingManager.class); + var dispatch = new StandardOutboundDispatch(fixture.coordinator, routing); + SourceId source = new SourceId(UUID.randomUUID()); + fixture.coordinator.admit(source, message("hello")); + fixture.coordinator.selectAndStage(1); + var selected = fixture.coordinator.takeForRouting(Duration.ZERO).orElseThrow(); + when(routing.lookupForLifecycle(any())).thenThrow(new FilterException(null, + FilterStatusCodes.DROP, selected.message(), "dropped")); + assertThat(dispatch.routeSelected(selected)).isEmpty(); + assertThat(fixture.pending.find(source)).isEmpty(); + assertThat(dispatch.takeForProvider("smpp.provider", Duration.ZERO)).isEmpty(); + dispatch.close(); + } + } + @Test void shutdownDrainsAllHandoffsAndRetainsOwnershipOnTimeoutOrFailure() throws Exception { try (var fixture = new OutboundIngressFixture(1)) { From e74ddaba09ba1058e81e397693e437dee3a53b21 Mon Sep 17 00:00:00 2001 From: pavlos Date: Tue, 6 Oct 2026 12:38:11 +0300 Subject: [PATCH 12/20] test(storage): verify combined outbound and embedded lifecycles Signed-off-by: pavlos --- docs/14-outbound-storage.md | 18 ++ .../OutboundEmbeddedLifecycleTest.java | 193 ++++++++++++++++++ .../smpp/client/SmppClientWorkerTest.java | 91 +++++++++ 3 files changed, 302 insertions(+) create mode 100644 sendium-core/src/test/java/gr/cytech/sendium/core/outbound/consumer/OutboundEmbeddedLifecycleTest.java diff --git a/docs/14-outbound-storage.md b/docs/14-outbound-storage.md index 9909c5f4..00c1e36e 100644 --- a/docs/14-outbound-storage.md +++ b/docs/14-outbound-storage.md @@ -654,3 +654,21 @@ and a `NON-DURABLE` warning. The library memory stores are not yet constructed b This milestone includes actionable errors and startup profile/non-durable logging. Metrics, readiness endpoints, and broader observability are deferred. Contract-level status is not a health endpoint. + +## Behavioral verification + +The memory slice is verified at the following connected boundaries: + +| Boundary | Coverage | +|---|---| +| HTTP admission through standalone processing and capacity recovery | `MessageLifecycleHttpTest` and `StandaloneOutboundPipelineTest` exercise real HTTP/CDI admission and memory stores; the latter runs the production processors against a controlled provider handoff. | +| SMPP ownership before acknowledgement and multipart publication | `SmppLifecycleAdmissionTest` exercises submit/response objects, held sources, reassembly and publication retry without another acknowledgement. | +| Combined multipart lifecycle and shutdown drain | `SmppClientWorkerTest` carries two held sources through copied-route rejection, real routing lookup, request generation, rerouting to another provider during quiescence, failed handoffs and handoff-only retry. Original sources remain owned until completion; handoff retries and duplicate callbacks create no additional provider requests. | +| Standalone shutdown ordering and worker-stop failure | `StandaloneOutboundShutdownTest` proves workers remain available during handoff drain and stage closure waits for successful worker-manager stop. | +| Application-owned generic assembly | `OutboundEmbeddedLifecycleTest`, outside the implementation package, supplies a selection-store implementation with its own bound/backend identity and a custom mutable message subtype. Real stores/coordinator preserve subtype fields through selection, returns, failed asynchronous handoff, completion retry and explicit start/quiesce/close. | +| Routing transitions and copied-route guards | `StandardOutboundDispatchTest` verifies recording retries preserve the original intent without lookup replay, routing drops complete sources and copied rules fail before assignment. | +| Startup profiles and memory bounds | `MessageStorageProfileTest`, `MessageStorage*StartupTest` and `StandaloneOutboundConfigurationTest` cover supported defaults, unsupported requests, early rejection and positive bounds. | + +Provider tests exercise real SMPP request generation and session callbacks with controlled transport/handoff +fixtures rather than an external provider socket. Embedded fixtures verify the shared contracts within +Sendium; deployment or adoption in another application remains that application's responsibility. diff --git a/sendium-core/src/test/java/gr/cytech/sendium/core/outbound/consumer/OutboundEmbeddedLifecycleTest.java b/sendium-core/src/test/java/gr/cytech/sendium/core/outbound/consumer/OutboundEmbeddedLifecycleTest.java new file mode 100644 index 00000000..b65bb63b --- /dev/null +++ b/sendium-core/src/test/java/gr/cytech/sendium/core/outbound/consumer/OutboundEmbeddedLifecycleTest.java @@ -0,0 +1,193 @@ +package gr.cytech.sendium.core.outbound.consumer; + +import gr.cytech.sendium.core.message.StandardMessage; +import gr.cytech.sendium.core.outbound.DefaultOutboundCoordinator; +import gr.cytech.sendium.core.outbound.OutboundWork.Destination; +import gr.cytech.sendium.core.outbound.OutboundWork.Selected; +import gr.cytech.sendium.core.outbound.OutboundWork.SelectionId; +import gr.cytech.sendium.core.outbound.OutboundWork.SourceId; +import gr.cytech.sendium.core.storage.OutboundStage; +import gr.cytech.sendium.core.storage.OutboundStorageException; +import gr.cytech.sendium.core.storage.SelectedRouterStore; +import gr.cytech.sendium.core.storage.memory.MemoryPendingMessageStore; +import gr.cytech.sendium.core.storage.memory.MemoryRoutedWorkStore; +import gr.cytech.sendium.core.storage.memory.MemorySelectedRouterStore; +import org.junit.jupiter.api.Test; + +import java.time.Duration; +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; +import java.util.UUID; +import java.util.concurrent.CompletableFuture; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.verify; + +class OutboundEmbeddedLifecycleTest { + @Test + void customMessagesSurviveApplicationSelectionPolicyReturnsAndAsynchronousHandoffRetry() throws Exception { + var pending = new MemoryPendingMessageStore(2, OutboundEmbeddedLifecycleTest::snapshot); + var selected = new ApplicationSelectedStore(new MemorySelectedRouterStore<>(pending, 2)); + var routed = new MemoryRoutedWorkStore(2, OutboundEmbeddedLifecycleTest::snapshot); + try (var coordinator = new DefaultOutboundCoordinator<>(pending, selected, routed)) { + assertThat(selected.status().state()).isEqualTo(OutboundStage.State.NEW); + coordinator.start(); + assertThat(selected.status().backend()).isEqualTo("application-selected"); + var firstSource = source(); + var secondSource = source(); + var first = message("custom-1"); + coordinator.admit(firstSource, first); + coordinator.admit(secondSource, message("custom-2")); + first.labels.add("caller mutation"); + + // The application-owned port narrows the requested batch without a core configuration or CDI container. + assertThat(coordinator.selectAndStage(10)).isEqualTo(1); + assertThat(selected.requestedLimit).isEqualTo(10); + var selection = coordinator.takeForRouting(Duration.ZERO).orElseThrow(); + assertThat(selection.message().externalId).isEqualTo("custom-1"); + assertThat(selection.message().labels).containsExactly("accepted"); + coordinator.route(selection, new Destination<>("application-provider", selection.message())); + var work = coordinator.takeForDestination("application-provider", Duration.ZERO).orElseThrow(); + work.message().labels.add("worker mutation"); + coordinator.returnToDestination(work); + var returned = coordinator.takeForDestination("application-provider", Duration.ZERO).orElseThrow(); + assertThat(returned.id()).isEqualTo(work.id()); + assertThat(returned.message().externalId).isEqualTo("custom-1"); + assertThat(returned.message().labels).containsExactly("accepted", "worker mutation"); + assertThat(pending.find(firstSource).orElseThrow().labels).containsExactly("accepted"); + + var handoff = new CompletableFuture(); + var completion = coordinator.complete(returned.id(), handoff); + assertThat(completion.toCompletableFuture()).isNotDone(); + handoff.completeExceptionally(new IllegalStateException("application handoff failed")); + assertThatThrownBy(() -> completion.toCompletableFuture().join()).hasCauseInstanceOf(IllegalStateException.class); + assertThat(pending.find(firstSource)).isPresent(); + assertThat(coordinator.takeForDestination("application-provider", Duration.ZERO)).isEmpty(); + coordinator.complete(returned.id(), CompletableFuture.completedStage(null)).toCompletableFuture().join(); + assertThat(pending.find(firstSource)).isEmpty(); + assertThat(pending.find(secondSource).orElseThrow().externalId).isEqualTo("custom-2"); + + // A late terminal callback cannot remove the other accepted source. + coordinator.complete(returned.id(), CompletableFuture.completedStage(null)).toCompletableFuture().join(); + assertThat(pending.find(secondSource)).isPresent(); + assertThat(coordinator.selectAndStage(10)).isEqualTo(1); + var second = coordinator.takeForRouting(Duration.ZERO).orElseThrow(); + assertThat(second.message().externalId).isEqualTo("custom-2"); + coordinator.discard(second.id(), CompletableFuture.completedStage(null)).toCompletableFuture().join(); + } + assertThat(selected.status().state()).isEqualTo(OutboundStage.State.CLOSED); + } + + @Test + void applicationOwnedShutdownReturnsTakenCustomWorkWithoutDeclaringItCompleted() throws Exception { + var pending = spy(new MemoryPendingMessageStore(2, OutboundEmbeddedLifecycleTest::snapshot)); + var router = spy(new MemorySelectedRouterStore<>(pending, 2)); + var selected = new ApplicationSelectedStore(router); + var routed = spy(new MemoryRoutedWorkStore(2, OutboundEmbeddedLifecycleTest::snapshot)); + try (var coordinator = new DefaultOutboundCoordinator<>(pending, selected, routed)) { + assertThatThrownBy(() -> coordinator.admit(source(), message("before-start"))) + .isInstanceOf(OutboundStorageException.class); + coordinator.start(); + coordinator.admit(source(), message("routed")); + coordinator.selectAndStage(1); + var initial = coordinator.takeForRouting(Duration.ZERO).orElseThrow(); + coordinator.route(initial, new Destination<>("custom-provider", initial.message())); + var destinationTake = coordinator.takeForDestination("custom-provider", Duration.ZERO).orElseThrow(); + coordinator.admit(source(), message("unassigned")); + coordinator.selectAndStage(1); + var routerTake = coordinator.takeForRouting(Duration.ZERO).orElseThrow(); + + coordinator.quiesce(); + assertThatThrownBy(() -> coordinator.admit(source(), message("after-quiesce"))) + .isInstanceOf(OutboundStorageException.class); + coordinator.close(); + verify(router).release(routerTake); + verify(routed).release(destinationTake); + verify(pending, never()).complete(any()); + assertThat(pending.status().state()).isEqualTo(OutboundStage.State.CLOSED); + assertThat(routed.status().state()).isEqualTo(OutboundStage.State.CLOSED); + } + } + + private static SourceId source() { + return new SourceId(UUID.randomUUID()); + } + + private static CustomMessage message(String externalId) { + var message = new CustomMessage(); + message.externalId = externalId; + message.labels.add("accepted"); + return message; + } + + private static CustomMessage snapshot(CustomMessage message) { + try { + var copy = (CustomMessage) message.clone(); + copy.labels = new ArrayList<>(message.labels); + return copy; + } catch (CloneNotSupportedException failure) { + throw new AssertionError(failure); + } + } + + private static final class CustomMessage extends StandardMessage { + private String externalId; + private List labels = new ArrayList<>(); + } + + /** Representative application implementation with its own selection bound and backend identity. */ + private static final class ApplicationSelectedStore implements SelectedRouterStore { + private final SelectedRouterStore delegate; + private int requestedLimit; + + private ApplicationSelectedStore(SelectedRouterStore delegate) { + this.delegate = delegate; + } + + @Override + public int selectAndStage(int limit) { + requestedLimit = limit; + return delegate.selectAndStage(Math.min(limit, 1)); + } + + @Override + public Optional> take(Duration timeout) throws InterruptedException { + return delegate.take(timeout); + } + + @Override + public void release(Selected selected) { + delegate.release(selected); + } + + @Override + public void markRouted(SelectionId selection) { + delegate.markRouted(selection); + } + + @Override + public void complete(SelectionId selection) { + delegate.complete(selection); + } + + @Override + public void open() { + delegate.open(); + } + + @Override + public Status status() { + return new Status("application-selected", false, delegate.status().state()); + } + + @Override + public void close() { + delegate.close(); + } + } +} diff --git a/sendium-core/src/test/java/gr/cytech/sendium/core/smpp/client/SmppClientWorkerTest.java b/sendium-core/src/test/java/gr/cytech/sendium/core/smpp/client/SmppClientWorkerTest.java index ecd0a353..6df0a616 100644 --- a/sendium-core/src/test/java/gr/cytech/sendium/core/smpp/client/SmppClientWorkerTest.java +++ b/sendium-core/src/test/java/gr/cytech/sendium/core/smpp/client/SmppClientWorkerTest.java @@ -10,10 +10,12 @@ import com.cloudhopper.smpp.type.Address; import gr.cytech.sendium.conf.PropertyChangeListener; import gr.cytech.sendium.conf.SendiumConfigurationProvider; +import gr.cytech.sendium.core.AbstractOutWorker; import gr.cytech.sendium.core.message.StandardMessage; import gr.cytech.sendium.core.queue.Queue; import gr.cytech.sendium.core.outbound.OutboundWork.Destination; import gr.cytech.sendium.core.outbound.OutboundWork.SourceId; +import gr.cytech.sendium.core.storage.OutboundStorageException; import gr.cytech.sendium.core.worker.DlrStorageException; import gr.cytech.sendium.core.worker.ForwardMoService; import gr.cytech.sendium.core.worker.Tracker; @@ -21,6 +23,7 @@ import gr.cytech.sendium.routing.StandardOutboundDispatch; import gr.cytech.sendium.routing.StandardRoutingManager; import gr.cytech.sendium.routing.RoutingLookupResult; +import gr.cytech.sendium.routing.RoutingFileParser; import org.junit.jupiter.api.Test; import utils.OutboundIngressFixture; @@ -43,6 +46,82 @@ class SmppClientWorkerTest { + @Test + void heldSourcesCopiedRouteRejectionMultipartReroutingAndHandoffRetryRemainOwnedThroughShutdown() throws Exception { + try (var fixture = new OutboundIngressFixture(2)) { + var first = new TestSmppClientWorker(new TestConfigurationProvider(Map.of( + "status.retry.router", Integer.toString(SmppConstants.STATUS_INVDSTADR))), new Queue<>(), new CapturingTracker()); + var second = spy(new TestSmppClientWorker(new TestConfigurationProvider(), new Queue<>(), new CapturingTracker())); + when(second.getFullName()).thenReturn("smppclient.second"); + var routing = new LifecycleRouting(java.util.List.of(first, second)); + routing.use("+" + first.getFullName() + "::default:"); + try (var dispatch = new StandardOutboundDispatch(fixture.coordinator, routing)) { + var firstSource = new SourceId(UUID.randomUUID()); + var secondSource = new SourceId(UUID.randomUUID()); + var firstPart = messageWithNetwork(); + firstPart.serial = firstSource.value().toString(); + firstPart.body = "a".repeat(100); + var secondPart = messageWithNetwork(); + secondPart.serial = secondSource.value().toString(); + secondPart.body = "a".repeat(100); + fixture.coordinator.admitHeld(firstSource, firstPart); + fixture.coordinator.admitHeld(secondSource, secondPart); + assertThat(fixture.coordinator.selectAndStage(2)).isZero(); + var prepared = messageWithNetwork(); + prepared.serial = firstPart.serial; + prepared.body = firstPart.body + secondPart.body; + prepared.reassembledParts = new java.util.ArrayList<>(java.util.List.of(firstPart.serial, secondPart.serial)); + fixture.coordinator.publishReady(Set.of(firstSource, secondSource), prepared); + assertThat(fixture.coordinator.selectAndStage(2)).isEqualTo(1); + var selected = fixture.coordinator.takeForRouting(Duration.ZERO).orElseThrow(); + assertThatThrownBy(() -> dispatch.routeSelected(selected)) + .isInstanceOfSatisfying(OutboundStorageException.class, failure -> + assertThat(failure.reason()).isEqualTo(OutboundStorageException.Reason.UNSUPPORTED)); + assertThat(first.coordinatedRequests).isEmpty(); + assertThat(dispatch.takeForProvider(first.getFullName(), Duration.ZERO)).isEmpty(); + + routing.use(first.getFullName() + "::default:"); + var returned = fixture.coordinator.takeForRouting(Duration.ZERO).orElseThrow(); + assertThat(returned.id()).isEqualTo(selected.id()); + dispatch.routeSelected(returned); + var parent = dispatch.takeForProvider(first.getFullName(), Duration.ZERO).orElseThrow(); + var execution = dispatch.submitToProvider(parent, first); + first.awaitRequests(2); + routing.use(second.getFullName() + "::default:"); + dispatch.quiesce(); + assertThat(dispatch.awaitProviderDrain(Duration.ZERO)).isFalse(); + respond(handler(first), first.coordinatedRequests.getFirst(), SmppConstants.STATUS_INVDSTADR); + second.awaitRequests(2); + assertThat(parent.destination()).isEqualTo(first.getFullName()); + assertThat(execution.attempts()).extracting(StandardOutboundDispatch.AttemptSnapshot::destination) + .containsExactly(first.getFullName(), second.getFullName()); + + second.failCoordinatedHandoff = true; + second.coordinatedRequests.forEach(request -> respond(handler(second), request, SmppConstants.STATUS_OK)); + respond(handler(first), first.coordinatedRequests.getLast(), SmppConstants.STATUS_OK); + assertThat(execution.completion().toCompletableFuture()).isNotDone(); + assertThat(dispatch.awaitProviderDrain(Duration.ZERO)).isFalse(); + assertThat(fixture.pending.find(firstSource).orElseThrow().body).isEqualTo(firstPart.body); + assertThat(fixture.pending.find(secondSource)).isPresent(); + assertThatThrownBy(dispatch::close).isInstanceOf(IllegalStateException.class); + + second.failCoordinatedHandoff = false; + execution.retryPendingHandoffs(); + execution.completion().toCompletableFuture().get(5, TimeUnit.SECONDS); + assertThat(dispatch.awaitProviderDrain(Duration.ofSeconds(1))).isTrue(); + assertThat(fixture.pending.find(firstSource)).isEmpty(); + assertThat(fixture.pending.find(secondSource)).isEmpty(); + assertThat(first.coordinatedRequests).hasSize(2); + assertThat(second.coordinatedRequests).hasSize(2); + int handoffs = second.coordinatedHandoffs; + respond(handler(second), second.coordinatedRequests.getFirst(), SmppConstants.STATUS_OK); + assertThat(second.coordinatedHandoffs).isEqualTo(handoffs); + assertThat(first.getRouterQueue().isEmpty()).isTrue(); + assertThat(second.getRouterQueue().isEmpty()).isTrue(); + } + } + } + @Test void workerSlotAndRateGatePreventConcurrentPreparationAndEarlySending() throws Exception { var scheduler = new ScheduledThreadPoolExecutor(2); @@ -733,6 +812,18 @@ private static StandardMessage messageWithNetwork() { return msg; } + private static final class LifecycleRouting extends StandardRoutingManager { + private final java.util.List providers; + + private LifecycleRouting(java.util.List providers) { + this.providers = providers; + } + + private void use(String rule) { + parseNewRoutingTable(RoutingFileParser.parseRoutingTable(java.util.List.of(rule)), providers); + } + } + private static class TestSmppClientWorker extends SmppClientWorker { private volatile boolean testPaused; private volatile int preparations; From c23607ed35c492545522e1b7ca083059efe99612 Mon Sep 17 00:00:00 2001 From: pavlos Date: Thu, 8 Oct 2026 14:39:20 +0300 Subject: [PATCH 13/20] refactor(storage): clarify outbound lifecycle API and documentation Signed-off-by: pavlos --- docs/04-smpp-configuration.md | 4 +- docs/06-http-api.md | 2 +- docs/09-configuration-reference.md | 9 +- docs/14-outbound-storage.md | 894 +++++------------- docs/DocumentationMap.md | 4 +- .../storage/StandaloneOutboundPipeline.java | 6 +- .../app/storage/MessageLifecycleHttpTest.java | 2 +- .../StandaloneOutboundPipelineTest.java | 2 +- .../StandaloneOutboundShutdownTest.java | 14 +- .../sendium/core/http/KannelResource.java | 2 +- .../outbound/DefaultOutboundCoordinator.java | 36 +- .../core/outbound/OutboundCoordinator.java | 46 +- .../core/smpp/server/SmppServerWorker.java | 6 +- .../core/storage/PendingMessageStore.java | 6 +- .../core/storage/SelectedRouterStore.java | 4 +- .../memory/MemoryPendingMessageStore.java | 2 +- .../memory/MemorySelectedRouterStore.java | 2 +- .../routing/StandardOutboundDispatch.java | 34 +- .../sendium/core/http/KannelResourceTest.java | 8 +- .../DefaultOutboundCoordinatorTest.java | 118 +-- .../OutboundEmbeddedLifecycleTest.java | 40 +- .../smpp/client/SmppClientWorkerTest.java | 62 +- .../server/SmppLifecycleAdmissionTest.java | 40 +- .../storage/memory/HeldAdmissionTest.java | 148 +-- .../storage/memory/MemoryStorageTest.java | 44 +- .../routing/StandardOutboundDispatchTest.java | 36 +- 26 files changed, 582 insertions(+), 989 deletions(-) diff --git a/docs/04-smpp-configuration.md b/docs/04-smpp-configuration.md index b7c67e65..5a3f00c1 100644 --- a/docs/04-smpp-configuration.md +++ b/docs/04-smpp-configuration.md @@ -64,14 +64,14 @@ All properties below should be prefixed with your instance path. For example, if ### Admission and acknowledgement -The standalone server uses the shared memory outbound coordinator. It admits a complete message, or retains +The standalone server uses the shared memory outbound coordinator. It accepts a complete message, or retains each concatenated part as held work, before success. Parts are acknowledged individually rather than waiting for the whole message. Completed/expired reassembly work retains those original source IDs. In that coordinated path, capacity rejection returns `STATUS_THROTTLED`, unsupported input returns `STATUS_SUBMITFAIL`, and other lifecycle/storage failures return `STATUS_SYSERR`. A failure after acknowledgement during ready publication is retried without another client response. Embedded workers -without a coordinator binding retain local ingress-queue admission before success. See [Outbound Storage Contracts](14-outbound-storage.md). +without a coordinator binding retain local ingress-queue admission before success. See [Outbound Message Storage](14-outbound-storage.md). ## 🧵 Thread Pool Configuration diff --git a/docs/06-http-api.md b/docs/06-http-api.md index 5bf07e9b..cc7036a1 100644 --- a/docs/06-http-api.md +++ b/docs/06-http-api.md @@ -90,7 +90,7 @@ For a manual installation, replace the environment variables with the HTTP `syst 123e4567-e89b-12d3-a456-426614174000 ``` -In the standalone runtime, `202 Accepted` means admission into the memory-backed pending store. The source remains retained through routing, provider processing, and required DLR handoffs. Capacity rejection returns `503`; a failing coordinator is never replaced by legacy queue admission. Embedded callers without a coordinator retain the legacy in-memory queue path. The memory profile is non-durable across restart. See [Outbound Storage Contracts](14-outbound-storage.md). +In the standalone runtime, `202 Accepted` means admission into the memory-backed pending store. The source remains retained through routing, provider processing, and required DLR handoffs. Capacity rejection returns `503`; a failing coordinator is never replaced by legacy queue admission. Embedded callers without a coordinator retain the legacy in-memory queue path. The memory profile is non-durable across restart. See [Outbound Message Storage](14-outbound-storage.md). Neither memory admission path writes an outbound message or DLR row to PostgreSQL. Acceptance does not prove that a viable route exists, that an upstream SMSC accepted the message, or that a handset received it. Durable DLR state is created only after an upstream provider outcome. Local-only Quick Start installations have no outbound route. Check routing configuration, the SMPP client connection, message lifecycle logs, submit response, and delivery receipt for those later stages. diff --git a/docs/09-configuration-reference.md b/docs/09-configuration-reference.md index 943c467e..2221cbad 100644 --- a/docs/09-configuration-reference.md +++ b/docs/09-configuration-reference.md @@ -70,10 +70,11 @@ profile is retained for the process lifetime. Startup logs the effective profile standalone pipeline binds HTTP/SMPP to one shared coordinator and executes bounded selection, routing, SMPP-client submission, retries and terminal handoffs. All capacity/batch settings must be positive. Pending capacity counts retained accepted sources (including held multipart parts); router capacity -counts queued plus taken unassigned selections. Routed capacity equals pending source capacity, so a -recorded selection cannot exhaust destination-record slots while its sources fit in pending storage. +counts queued plus taken unassigned selections. Routed capacity equals pending source capacity, so +normal source-retaining operation does not introduce a smaller destination-record bound. Partial terminal +cleanup may still occupy routed slots after sources are removed; such cleanup failures require retry. Selection and each destination take loop process at most the configured batch size per poll. -See [Outbound Storage Contracts](14-outbound-storage.md). +See [Outbound Message Storage](14-outbound-storage.md). The producer and startup observer live only in `sendium-app`. Embedding `sendium-core` does not activate this validation, configure an outbound coordinator, or enforce the standalone profile list. @@ -135,4 +136,4 @@ When the HTTP server is running, Sendium exposes: - [SMPP Configuration](04-smpp-configuration.md) - [Routing Engine](05-routing-engine.md) - [DLR Persistence](13-dlr-persistence.md) -- [Outbound Storage Contracts](14-outbound-storage.md) +- [Outbound Message Storage](14-outbound-storage.md) diff --git a/docs/14-outbound-storage.md b/docs/14-outbound-storage.md index 00c1e36e..45ca5e63 100644 --- a/docs/14-outbound-storage.md +++ b/docs/14-outbound-storage.md @@ -1,674 +1,252 @@ -# Outbound storage contracts - -This describes the initial contracts for [#338](https://github.com/cytechmobile/sendium/issues/338), -under [#337](https://github.com/cytechmobile/sendium/issues/337). The stage contracts are library APIs, -and standalone profile selection, early validation, and startup logging are implemented. All three -memory stage stores and the default coordinator are available as explicitly constructed library components. -The standalone application binds HTTP/SMPP admission to one shared memory coordinator and runs selection, -routing, SMPP-client provider execution, retries and terminal handoffs end to end. The default runtime -remains explicitly non-durable. Importing the core library alone does not activate this assembly. - -Shared storage configuration uses the channel-neutral `sendium.message.*` namespace, with -`MessageStorageProfile` and `StandaloneMessageStorage` as the standalone profile classes. The same -ownership boundaries can serve future OTT provider integrations; the current activated provider -execution boundary remains SMPP-client submission. - -## Ownership model - -```text -accepted pending sources - | ready admission, or publishReady after held admission - v -ready pending work - | bounded, eligible, priority-aware select-and-stage - v -selected router work ---- take/release ---- router execution - | record one destination assignment, then mark routed - v -destination work ------- take/release ---- worker execution - | all provider parts terminal AND successful required handoffs - v -current destination work terminal - | complete pending sources - | complete selected work - v -forget routed terminal bookkeeping -``` +# Outbound Message Storage + +Sendium's standalone application processes HTTP/SMPP submissions through one shared outbound lifecycle. +Accepted sources remain owned until provider processing and every required tracking/delivery-report +(DLR) handoff finish. **Acceptance, queue removal and provider submission are not terminal completion.** -Pending sources remain authoritative throughout. Taking work does not delete source or stage -ownership. Source IDs, selection IDs, and destination work IDs are distinct opaque UUID-backed types; -none is inferred from mutable message equality or a protocol serial. One selected item may retain -several source IDs. Held admission and ready-work publication retain those sources without treating -an aggregate as a new independent admission. Protocol wiring follows separately; detailed multipart -grouping, deduplication, recovery, and codec contracts are owned by #343. - -Each routed selection has one destination and one work identity through execution, same-worker retry, -and terminal cleanup. Direct worker-to-worker forwarding is outside this abstraction's scope. - -## Sendium retry integration boundary - -The new library supports returning unfinished routing work through `returnToRouter(Selected)` and -retrying taken work at the same destination through `returnToDestination(Routed)`. The former cannot -be used after destination assignment and is not a worker-to-router transition. - -Sendium's existing `AbstractOutWorker` failure policies can call `enqueueToRouter` to let routing choose -again. Legacy queue behavior remains intact; lifecycle provider retries perform a new lookup under the -original parent work identity, retaining every required provider-part handoff. No direct worker-to-worker -API is substituted for it. Consumer-specific dispatchers, including mCore's queue worker, remain the -consumer team's responsibility and require separate approval for Sendium support. - -## Copied routing boundary - -The existing `+vendor`/`copied` routing feature and its public API remain available on the existing -routing path. The new storage abstraction does not support copied routing. The opt-in lifecycle -lookup rejects a matching copied-route rule as `UNSUPPORTED` before assignment or dispatch, even -if lookup would produce only one destination. It does not silently choose one result, strip the -copy flag, or emulate fan-out using several assignments for the same source. - -This guard does not change existing routing behavior. The single-destination scope is the owner's revision to the copied-route -requirements previously described in #337/#338. Those issue bodies have not been edited here. - -`StandardOutboundDispatch`, constructed explicitly with a coordinator and Sendium's routing manager, -can route one already selected item to one recorded destination and take that work for provider -processing. The dispatcher retains the selected handle and original destination intent if recording -needs a retry. `retryRoutingTransitions()` resumes that transition without repeating lookup or filters. -A routing miss returns the selection to the router; a matched copied rule fails before -assignment and likewise returns the selection. `finishProviderParts` is called only after the provider -has reported an outcome for every part. It waits for each part's required DLR/tracking handoff before -invoking coordinated completion. While a part handoff is outstanding, its sources remain retained; -failure leaves them owned for completion-only retry with successful handoff stages. Duplicate terminal -callbacks do not release another source. A successful stage represents a part requiring no handoff. - -### Coordinated SMPP provider processing - -`submitToProvider` schedules an explicitly taken work item through -`SmppClientWorker.submitPreparedCoordinated`, which applies rate limiting, character mapping and -before-processing filters before `submitCoordinated` generates provider requests. It registers every -request before sending and uses typed request references for provider callbacks. -The session handler recognizes these references for submit responses, request expiry and recoverable -PDU errors before applying its legacy message casts. Ordinary message references retain their existing behavior. - -Each request is claimed once, so duplicate callbacks cannot start another retry or tracking handoff. -Provider acceptance and rejection use the tracker's completion-stage handoff boundary; asynchronous -trackers can return a stage for the actual handoff. The standard rejection handoff propagates an -interrupted DLR router enqueue instead of classifying it as success. A failed handoff stays pending -and is retried through `ProviderExecution.retryPendingHandoffs`, without resending an accepted SMS. -Terminal store cleanup can be retried through `retryTerminalCleanup` without repeating provider processing. - -The production multipart retry payload choices are retained: the first request references the full -original message and later requests reference individual cloned parts. A first-send exception aborts -the remaining sends and retries the full payload; a later send exception retries that part. Negative -message IDs on cloned later parts retain the existing no-tracking behavior. All request outcomes and -replacement attempts remain under the parent work, without new pending admissions or copied-route branches. - -For retry responses, the worker executes its existing failure-filter/retry-counter/delay-policy logic -inside a scoped scheduling capture. Worker, router and delayed queue actions are redirected to tracked -replacement attempts. Router retries perform lifecycle routing lookup, including copied-route rejection; -same-worker retries reuse that worker. The scope is removed before replacement work begins and does -not change scheduling for ordinary legacy messages. Parent sources complete only once the original -outstanding requests, their replacement attempts, and required tracking handoffs all finish. - -The parent's recorded destination is its **initial assignment**, not an assertion that every later -attempt uses that provider. `ProviderExecution.attempts()` exposes ordered runtime attempt snapshots -with their resolved destination, state and preparation/routing failure. Each attempt is scheduled -explicitly, including same-worker replacements. Router retries resolve their own destination without -overwriting the parent record or re-admitting its sources. This lets a replacement finish at provider B -while an original multipart request remains outstanding at provider A. - -Paused or disconnected workers leave attempts queued; stopped workers and failed routing/preparation -leave attempts failed but unfinished. `retryPendingHandoffs()` also reschedules those failed attempts, -without re-running successfully submitted attempts. Provider tracking failures remain inside the -submission's part-handoff bookkeeping. Concurrent preparation/submission is limited by the worker's -configured thread count when its scheduling slots are first established; required tracking completion -does not hold a sending slot. Before-processing filter drops finish without sending; retry/re-enqueue -outcomes use the existing end-retry policy, including the same-worker fallback when no router exists. - -The default dispatcher owns a two-thread scheduled executor. Assembly may instead supply a -`ScheduledExecutorService`; the dispatcher does not close that application-owned executor. -Graceful shutdown uses the following application-owned ordering: - -1. Call dispatcher `quiesce()`. It quiesces the coordinator, stopping admission, selection and takes, - and prevents new provider submissions. An already-taken unassigned selection presented to the - dispatcher is returned to routing without recording a destination. -2. Keep destination workers, provider callbacks and the execution scheduler running. Call - `awaitProviderDrain(timeout)` to await active provider executions, including replacement attempts, - required handoffs and terminal cleanup. A timeout returns `false`; a failed completion throws an - actionable exception. Neither result releases ownership or stops the scheduler. Retry pending - handoffs/attempts or terminal cleanup through the existing execution handles and await again. -3. After a successful drain, stop/join application routing and destination execution loops and workers. - Application-wide worker stop retains chosen worker queues; runtime worker removal still uses its - existing return-to-router policy. -4. Close the dispatcher, then the coordinator. Dispatcher close refuses outstanding/failed executions - and never shuts down an application-owned scheduler. Coordinator close returns unsubmitted taken - work to its recorded destination and taken unassigned work to routing before closing stages. - -Stopping provider workers before drain can prevent callbacks or replacement attempts from finishing. -Unresolved shutdown must retain the live ownership and report failure rather than requeue a parent -while multipart callbacks are active. Memory closure remains non-durable and clears memory state; -returning ownership during shutdown does not promise restart recovery. - -The standalone application owns separate lifecycle processors alongside retained legacy entry points. -Its early shutdown observer applies this ordering before normal worker/connection shutdown observers. -Drain-first shutdown has no forced cutoff: unresolved callbacks or required handoffs keep shutdown -waiting, with actionable failures and periodic waiting logs. Runtime attempt bookkeeping is non-durable; it does not -provide provider-outcome checkpoints or restart recovery. +Only **`memory/memory/memory`** is implemented: unfinished work is lost on process closure or restart. +The contracts are message-neutral; the current standalone provider adapter executes SMPP-client submissions. +This is the initial slice of [#338](https://github.com/cytechmobile/sendium/issues/338), under [#337](https://github.com/cytechmobile/sendium/issues/337). ## Components -The public APIs are grouped by responsibility under `gr.cytech.sendium.core`: - -```text -outbound/ - OutboundCoordinator.java - DefaultOutboundCoordinator.java - OutboundWork.java -storage/ - OutboundStage.java - PendingMessageStore.java - SelectedRouterStore.java - RoutedWorkStore.java - OutboundStorageException.java - memory/ - MemoryPendingMessageStore.java - MemorySelectedRouterStore.java - MemoryRoutedWorkStore.java +`sendium-core` supplies reusable contracts and implementations. `sendium-app` constructs and runs the +standalone pipeline; importing the core library alone does not activate this storage assembly. + +```mermaid +flowchart LR + ingress["HTTP / SMPP admission"] --> coordinator["OutboundCoordinator
ownership and transitions"] + pipeline["StandaloneOutboundPipeline
construction, polling, shutdown"] --> coordinator + pipeline --> dispatch["StandardOutboundDispatch
routing and provider executions"] + dispatch --> coordinator + dispatch --> routing["StandardRoutingManager
rules and filters"] + dispatch --> provider["SmppClientWorker
requests and callbacks"] + provider --> handoff["Tracker / DlrStorage
required handoff"] + handoff -->|Completion condition| coordinator + coordinator --> pending["PendingMessageStore
accepted sources"] + coordinator --> selected["SelectedRouterStore
selected backlog"] + coordinator --> routed["RoutedWorkStore
recorded destinations"] ``` -Memory implementations live under `storage.memory`. Standalone configuration -assembly lives separately in `sendium-app`, under `gr.cytech.sendium.app.storage`: +| Component | Responsibility | +|---|---| +| `MessageStorageProfile` / `StandaloneMessageStorage` | Validate startup selectors, produce the profile and report its non-durable guarantee. | +| `StandaloneOutboundPipeline` | Start one shared coordinator; drive bounded selection, routing, dispatch, retries and drain-first shutdown. | +| `KannelResource` / `SmppServerWorker` | Admit before HTTP 202 / SMPP success; retain original source identities through multipart preparation. | +| `PendingMessageStore` / `MemoryPendingMessageStore` | Retain accepted sources, held parts and ready work through source completion. | +| `SelectedRouterStore` / `MemorySelectedRouterStore` | Own bounded selection and unassigned routing work; retain selected bookkeeping after routing. | +| `RoutedWorkStore` / `MemoryRoutedWorkStore` | Record one destination per selection and schedule its unfinished work by destination. | +| `DefaultOutboundCoordinator` / `OutboundWork` | Enforce typed source/selection/work ownership, transitions, handoff-dependent completion and retryable cleanup. | +| `StandardRoutingManager` / `StandardOutboundDispatch` | Resolve rules/filters, retain recording intents and schedule tracked provider attempts. | +| `SmppClientWorker` / `SmppClientSessionHandler` / `CoordinatedSmppSubmission` | Prepare/send requests, process callbacks once and join part/replacement outcomes. | +| `Tracker` / `StandardMessageTracker` | Signal successful required handoff; integrate with the independently configured DLR subsystem. | + +## Message Lifecycle + +```mermaid +sequenceDiagram + participant Client + participant Ingress as HTTP / SMPP ingress + participant C as Coordinator and stores + participant D as Standalone processors / dispatcher + participant R as Routing manager + participant P as Provider execution and callbacks + participant H as Required tracking / DLR handoff + Client->>Ingress: Submit message + Ingress->>C: accept source or acceptHeld part + C-->>Ingress: Accepted ownership + Ingress-->>Client: Success + gateway UUID + D->>C: selectToRouter - select bounded ready work into router backlog + D->>C: Take selected work for routing + C-->>D: Selected payload + original source IDs + D->>R: Resolve one destination and apply routing filters + R-->>D: Destination + D->>C: recordToRouted - record choice and mark selected work routed + D->>C: takeFromRouted - take assigned work for execution + D->>P: Schedule prepared provider attempt + D->>C: Register completion condition + Note over C,P: Sources remain retained while execution is outstanding + P->>H: Provider outcomes and required handoffs + H-->>P: Handoff succeeds + P-->>C: All parts, replacements and handoffs complete + C->>C: Mark routed work terminal + C->>C: Complete pending sources and selected state + C->>C: Forget routed record +``` -- `MessageStorageProfile` is an immutable validated configuration value. Its constructor is the single - supported-profile validator; it accepts only `memory/memory/memory`. -- `StandaloneMessageStorage` produces that profile as a CDI singleton from runtime configuration and - requires it in an early startup observer. The observer logs the effective profile and non-durable warning. +- **Identity:** HTTP/SMPP reuse each gateway UUID as its typed `SourceId`. `SelectionId` and `WorkId` + identify later steps independently of mutable payloads and provider IDs. Return the actual take handle, + with its updated payload, rather than a reconstructed wrapper; stale returns must not overwrite live work. +- **Incoming multipart:** `acceptHeld` retains each part before acknowledgement. The existing reassembler + makes an aggregate or expired individual parts ready using `makeHeldReady(originalSourceIds, payload)`. + This creates no new admission; publication retries do not re-acknowledge or reassemble the input. +- **Outgoing multipart:** provider requests and replacements share one routed parent and original source + set. They are not copied-route branches. No source is removed until all required completion conditions succeed. +- **Completion:** `complete(workId, stage)` can register an outstanding condition; it does not immediately + delete sources. Successful cleanup marks routed work terminal, completes pending sources and selected + bookkeeping, then forgets routed ownership. Successful steps are remembered for cleanup-only retry. +- **DLR boundary:** completion waits for required handoff acceptance, not a later handset-delivery receipt. + Receipt storage/delivery remains independent of outbound backend selection. + +`StandardOutboundDispatch.routeSelected()` performs routing lookup and filters. +`OutboundCoordinator.recordToRouted(selected, destination)` records that already-chosen result and +advances stage ownership; it does not choose the destination or send the message. +`takeFromRouted(destination, timeout)` takes one assigned item without completing ownership; +`returnToRouted(work)` returns an unfinished take to scheduling at its unchanged destination. +`makeHeldReady(sources, preparedMessage)` binds held sources to the caller-prepared payload without +assembling or accepting new input. `selectToRouter(limit)` then selects eligible work into the router +backlog and returns the newly staged count; it does not evaluate routing rules. + +## Ownership and Capacity + +The stores are process-local objects, not three independent batches. The routed store contains queues +for recorded destinations; it is **not** the worker's legacy `msgQ` or a disk directory. + +For three groups of 100 distinct, unfinished, single-source messages: + +| Location in the flow | Pending source records | Selected records | Routed records | +|---|---:|---:|---:| +| 100 ready, not selected | 100 | 0 | 0 | +| 100 selected, not assigned | 100 | 100 | 0 | +| 100 assigned, queued or executing | 100 | 100 retained as bookkeeping | 100 | +| Total logical ownership | **300** | **200** | **100** | + +Only the unassigned group occupies router queue capacity. Recording a destination frees its routing +slot but retains the selected record until cleanup. These counts are not physical payload-copy or RAM counts. + +| Setting | Default | Meaning | +|---|---:|---| +| `sendium.message.pending.capacity` | 10000 | All retained accepted sources, including held parts and sources executing at later stages. | +| `sendium.message.router-queue.capacity` | 1000 | Queued plus taken unassigned selections; taking does not free a slot. | +| `sendium.message.selection-batch-size` | 100 | Maximum new selections, routing takes and takes for each eligible destination per processing cycle. | +| Routed capacity | Same as pending capacity | Retained destination records across all workers, including terminal records awaiting cleanup; no separate setting. | + +With 950 occupied router slots, capacity 1000 and batch size 100, at most **50** new selections fit. +A group of N accepted multipart sources uses N pending slots but one selected slot and one routed record. +Partial terminal cleanup can retain routed records after source removal, so equal capacities do not +eliminate cleanup-related backpressure or provide cross-store transactions. + +The poller has a 100-millisecond fixed delay after each cycle. Batch size is not TPS or provider +concurrency: attempts use worker sending slots/rate limiting and a default shared two-thread scheduler. +Awaiting asynchronous outcomes does not hold a sending slot. Count limits do not bound message bytes. + +## Retries and Failures + +```mermaid +flowchart LR + parent["Parent WorkId W1
recorded destination A"] --> original["Attempt at A
multipart requests"] + original --> outstanding["Original requests still outstanding"] + original --> policy{Retry policy} + policy -->|Same worker| same[Replacement at A] + policy -->|Router retry| lookup[Fresh lifecycle lookup] + lookup --> replacement[Replacement at B] + outstanding --> join["Join required outcomes
and handoffs"] + same --> join + replacement --> join + join --> cleanup[Complete parent and original sources] +``` -These classes are absent from the `sendium-core` artifact; they do not activate inside an embedding application. +The parent keeps its **initial destination**, while runtime attempts carry their actual destinations. +Router retries resolve routing again without new admissions or a parent transfer. Existing full-message +versus individual-part retry choices are preserved. Attempt progress is non-durable; this is not exactly-once delivery. +`requeueForRouting` puts taken unassigned selections back in the selected backlog. +`returnToRouted` preserves the assigned destination. Provider router retries are separate runtime +attempts, not either storage-return operation. -| Contract | Responsibility | -|---|---| -| `PendingMessageStore` | Admit ready or held sources, publish prepared work using held source IDs, and complete sources idempotently. | -| `SelectedRouterStore` | Own bounded select-and-stage, exclusive runtime takes, return-to-router, and retained selected state. | -| `RoutedWorkStore` | Record one destination, own same-destination scheduling/retry, and report terminal source ownership. | -| `OutboundCoordinator` | Application-facing admission, transitions, required handoff, source completion, and lifecycle coordination. | -| `DefaultOutboundCoordinator` | Executable coordinator for fresh non-durable stages, with owned transitions and retryable handoff/cleanup. | -| `OutboundWork` | Source/selection/work identities and typed execution, assignment, and completion values. | -| `OutboundStage` | Explicit open/close, backend identity, durability declaration, and lifecycle availability. | -| `OutboundStorageException` | Identify the failing stage and distinguish unavailable, capacity, ownership, unsupported, and invalid-transition failures. | - -The selected-router implementation receives its source access and selection/preparation collaborators -through construction. It can use Sendium's pending implementation or application-owned queries and -transactions. The shared API does not expose a SQL query language or require the application to -materialize its entire backlog. A selected source is not eligible for another concurrent selection. -Stage implementations enforce state transitions; malformed arguments fail immediately, while invalid -ownership/state transitions use the categorized storage failure. - -`OutboundWork.Assignment` carries exactly one `Destination`, and `Routed` identifies its execution -with a `WorkId`. The work values freeze source identity sets, not message objects. Worker and filter -mutations are intentional execution state. They continue through routing and same-worker returns; -they must not silently change a retained earlier-stage record through a shared mutable reference. - -The contract requires consistent ownership and stored payload/stage, not a public copying service or -a copy on every operation. A backend may use independent snapshots, encoding/mapping, or explicit -ownership transitions to achieve that consistency. The concrete mechanism will be chosen with the -memory implementation. Custom message subtypes and their fields must be preserved; flattening a -subclass to `StandardMessage` is not acceptable. Codec/schema design comes later. - -## Recovery follows recorded stage ownership - -After reconciling an interrupted transition by stable identity, recovery resumes from the latest -safely recorded stage for each source: - -| Surviving state | Resume behavior | +| Boundary | Behavior | |---|---| -| Destination and execution payload recorded | Restore work at that destination's worker queue. Do not rerun routing or routing filters. | -| Selected work, without recorded destination work | Resume routing from the selected state. | -| Accepted pending source only | Repeat selection and routing. | - -Earlier retained records must not independently schedule the same work. Keeping a pending record -through completion does not mean routing it alongside its recorded worker-stage message. - -A stored payload must match the processing stage that will resume it. For example, silently replacing -a pre-routing payload with its routing-filter output while retaining the pre-routing stage could apply -the transformation twice. Correctly recorded routed work instead resumes with its saved destination -and execution payload, without running that routing filter again. - -Backends may retain separate accepted/execution representations or explicitly advance payload and -stage together, provided they retain the recoverable source information required by the configured -profile and can reconcile interrupted transitions. This does not require permanently immutable -copies of every intermediate message. Work performed after the last recorded boundary may repeat; -provider-outcome checkpoints and exactly-once submission remain outside the guarantee. - -These are requirements for the later durable implementations, not a restart guarantee for memory. - -## Memory pending and selected-router implementation - -These two classes are plain Java components with no CDI activation or background refill threads: - -- `MemoryPendingMessageStore` takes a positive maximum source count and a - `UnaryOperator` snapshot function. An optional `Function` supplies absolute - eligibility time when ready work is first published. Ordinary `admit` publishes a singleton immediately; - `admitHeld` does not evaluate eligibility or put anything in the selection index. The default time policy - makes published work immediately eligible. No new interpretation of HTTP deferred-delivery or validity - fields is introduced by this library step. -- `MemorySelectedRouterStore` takes that pending store, a positive routing capacity, and optionally - a `Clock` (UTC by default). Open pending first, then the selected store. A pending instance accepts - only one selected-router owner for its lifetime, including after that router closes. - -The snapshot function is a concrete memory-backend constructor dependency, not a new public copier -interface. It must preserve the concrete subtype and all fields, detach mutable state, and not retain -references that a caller could later mutate. The backend rejects null, identity, and subtype-flattening -results. It cannot generically prove deep isolation of arbitrary application-specific objects; supplying -a correct snapshot function belongs to assembly. Functions run under the store lock and must be -side-effect-free, nonblocking, and must not reenter these stores. The standalone message mapper is -supplied when ingress is wired; this step does not serialize messages or define a durable codec. - -Pending admission is idempotent by source ID while that source exists, even at capacity. Repeating either -admission method preserves the first accepted payload and admission disposition: an `admit` retry cannot -make a held source ready, and an `admitHeld` retry cannot withdraw ordinarily admitted work. New source -admission fails with `CAPACITY_EXCEEDED` when the bound is reached. Admission or snapshot failure -does not reserve a source slot. A single-source `find` returns an isolated value for backend consumers; -there is no all-backlog loading API. Pending completion remains the caller's terminal-processing -decision, not a side effect of selection, take, or routing. - -Selection uses priority-indexed ordered sets maintained at ready publication. It inspects due candidates rather -than copying or sorting the whole backlog on each refill. Higher numeric priorities select first; within -a priority, earlier eligibility comes first, then ready-publication order (admission order for ordinary -singletons). Future work in a higher-priority group -does not block eligible work at a lower priority. A batch captures one clock instant. This is an -in-process scheduling policy, not a durable global-order guarantee. - -`selectAndStage(limit)` publishes at most the smaller of the requested limit and free routing slots. -Each selected ready item is removed from the eligibility index while all its sources remain in pending storage. -A prepared aggregate occupies one routing slot, not one per source. Shared -locking makes staging and removal from eligibility indivisible per ready item. If snapshotting a later -item fails, the published prefix remains available and the failed item remains eligible; retry neither -duplicates selected sources nor loses accepted work. - -The selected backlog schedules in publication order. Routing capacity counts both queued and taken -items, so returning taken work never requires another slot. `markRouted` releases the routing slot but -retains selected ownership; only after pending completion may selected `complete` remove the record. -Routed sources are not eligible for another batch. The caller invokes bounded refill as slots become -available; taking work does not implicitly select a new batch. - -Take returns an isolated execution projection. Return the actual `Selected` value from that take after -updating its message; it serves as the local attempt handle, while its stable selection ID remains the -stage identity. Return snapshots the updated execution payload without changing accepted content. -Duplicate returns do not enqueue duplicates, and an older attempt cannot replace an active newer -take. Reconstructed or unrelated projections are rejected. This local handle is not a persisted dequeue -stage or a multi-process lease. - -Blocking takes support zero-time polling, timeout, interruption, and wake-up when work is published -or either store closes. Close selected before pending after stopping execution. Closing is idempotent; -neither store can reopen after close. Closing a selected store does not complete its pending sources -or make its claimed sources eligible again; shutdown finishes by closing the pending store as well. -All state is instance-local. New instances start empty and provide no restart recovery. Counts bound -admission and active routing, not message byte size; required terminal bookkeeping is removed by -coordinated cleanup in the later integration tasks. - -## Held-source admission and ready-work publication - -The pending interface and coordinator expose two operations for the initial multipart ownership boundary: - -```java -coordinator.admitHeld(firstSourceId, firstPart); -coordinator.admitHeld(secondSourceId, secondPart); - -// The existing reassembler supplies this message and its accepted source IDs. -coordinator.publishReady(Set.of(firstSourceId, secondSourceId), assembledMessage); - -coordinator.selectAndStage(batchLimit); +| Unsupported backend profile | Fail startup with requested/supported choices; no fallback to memory. | +| Admission capacity/unavailability | HTTP 503; SMPP capacity returns `STATUS_THROTTLED`. Failed admission is not acknowledged as success. | +| Routing miss / nonterminal lookup failure | Use `requeueForRouting` with ownership retained. A terminal routing drop uses `discard`. | +| Copied route / unsupported standalone destination | Reject before assignment, even if a copied rule resolves to one target. The existing legacy copied-routing API remains separate. | +| Destination-recording failure | Retry the retained original intent without repeating lookup or routing filters. | +| Required handoff failure | Retain ownership and retry the handoff, not an accepted provider request. | +| Terminal cleanup failure | Retry unfinished cleanup without awaiting another successful handoff or redispatching provider work. | + +Each provider response is claimed once; duplicate callbacks cannot create another replacement or +complete unrelated sources. Admission deduplication applies while ownership is retained, not forever. +Runtime worker removal's legacy queue return does not migrate coordinator-owned parent records. + +## Graceful Shutdown + +```mermaid +flowchart TD + begin["beginShutdown - stop new acceptance,
selection, takes and submissions"] --> stopPolling[Stop and join polling] + stopPolling --> drain["Retry retained intents; drain active
requests, replacements, handoffs and cleanup"] + drain --> ready{Drain successful?} + ready -->|No| wait["Keep ownership and providers active
report and retry"] + wait --> drain + ready -->|Yes| stopWorkers[Stop legacy router and workers] + stopWorkers --> stopped{Worker manager stopped?} + stopped -->|No| refuse[Report failure; leave stages open] + stopped -->|Yes| closeDispatch[Close dispatcher] + closeDispatch --> restore["Return unassigned takes to routing
and unsubmitted takes to their destination"] + restore --> closeStores[Close routed, selected, then pending stores] ``` -This example describes the library boundary used by the opt-in admission integration below. Successful held admission gives -the protocol integration an ownership boundary at which to acknowledge an individual part without -waiting for assembly. Held records consume pending source capacity but can never be selected on their -own, regardless of their priority or timestamp. A snapshot/admission failure must not be acknowledged. - -`publishReady` accepts a nonempty set of already admitted, initially held source IDs and one prepared -execution message. It validates all sources and prepares the snapshot/eligibility metadata before binding -the whole set and exposing one candidate to normal selection. It does not enqueue directly to the router, -create a new accepted source, free source capacity, or overwrite the original accepted part payloads. -Publication therefore works even when the held sources fill pending capacity. Selection still observes -batch size, routing capacity, the prepared message's priority, and its eligibility time. - -In memory, accepted-source records and ready scheduling entries are separate. Each source links to -its first ready entry; multiple sources can share one entry. The shared lock prevents a selector from -seeing a partially bound group. The ready-entry link remains after selection so retries do not publish -another candidate. The source snapshot remains available through `find(sourceId)` independently of -the prepared aggregate and its subsequent execution mutations. - -While every source is retained, repeating the exact source set is a successful no-op: the first published -payload, priority, and eligibility time win, including after the item is selected, taken, or routed. Missing -or ordinarily admitted sources and overlapping/subset/superset regroupings fail as `INVALID_TRANSITION`. -All validation precedes binding, so failure cannot consume otherwise-unpublished sources. Snapshot or -eligibility failure leaves held sources available for a whole-publication retry. Once any source has been -removed during terminal cleanup, publishing that set is invalid; publication never recreates sources or -retains completed-publication tombstones forever. - -The caller owns assembly and expiry decisions. To release an expired incomplete part independently, -publish a singleton source set and its execution message. This primitive introduces no UDH parsing, -group-key policy, ordinal deduplication, timer, persisted deadline, or codec. Those detailed contracts -remain #343 work; the admission integration reuses the existing reassembler. - -`admitHeld` requires a running coordinator, just like ordinary admission. `publishReady` is an -existing-work transition and remains available while quiescing, so an assembler may release already -accepted work during shutdown. New selection/takes remain stopped. Held and prepared memory state -is instance-local and disappears on close/restart; no new durability guarantee is implied. - -## HTTP/SMPP admission binding - -`KannelResource` and the standard outgoing-worker factory resolve a default-qualified CDI -`OutboundCoordinator` when one is supplied by application assembly. HTTP calls -`admit` on that coordinator; the factory binds it to each standard SMPP server before worker startup. -Other explicitly assembled SMPP workers can use `setIngressCoordinator` with their own message type. -The SMPP binding cannot be changed after startup or any submission has begun. - -`StandaloneOutboundPipeline` supplies a default-qualified, started singleton coordinator in `sendium-app`. -Its static default producer validates the profile and positive memory bounds before construction. -With no coordinator binding in an embedded application, -the existing memory-queue admission path remains available. When a binding exists, resolution or -admission failure does not fall back to that queue. Applications must supply a long-lived, started -coordinator and connect selection, dispatch, completion, and retry policies before enabling the new -pipeline. Standalone assembly uses `StandaloneMessageSnapshot` to detach mutable protocol collections, -byte arrays and nested map/list attribute values. Unknown mutable extension objects and custom message -subtypes require application-owned assembly and a suitable snapshot policy rather than shallow copying. - -The standalone processors poll every 100 milliseconds, honor the global routing pause, select a bounded -batch, route selected work, and take bounded work for each available SMPP-client destination. Misses -and lookup failures return selections; routing filter drops complete their source without provider -dispatch. Recorded transition failures retain their original intent for retry; other destination work -can continue. Provider execution handles are retained for handoff, attempt and terminal-cleanup retries. -The activated provider boundary is SMPP-client submission: copied routes and other legacy worker types -are rejected before assignment with an explicit unsupported error and retained routing ownership. -Legacy queues remain available for callers explicitly using those entry points, not as admission fallback. - -### HTTP - -After authentication, parameter handling, gateway UUID generation, and DLR return-metadata creation, -HTTP admission calls `admit` with a typed source ID wrapping the same UUID as the gateway message ID. -It returns `202` only after admission succeeds; the response contains that gateway UUID as text. -The source ID remains a distinct lifecycle type so other applications can use their own protocol IDs. -Lifecycle failures map to: - -| Failure | HTTP response | -|---|---| -| Capacity, unavailable storage/lifecycle, or ownership conflict | `503` with a retry-later response | -| Unsupported submission | `400` | -| Invalid internal transition or unexpected resolution/processing error | `500` | - -Error responses do not expose storage exception details. The legacy interrupted-admission path also -returns `503` and preserves the thread's interrupt flag. - -### SMPP and existing reassembly - -Before-insert filters and gateway UUID/timestamp assignment still precede admission. A complete SMS -is admitted directly as ready work. A submission with a concatenation header must pass the existing -reassembler's supported-header check, then uses `admitHeld` before `STATUS_OK`. It enters the ingress -queue with its accepted source ID (the same UUID as that part's gateway message ID); it is not -independently selected for routing. In legacy mode, local -ingress-queue insertion now also precedes `STATUS_OK`. - -Capacity rejection returns `STATUS_THROTTLED`; unsupported input returns `STATUS_SUBMITFAIL`; -other lifecycle/storage failures return `STATUS_SYSERR`. Rejections do not publish a successful response -or enter the legacy router queue. No new UDH syntax or detailed ordinal-validation policy is introduced. - -The existing reassembler produces either an aggregate or individually expired parts. Their internal -`InEvent` values carry an immutable source-ID set and no `submitSm`; processing calls `publishReady` -instead of admitting a new source or enqueueing directly to the legacy router. Publication failure requeues -that prepared event with the same payload and source IDs. It does not reassemble again, issue another -client acknowledgement, or release accepted sources. Original part payloads remain in pending storage. - -Per-worker identity-based metadata associates accepted raw part objects with source IDs until the -association is handed to a prepared event. The existing first-part-wins ordinal behavior remains; a -default-compatible `onDuplicateMessagePart` listener hook associates an ignored duplicate's held source -with the retained original. Its source therefore finishes with that message instead of leaking pending -capacity. This does not change duplicate response IDs or downstream receipt policy, which remain part -of the detailed multipart follow-up. - -Part insertion and completion/expiry callbacks are serialized within the existing handler so source -associations cannot cross group removal. Delayed tasks capture the actual group instance: a cancelled -old timer cannot consume a newer group that reused the same reference. Expiry only publishes the -parts already in that group, not accepted parts still waiting in the ingress queue. Group-key rules, -body assembly, relative timeout settings, and first-part-wins policy otherwise remain unchanged. - -Standalone dispatch and source-retaining provider retries are connected through the shared coordinator. -No mCore-specific workflow or storage adapter is added. - -## Memory routed-work implementation - -`MemoryRoutedWorkStore` is independently constructed with a positive selection capacity and a -`UnaryOperator` snapshot function, with the same isolation/subtype requirements as the pending -memory backend. It has its own lock and does not depend on a concrete pending or selected store. -Coordination across stores belongs to `DefaultOutboundCoordinator`. - -```text -record assignment → QUEUED → take → TAKEN - ↑ | - +-- release --+ - | - complete - | - COMPLETED - | - repeatable cleanup result - | - coordinator finishes cleanup - | - forget -``` +Providers, callbacks and replacement scheduling stay active during drain. Standalone shutdown has no +forced drain cutoff; unresolved work keeps it waiting and logging. Stop failure or refused closure is +not terminal success. Application-owned schedulers are not closed by the dispatcher. -- `record` retains one assignment per selection. Matching retries return its original work ID and - original assignment payload, even after completion, without re-enqueueing it. A - changed source set/destination is invalid. A source cannot belong to two retained routed selections. -- The `record` result describes recorded work; it does not claim execution. Only `take` - claims a work item. Each destination schedules in publication order. Take and return preserve one - active taker and updated execution state; return the actual take result as its local attempt handle. -- `complete` requires taken work after the caller has established terminal processing and required - handoff. It returns `Completed` directly; repeated completion returns the same selection/source set. - There is no transferred-predecessor case or empty completion result. -- `forget` rejects nonterminal selections. After the coordinator completes pending/selected cleanup, - it removes that selection's one work record and source claims, freeing capacity. Repeated - forget is harmless. Unknown work IDs fail explicitly. Replay guarantees apply while records are - retained; the coordinator must reject stale routing requests and absorb duplicate terminal - callbacks after cleanup rather than recreating old work. - -Capacity counts retained selections, including taken work and terminal selections awaiting cleanup. -Same-worker return reuses that selection's slot even at full capacity. The same entry is indexed by -selection ID and work ID; no successor chain or forwarding history is stored. The original assignment -payload is retained for record retries alongside current execution state. Selection count does not bound -message bytes. Sendium's retry/rerouting policy is integrated in the later worker task. - -All required snapshots, including the returned result, are prepared before assignment publication. -If a snapshot fails, no partial assignment is published, and the current work remains -retryable. Takes are interruptible and wake on publication or close. Closure clears instance-local state -without manufacturing completion; closed instances cannot reopen and fresh instances recover nothing. - -## Default lifecycle coordinator - -Construct `DefaultOutboundCoordinator` with the pending, selected-router, and routed-work stores -as three direct dependencies. Construction is passive. `start()` opens them in that order; failed startup -closes every attempted stage in reverse order, including a stage whose open failed, and preserves -cleanup failures as suppressed exceptions. - -This implementation currently requires fresh, unopened, non-durable stages. Already-opened stages -are not silently adopted, and durable stages fail with `UNSUPPORTED` because startup recovery and -cross-backend reconciliation are not implemented yet. Non-durable embedding implementations can -use the interfaces directly; there is no requirement for their backend identity string to be `memory`. - -The application must use the coordinator exclusively for scheduling and transitions. `admit` does not -implicitly refill the router; invoke bounded `selectAndStage` explicitly. Blocking takes poll the owned -store with a zero timeout and then wait on the coordinator's publication/quiescence condition. -They support interruption and timeouts and wake when coordinated work is published or quiescence begins. - -### Owned transitions - -The coordinator tracks live selections, source ownership, and destination executions. It validates the -actual take projection when returning or routing work. Recording a destination precedes marking the -selection routed. A failed route retains its original destination intent; retry the same `route` operation, -not the routing/filter lookup. While that transition is unresolved, its source cannot return to routing -or be dispatched at the destination. Other existing destination work can still complete and free capacity. - -A backend that committed an assignment before reporting a failure can be reconciled using the same selection/work identities; -observing its published take associates it with the retained intent. An unexposed take whose return -failed is retained for a return retry rather than disappearing from ownership. - -### Required handoff and cleanup - -`complete(workId, requiredHandoff)` begins terminal processing for a taken destination. `discard` -does the equivalent for a taken, unassigned routing item. Concurrent duplicate reports share the active -completion attempt. No source is removed while the handoff is pending, failed, or cancelled. Retry a -failed handoff explicitly with another completion call; the work cannot meanwhile be returned for -provider redispatch. - -Once the handoff succeeds, the coordinator remembers that success and performs: - -```text -destination complete (when routed) - → verify returned selection/source identities - → pending sources complete - → selected record complete - → routed bookkeeping forget (when routed) - → remove coordinator ownership -``` +`beginShutdown()` starts this process by stopping new lifecycle work; it does not itself drain providers, +stop workers or close stores. The application performs those steps before `close()`. -Each successful cleanup step is remembered. A failed step is retried through `complete`/`discard` -without awaiting another handoff or dispatching provider work. Partial multi-source deletion retries -the full source set idempotently. Duplicate admission of a still-owned source does not recreate a -source record during an unfinished cleanup. The source identity must represent the same admission; -after all ownership is forgotten, long-term admission deduplication is not promised. - -Completion results are read-only stages: cancelling a caller's `toCompletableFuture()` view cannot -cancel the owned handoff or cleanup. Returned-stage listeners run outside the coordinator lock. -Handoff or cleanup failures settle that attempt exceptionally and retain ownership for retry. - -To avoid retaining completed-ID tombstones forever, terminal reports for unknown/no-longer-owned -work or selections are harmless no-ops. They never recreate work, remove another source, or wait on -an unrelated supplied handoff. Route and return requests still require live ownership and -reject stale IDs/projections. Unknown-ID errors raised for an operation on live owned work propagate; -they are not treated as successful cleanup. - -### Quiescence and close - -`quiesce()` stops new admission, selection, and takes while allowing existing transitions, returns, -handoff completion, and cleanup retries. The application stops/joins its processors before `close`. -Close returns taken routing work to the router and taken destination work to that same destination, -then closes routed, selected, and pending stores. It never completes sources merely to shut down. - -Outstanding handoff attempts, unfinished routing intents, and failed terminal cleanup must be -resolved/retried before close can finish. A refused close leaves the coordinator quiescing and its -stores open. A return failure also leaves ownership available for a subsequent close retry. Once -draining succeeds, close attempts every store and preserves close failures; closure is idempotent and -restart is forbidden. This library lifecycle does not stop application executors or provide a forced -shutdown policy. The standalone assembly owns its processors and applies the drain-first ordering above. - -## Completion and retry boundaries - -- Admission retries reuse the same source ID and do not overwrite the accepted state while the - source remains pending. A new ID is a new admission; protocol-level deduplication and deduplication - after terminal removal are not implied. -- Returning taken work includes its updated execution projection, preserving current-process filter - and retry mutations without silently changing earlier-stage state. The take's identities and - destination cannot change on return. This does not promise durable retry counters or timing. -- `record` publishes a single assignment before its work can be taken. Retrying the same selection - returns its original work ID without reactivating completed work. Conflicting source - identities or destinations fail as `INVALID_TRANSITION`, rather than creating another assignment. -- Marking selected work routed stops router scheduling but retains the selected record until terminal - source cleanup. This leaves room for a future durable router with memory-backed routed work. -- Provider multipart is aggregated by provider-processing integration: all parts must have terminal - outcomes and all required handoffs must succeed before completing the one destination work item. - Part submissions are not represented as multiple routed destinations. -- `complete(work, requiredHandoff)` is called after all provider parts have terminal outcomes. The supplied - `CompletionStage` represents successful acceptance by the required DLR/tracking subsystem, - not eventual delivery of the downstream receipt. Use an already-successful stage when no handoff - is required. For multipart, it covers all required part handoffs. Failure/cancellation retains ownership; - a later retry supplies another handoff attempt. -- Source cleanup occurs only when the active destination work satisfies that rule. Retry failed source cleanup without - dispatching terminal provider work again in the active process. The routed store retains a repeatable - completion result until cleanup succeeds and it can be forgotten. The coordinator must suppress - duplicate terminal callbacks, including callbacks arriving after store bookkeeping was forgotten. -- A terminal filter drop before routing uses `discard` with the same handoff requirement. A routing - miss uses return-to-router; it is not a terminal drop. Assignments without a destination are invalid. - -These contracts do not assume atomic transactions across independently selected stores. Physical -backends and the coordinator must reconcile interrupted transitions by stable identity. Refinement -of durable batch/transition semantics belongs to #344. No durable provider-outcome checkpoint or -exactly-once provider submission is promised. - -## Embedding and lifecycle - -An application assembles compatible `PendingMessageStore`, `SelectedRouterStore`, and -`RoutedWorkStore` instances and passes them directly as three constructor dependencies to its -coordinator implementation. There is no dependency-bundle type or public copier contract. -`DefaultOutboundCoordinator` provides that constructor and the initial non-durable lifecycle. Constructors -have no activation side effects. The standalone profile is assembled -through CDI in `sendium-app`; the reusable lifecycle/storage APIs require no CDI annotations or container. - -For example, mCore's `Message extends StandardMessage`, billing-aware batch preparation, -asynchronous committed tracking, and own lifecycle orchestration can be adapted without importing -its domain classes into Sendium. An application's tracker returning after an in-memory enqueue is -not a committed handoff; its completion signal must correspond to the promised storage acceptance. -mCore adapters and adoption are owned by its team, not #338. - -Graceful shutdown is two-phase: `quiesce()` stops new admission, selection, and takes; the application -stops and joins execution, resolves or drains in-flight provider operations, and returns unfinished -work; `close()` then closes owned stages. Completion and return operations remain available during -quiescence. Unrouted work returns to the router; already-routed work retains its destination. Runtime -worker removal is a separate policy, not an alias for application shutdown. Closure is idempotent, -cannot mark unfinished work terminal, and does not imply that memory survives a restart. - -## Standalone profile selection - -The standalone application reads these selectors once during startup: - -```properties -sendium.message.pending.backend=memory -sendium.message.router-queue.backend=memory -sendium.message.routed-work.backend=memory -``` +Execution becomes active when `submitToProvider` registers its scheduled attempt, even before the first +network send. These attempts must drain. Queued routed work not handed to that boundary keeps its chosen +destination; shutdown does not submit the entire pending/selected/destination backlog merely to empty it. + +For the 100-message groups above, ready work stays pending, unassigned takes return to routing, and +unsubmitted work stays at its destination. Started executions complete normally before closure. +**Memory close clears remaining unfinished records. There is no shutdown-time switch to disk storage.** + +## Configuration and Restart Guarantees -| Pending / router / routed | Availability | Restart behavior | +All selectors and bounds are global, startup-only Quarkus configuration. See the +[configuration reference](09-configuration-reference.md#outbound-message-storage-profile) for defaults, +environment variables and overrides. Selectors default to `memory`; blank, unknown or unimplemented +values fail. Capacity/batch values must be positive. + +`StandaloneMessageStorage` validates before watcher/worker startup and reports the non-durable profile. +`StandaloneOutboundPipeline` separately starts the shared coordinator and activates polling after +existing router/worker startup. A failing coordinated admission never falls back to a legacy queue. + +| Pending / selected / routed | Availability | Restart behavior | |---|---|---| -| `memory/memory/memory` | Only selectable profile; stage implementations/coordinator wiring follow in #338 | Non-durable. | -| `file/memory/memory` | #339 | Recover accepted work; reselect and reroute. | -| `file/file/memory` | #345 | Recover selected work; reroute. | -| `file/memory/file` | #345 | Reselect not-yet-routed work; restore recorded destinations. | -| `file/file/file` | #345 | Restore selected work and recorded destinations. | -| PostgreSQL profiles | #333 | Only explicitly implemented combinations will be supported. | - -Standalone validation accepts only this profile and rejects unsupported requests before -admission, without falling back to memory. A custom backend identity in the library does not register -a supported standalone profile. DLR storage remains independent of outbound stage selection. -Future durable guarantees require surviving storage and remain at least once. - -Missing selectors default to `memory`. Explicit blanks, unknown values, and every unimplemented -combination fail startup with the requested profile and supported choice. The values are case-sensitive -and are read from Quarkus runtime configuration, not hot-reloaded `smsg.properties` worker settings. -Use the corresponding environment variables `SENDIUM_MESSAGE_PENDING_BACKEND`, -`SENDIUM_MESSAGE_ROUTER_QUEUE_BACKEND`, and `SENDIUM_MESSAGE_ROUTED_WORK_BACKEND`, or JVM `-D` -properties. Changes require a restart; they do not require rebuilding the application. - -The startup observer uses `Interceptor.Priority.PLATFORM_BEFORE`, ahead of the existing file watchers -(`LIBRARY_BEFORE`) and router/worker observers (`APPLICATION`). Injecting the singleton profile into -that observer forces validation before those later observers run. Successful startup logs the profile -and a `NON-DURABLE` warning. The library memory stores are not yet constructed by this startup observer. - -This milestone includes actionable errors and startup profile/non-durable logging. Metrics, -readiness endpoints, and broader observability are deferred. Contract-level status is not a health endpoint. - -## Behavioral verification - -The memory slice is verified at the following connected boundaries: - -| Boundary | Coverage | -|---|---| -| HTTP admission through standalone processing and capacity recovery | `MessageLifecycleHttpTest` and `StandaloneOutboundPipelineTest` exercise real HTTP/CDI admission and memory stores; the latter runs the production processors against a controlled provider handoff. | -| SMPP ownership before acknowledgement and multipart publication | `SmppLifecycleAdmissionTest` exercises submit/response objects, held sources, reassembly and publication retry without another acknowledgement. | -| Combined multipart lifecycle and shutdown drain | `SmppClientWorkerTest` carries two held sources through copied-route rejection, real routing lookup, request generation, rerouting to another provider during quiescence, failed handoffs and handoff-only retry. Original sources remain owned until completion; handoff retries and duplicate callbacks create no additional provider requests. | -| Standalone shutdown ordering and worker-stop failure | `StandaloneOutboundShutdownTest` proves workers remain available during handoff drain and stage closure waits for successful worker-manager stop. | -| Application-owned generic assembly | `OutboundEmbeddedLifecycleTest`, outside the implementation package, supplies a selection-store implementation with its own bound/backend identity and a custom mutable message subtype. Real stores/coordinator preserve subtype fields through selection, returns, failed asynchronous handoff, completion retry and explicit start/quiesce/close. | -| Routing transitions and copied-route guards | `StandardOutboundDispatchTest` verifies recording retries preserve the original intent without lookup replay, routing drops complete sources and copied rules fail before assignment. | -| Startup profiles and memory bounds | `MessageStorageProfileTest`, `MessageStorage*StartupTest` and `StandaloneOutboundConfigurationTest` cover supported defaults, unsupported requests, early rejection and positive bounds. | - -Provider tests exercise real SMPP request generation and session callbacks with controlled transport/handoff -fixtures rather than an external provider socket. Embedded fixtures verify the shared contracts within -Sendium; deployment or adoption in another application remains that application's responsibility. +| `memory/memory/memory` | Implemented and activated; default | No recovery across closure/restart. | +| `file/memory/memory` | Future [#339](https://github.com/cytechmobile/sendium/issues/339) | Recover sources; reselect and reroute. | +| `file/file/memory` | Future [#345](https://github.com/cytechmobile/sendium/issues/345) | Recover selected work; reroute. | +| `file/memory/file` | Future #345 | Reselect not-yet-routed sources; restore recorded destinations. | +| `file/file/file` | Future #345 | Restore selected work and recorded destinations. | +| PostgreSQL profiles | Future [#333](https://github.com/cytechmobile/sendium/issues/333) | Only explicitly implemented combinations. | + +Future profiles are not selectable today. Memory remains the non-durable default; planned file profiles +are opt-in. Durable recovery must reconcile identities and resume the latest recorded stage without +scheduling earlier retained records independently. It requires surviving storage and remains at least +once: provider submission before recorded completion may repeat. Physical backends, codecs and durable +provider-outcome checkpoints belong to subsequent issues. + +## Embedding and API Reference + +Embedding applications supply compatible `M extends StandardMessage` stores, selection/preparation, +snapshot policy, tracking handoffs and lifecycle orchestration. Constructors are passive; the default +coordinator requires fresh non-durable stages and does not implement durable startup recovery. + +Custom subtype fields and runtime mutations must survive transitions/returns without changing retained +earlier-stage payloads. `StandaloneMessageSnapshot` handles standard protocol payloads and supported +mutable fields; custom types require an application-owned mapper/assembly. Shared message naming leaves +room for future OTT integrations without claiming current Viber/WhatsApp execution support. + +Detailed operation, idempotence and failure rules are defined in the tracked Java contracts: + +- [OutboundCoordinator](../sendium-core/src/main/java/gr/cytech/sendium/core/outbound/OutboundCoordinator.java) and [OutboundWork](../sendium-core/src/main/java/gr/cytech/sendium/core/outbound/OutboundWork.java) +- [PendingMessageStore](../sendium-core/src/main/java/gr/cytech/sendium/core/storage/PendingMessageStore.java) +- [SelectedRouterStore](../sendium-core/src/main/java/gr/cytech/sendium/core/storage/SelectedRouterStore.java) +- [RoutedWorkStore](../sendium-core/src/main/java/gr/cytech/sendium/core/storage/RoutedWorkStore.java) +- [OutboundStage](../sendium-core/src/main/java/gr/cytech/sendium/core/storage/OutboundStage.java) and [OutboundStorageException](../sendium-core/src/main/java/gr/cytech/sendium/core/storage/OutboundStorageException.java) + +Verification covers real HTTP/CDI admission, SMPP acknowledgement/reassembly, multipart callbacks, +rerouting and failed handoffs while shutting down, custom-message isolation and application-owned lifecycle, +shutdown ordering and startup rejection. Provider transport/handoffs use controlled fixtures, not a live +external provider. Key tests are [StandaloneOutboundPipelineTest](../sendium-app/src/test/java/gr/cytech/sendium/app/storage/StandaloneOutboundPipelineTest.java), +[SmppClientWorkerTest](../sendium-core/src/test/java/gr/cytech/sendium/core/smpp/client/SmppClientWorkerTest.java) and +[OutboundEmbeddedLifecycleTest](../sendium-core/src/test/java/gr/cytech/sendium/core/outbound/consumer/OutboundEmbeddedLifecycleTest.java). diff --git a/docs/DocumentationMap.md b/docs/DocumentationMap.md index 59788409..8c74c4db 100644 --- a/docs/DocumentationMap.md +++ b/docs/DocumentationMap.md @@ -22,7 +22,7 @@ Sendium is an open-source, headless SMS gateway for high-throughput messaging. I | Understand releases and publishing | [11. Release Process](11-release-process.md) | | Review current features and roadmap | [12. Features And Roadmap](12-features-roadmap.md) | | Configure PostgreSQL DLR persistence and review durability | [13. DLR Persistence](13-dlr-persistence.md) | -| Review outbound storage extension contracts | [14. Outbound Storage Contracts](14-outbound-storage.md) | +| Understand outbound storage and extension contracts | [14. Outbound Message Storage](14-outbound-storage.md) | | Contribute code or docs | [Contributing](../.github/CONTRIBUTING.md) | ## Core Concepts @@ -64,7 +64,7 @@ Sendium expects these files in the configured `conf` directory. The Docker quick | [11. Release Process](11-release-process.md) | Release Please flow, Conventional Commit rules, release PR handling, GitHub Packages, and Docker publishing. | | [12. Features And Roadmap](12-features-roadmap.md) | Current product capabilities, planned roadmap phases, and related feature documentation. | | [13. DLR Persistence](13-dlr-persistence.md) | PostgreSQL setup, retention, restart guarantees, and durability limits. | -| [14. Outbound Storage Contracts](14-outbound-storage.md) | Initial stage APIs, ownership transitions, embedding boundaries, and planned standalone profiles. | +| [14. Outbound Message Storage](14-outbound-storage.md) | Diagram-led lifecycle guide, class responsibilities, capacity bounds, retries, shutdown, profiles, and extension contracts. | | [Kannel migration converter](https://cytechmobile.github.io/sendium/kannelconverter/) | Browser-only helper for turning a legacy `kannel.conf` into Sendium starter files. | ## API Discovery diff --git a/sendium-app/src/main/java/gr/cytech/sendium/app/storage/StandaloneOutboundPipeline.java b/sendium-app/src/main/java/gr/cytech/sendium/app/storage/StandaloneOutboundPipeline.java index 060080ce..99a61bd8 100644 --- a/sendium-app/src/main/java/gr/cytech/sendium/app/storage/StandaloneOutboundPipeline.java +++ b/sendium-app/src/main/java/gr/cytech/sendium/app/storage/StandaloneOutboundPipeline.java @@ -93,7 +93,7 @@ void process() { if (configuration.getBlnPrpt(AbstractRoutingManager._pause)) { return; } - coordinator.selectAndStage(batchSize); + coordinator.selectToRouter(batchSize); for (int i = 0; i < batchSize; i++) { var selected = coordinator.takeForRouting(Duration.ZERO); if (selected.isEmpty()) { @@ -129,7 +129,7 @@ void process() { var execution = dispatch.submitToProvider(work.orElseThrow(), (SmppClientWorker) client); executions.put(work.orElseThrow().id(), execution); } catch (Exception failure) { - coordinator.returnToDestination(work.orElseThrow()); + coordinator.returnToRouted(work.orElseThrow()); throw failure; } } @@ -160,7 +160,7 @@ void stop(@Observes @Priority(Interceptor.Priority.PLATFORM_BEFORE) ShutdownEven if (dispatch == null) { return; } - dispatch.quiesce(); + dispatch.beginShutdown(); processors.shutdown(); while (!processors.awaitTermination(1, TimeUnit.SECONDS)) { logger.warn("Waiting for standalone outbound processors to stop before ownership restoration"); diff --git a/sendium-app/src/test/java/gr/cytech/sendium/app/storage/MessageLifecycleHttpTest.java b/sendium-app/src/test/java/gr/cytech/sendium/app/storage/MessageLifecycleHttpTest.java index 7e1fdafd..f32e660d 100644 --- a/sendium-app/src/test/java/gr/cytech/sendium/app/storage/MessageLifecycleHttpTest.java +++ b/sendium-app/src/test/java/gr/cytech/sendium/app/storage/MessageLifecycleHttpTest.java @@ -63,7 +63,7 @@ void httpUsesTheExplicitCoordinatorBeanAndReturnsBackpressureAtCapacity() throws var full = client.send(request, HttpResponse.BodyHandlers.ofString()); assertThat(full.statusCode()).isEqualTo(503); assertThat(legacy.getRouterQueue().isEmpty()).isTrue(); - assertThat(coordinator.selectAndStage(1)).isEqualTo(1); + assertThat(coordinator.selectToRouter(1)).isEqualTo(1); var selected = coordinator.takeForRouting(Duration.ZERO).orElseThrow(); assertThat(selected.message().serial).isEqualTo(accepted.body()); assertThat(selected.sources()).singleElement().satisfies(source -> diff --git a/sendium-app/src/test/java/gr/cytech/sendium/app/storage/StandaloneOutboundPipelineTest.java b/sendium-app/src/test/java/gr/cytech/sendium/app/storage/StandaloneOutboundPipelineTest.java index 8d913a8f..2e4c6e4e 100644 --- a/sendium-app/src/test/java/gr/cytech/sendium/app/storage/StandaloneOutboundPipelineTest.java +++ b/sendium-app/src/test/java/gr/cytech/sendium/app/storage/StandaloneOutboundPipelineTest.java @@ -67,7 +67,7 @@ void productionAssemblyProcessesHttpThroughProviderCompletionWithoutLegacyAdmiss assertThat(providers.message.body).isEqualTo("Hello"); assertThat(legacy.getRouterQueue().isEmpty()).isTrue(); assertThat(client.send(request, HttpResponse.BodyHandlers.ofString()).statusCode()).isEqualTo(503); - assertThat(coordinator.selectAndStage(1)).isZero(); + assertThat(coordinator.selectToRouter(1)).isZero(); providers.handoff.complete(null); long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(5); diff --git a/sendium-app/src/test/java/gr/cytech/sendium/app/storage/StandaloneOutboundShutdownTest.java b/sendium-app/src/test/java/gr/cytech/sendium/app/storage/StandaloneOutboundShutdownTest.java index d0f34770..dd52c870 100644 --- a/sendium-app/src/test/java/gr/cytech/sendium/app/storage/StandaloneOutboundShutdownTest.java +++ b/sendium-app/src/test/java/gr/cytech/sendium/app/storage/StandaloneOutboundShutdownTest.java @@ -36,12 +36,12 @@ void standaloneShutdownDrainsProvidersAndRefusesStageClosureUntilWorkersStop() t var config = mock(Config.class); when(config.getOptionalValue(anyString(), eq(Integer.class))).thenReturn(Optional.empty()); var coordinator = spy(StandaloneOutboundPipeline.coordinator(new MessageStorageProfile("memory", "memory", "memory"), config)); - var quiesced = new CountDownLatch(1); + var shutdownStarted = new CountDownLatch(1); doAnswer(invocation -> { invocation.callRealMethod(); - quiesced.countDown(); + shutdownStarted.countDown(); return null; - }).when(coordinator).quiesce(); + }).when(coordinator).beginShutdown(); var provider = new StandaloneOutboundPipelineTest.Providers(); var routing = mock(StandardRoutingManager.class); when(routing.lookupForLifecycle(any())).thenReturn(new RoutingLookupResult(List.of(provider.worker), true)); @@ -58,7 +58,7 @@ void standaloneShutdownDrainsProvidersAndRefusesStageClosureUntilWorkersStop() t var message = new StandardMessage(); message.body = "hello"; - coordinator.admit(new SourceId(UUID.randomUUID()), message); + coordinator.accept(new SourceId(UUID.randomUUID()), message); try (var stopping = Executors.newSingleThreadExecutor()) { try { assertThat(provider.submitted.await(5, TimeUnit.SECONDS)).isTrue(); @@ -66,18 +66,18 @@ void standaloneShutdownDrainsProvidersAndRefusesStageClosureUntilWorkersStop() t pipeline.stop(null); return null; }); - assertThat(quiesced.await(5, TimeUnit.SECONDS)).isTrue(); + assertThat(shutdownStarted.await(5, TimeUnit.SECONDS)).isTrue(); assertThat(stopped.isDone()).isFalse(); verify(workers, never()).stop(); verify(coordinator, never()).close(); - assertThatThrownBy(() -> coordinator.admit(new SourceId(UUID.randomUUID()), message)) + assertThatThrownBy(() -> coordinator.accept(new SourceId(UUID.randomUUID()), message)) .isInstanceOf(OutboundStorageException.class); provider.handoff.complete(null); assertThatThrownBy(() -> stopped.get(5, TimeUnit.SECONDS)).hasCauseInstanceOf(IllegalStateException.class); verify(coordinator, never()).close(); pipeline.stop(null); var order = inOrder(coordinator, routing, workers); - order.verify(coordinator).quiesce(); + order.verify(coordinator).beginShutdown(); order.verify(routing).stop(); order.verify(workers).stop(); order.verify(coordinator).close(); diff --git a/sendium-core/src/main/java/gr/cytech/sendium/core/http/KannelResource.java b/sendium-core/src/main/java/gr/cytech/sendium/core/http/KannelResource.java index e4ec69d8..258e41a0 100644 --- a/sendium-core/src/main/java/gr/cytech/sendium/core/http/KannelResource.java +++ b/sendium-core/src/main/java/gr/cytech/sendium/core/http/KannelResource.java @@ -220,7 +220,7 @@ public Response receiveSms( if (outboundCoordinators.isUnsatisfied()) { queueProvider.getRouterQueue().enqueue(msg); } else { - outboundCoordinators.get().admit(new SourceId(gatewayMessageId), msg); + outboundCoordinators.get().accept(new SourceId(gatewayMessageId), msg); } if (MessageTrace.shouldLog(configurationHandler, MessageTrace.EVENT_ACCEPTED)) { logger.info("message.accepted ingress=http {}", MessageTrace.identifiers(msg)); diff --git a/sendium-core/src/main/java/gr/cytech/sendium/core/outbound/DefaultOutboundCoordinator.java b/sendium-core/src/main/java/gr/cytech/sendium/core/outbound/DefaultOutboundCoordinator.java index 167e9a01..2cc09cf1 100644 --- a/sendium-core/src/main/java/gr/cytech/sendium/core/outbound/DefaultOutboundCoordinator.java +++ b/sendium-core/src/main/java/gr/cytech/sendium/core/outbound/DefaultOutboundCoordinator.java @@ -107,11 +107,11 @@ private void open(OutboundStage stage, Role role, List attempted) } @Override - public void quiesce() { + public void beginShutdown() { lock.lock(); try { if (lifecycle == Lifecycle.RUNNING) { - lifecycle = Lifecycle.QUIESCING; + lifecycle = Lifecycle.SHUTTING_DOWN; changed.signalAll(); } else if (lifecycle == Lifecycle.NEW) { throw unavailable(Role.PENDING, "Coordinator has not started"); @@ -122,11 +122,11 @@ public void quiesce() { } @Override - public void admit(SourceId source, M message) { - admit(source, message, false); + public void accept(SourceId source, M message) { + accept(source, message, false); } - private void admit(SourceId source, M message, boolean held) { + private void accept(SourceId source, M message, boolean held) { Objects.requireNonNull(source, "source"); Objects.requireNonNull(message, "message"); lock.lock(); @@ -146,27 +146,27 @@ private void admit(SourceId source, M message, boolean held) { } @Override - public void admitHeld(SourceId source, M message) { - admit(source, message, true); + public void acceptHeld(SourceId source, M message) { + accept(source, message, true); } @Override - public void publishReady(Set sources, M message) { + public void makeHeldReady(Set sources, M message) { lock.lock(); try { requireActive(Role.PENDING); - pending.publishReady(sources, message); + pending.makeHeldReady(sources, message); } finally { lock.unlock(); } } @Override - public int selectAndStage(int limit) { + public int selectToRouter(int limit) { lock.lock(); try { requireRunning(Role.ROUTER_QUEUE); - return router.selectAndStage(limit); + return router.selectToRouter(limit); } finally { changed.signalAll(); lock.unlock(); @@ -215,7 +215,7 @@ public Optional> takeForRouting(Duration timeout) throws Interrupted } @Override - public void returnToRouter(Selected selected) { + public void requeueForRouting(Selected selected) { Objects.requireNonNull(selected, "selected"); lock.lock(); try { @@ -240,7 +240,7 @@ public void returnToRouter(Selected selected) { } @Override - public Routed route(Selected selected, Destination destination) { + public Routed recordToRouted(Selected selected, Destination destination) { Objects.requireNonNull(selected, "selected"); Objects.requireNonNull(destination, "destination"); lock.lock(); @@ -271,7 +271,7 @@ public Routed route(Selected selected, Destination destination) { } @Override - public Optional> takeForDestination(String destination, Duration timeout) throws InterruptedException { + public Optional> takeFromRouted(String destination, Duration timeout) throws InterruptedException { Objects.requireNonNull(destination, "destination"); if (destination.isBlank()) { throw new IllegalArgumentException("Destination must not be blank"); @@ -326,7 +326,7 @@ public Optional> takeForDestination(String destination, Duration timeo } @Override - public void returnToDestination(Routed taken) { + public void returnToRouted(Routed taken) { Objects.requireNonNull(taken, "taken"); lock.lock(); try { @@ -484,7 +484,7 @@ public void close() { lifecycle = Lifecycle.CLOSED; return; } - lifecycle = Lifecycle.QUIESCING; + lifecycle = Lifecycle.SHUTTING_DOWN; changed.signalAll(); for (Context context : selections.values()) { if (context.attempt != null || (context.finishing && context.handoffAccepted)) { @@ -591,7 +591,7 @@ private void requireRunning(Role role) { } private void requireActive(Role role) { - if (lifecycle != Lifecycle.RUNNING && lifecycle != Lifecycle.QUIESCING) { + if (lifecycle != Lifecycle.RUNNING && lifecycle != Lifecycle.SHUTTING_DOWN) { throw unavailable(role, "Coordinator is not active"); } } @@ -617,7 +617,7 @@ private static long timeoutNanos(Duration timeout) { } private enum Lifecycle { - NEW, RUNNING, QUIESCING, CLOSED + NEW, RUNNING, SHUTTING_DOWN, CLOSED } private static final class Attempt { diff --git a/sendium-core/src/main/java/gr/cytech/sendium/core/outbound/OutboundCoordinator.java b/sendium-core/src/main/java/gr/cytech/sendium/core/outbound/OutboundCoordinator.java index ac5db96a..35186325 100644 --- a/sendium-core/src/main/java/gr/cytech/sendium/core/outbound/OutboundCoordinator.java +++ b/sendium-core/src/main/java/gr/cytech/sendium/core/outbound/OutboundCoordinator.java @@ -29,41 +29,55 @@ public interface OutboundCoordinator extends AutoClos void start(); /** - * Stops new admission, selection, and takes. Existing routing, handoff, completion, and return - * operations remain valid while the application stops and joins execution. Idempotent. + * Begins shutdown by stopping new acceptance, selection, and takes. Existing routing, handoff, + * completion, and return operations remain valid while the application drains and stops execution. + * This does not drain providers, stop workers, or close stores. Idempotent. */ - void quiesce(); + void beginShutdown(); - void admit(SourceId source, M message); + /** Accepts source ownership and makes the message ready for selection according to the store policy. */ + void accept(SourceId source, M message); /** Accepts source ownership without making this source routable yet. */ - void admitHeld(SourceId source, M message); + void acceptHeld(SourceId source, M message); /** - * Makes one execution message available to bounded selection using the original held source IDs. + * Makes already-accepted held sources available to bounded selection using one prepared execution message. * The caller decides which sources form an assembled message or an individual expired part. - * This is an existing-work transition and remains available during quiescence; new admissions do not. + * This is an existing-work transition and remains available while shutting down; new admissions do not. */ - void publishReady(Set sources, M message); + void makeHeldReady(Set sources, M message); - int selectAndStage(int limit); + /** Selects at most limit eligible pending items into the router backlog, returning the newly staged count. */ + int selectToRouter(int limit); Optional> takeForRouting(Duration timeout) throws InterruptedException; - /** Returns unfinished routing work; this is not a transition from a worker back to routing. */ - void returnToRouter(Selected selected); + /** + * Requeues the current taken, unassigned selection for another routing attempt using its updated payload. + * Use the original take handle. This does not repeat pending selection, choose a destination, or accept + * new input; work with a retained destination intent cannot be requeued through this operation. + */ + void requeueForRouting(Selected selected); /** - * Records one destination before marking selected work routed; dispatch uses takeForDestination. + * Records one already-chosen destination before marking selected work routed. This operation does + * not perform routing lookup, run routing filters, or send to the provider; dispatch uses takeFromRouted. * The selection and its source set must match work owned by this coordinator. Routing integration * must reject copied-route requests as UNSUPPORTED before assignment or dispatch, even if lookup * yields only one destination. It must not drop copies or submit several assignments for one selection. */ - Routed route(Selected selected, Destination destination); + Routed recordToRouted(Selected selected, Destination destination); - Optional> takeForDestination(String destination, Duration timeout) throws InterruptedException; + /** Takes already-routed work for the named destination without completing its stored ownership or sending it. */ + Optional> takeFromRouted(String destination, Duration timeout) throws InterruptedException; - void returnToDestination(Routed work); + /** + * Returns the current unfinished take to routed scheduling at the same recorded destination. + * Use the original take handle with its updated payload. Once terminal completion begins, retry + * handoff/cleanup instead of returning the work for provider redispatch. + */ + void returnToRouted(Routed work); /** * Reports terminal processing of all provider parts for this work, conditional on successful required @@ -83,7 +97,7 @@ public interface OutboundCoordinator extends AutoClos CompletionStage discard(SelectionId selection, CompletionStage requiredHandoff); /** - * After quiesce and application-owned stop/join, returns unfinished work to its owning stage and + * After beginShutdown and application-owned drain/stop/join, returns unfinished work to its owning stage and * closes stores. In-flight provider operations must first be resolved or drained by the application; * closing may not classify them as terminal. Closure is idempotent. Memory closure promises no * restart recovery, and a closed coordinator cannot be restarted. diff --git a/sendium-core/src/main/java/gr/cytech/sendium/core/smpp/server/SmppServerWorker.java b/sendium-core/src/main/java/gr/cytech/sendium/core/smpp/server/SmppServerWorker.java index da2dbcec..7d10c067 100644 --- a/sendium-core/src/main/java/gr/cytech/sendium/core/smpp/server/SmppServerWorker.java +++ b/sendium-core/src/main/java/gr/cytech/sendium/core/smpp/server/SmppServerWorker.java @@ -1024,7 +1024,7 @@ private boolean admitToLifecycle(InEvent event, UUID gatewayMessageId) { throw new OutboundStorageException(OutboundStage.Role.PENDING, OutboundStorageException.Reason.UNSUPPORTED, "Unsupported multipart header"); } - ingressCoordinator.admitHeld(source, event.pMsg); + ingressCoordinator.acceptHeld(source, event.pMsg); synchronized (pendingPartSources) { pendingPartSources.computeIfAbsent(event.pMsg, ignored -> new HashSet<>()).add(source); } @@ -1033,7 +1033,7 @@ private boolean admitToLifecycle(InEvent event, UUID gatewayMessageId) { accepted.notifyClient = event.notifyClient; inEventQueue.add(accepted); } else { - ingressCoordinator.admit(source, event.pMsg); + ingressCoordinator.accept(source, event.pMsg); } if (MessageTrace.shouldLog(configurationProvider, MessageTrace.EVENT_ACCEPTED)) { logger.info("message.accepted ingress=smppserver worker={} {}", getFullName(), MessageTrace.identifiers(event.pMsg)); @@ -1090,7 +1090,7 @@ public void handleIngressMessages(List> events) { throw new IllegalStateException("Accepted ingress event has no lifecycle source ownership"); } if (event.submitSm == null) { - ingressCoordinator.publishReady(event.sourceIds, event.pMsg); + ingressCoordinator.makeHeldReady(event.sourceIds, event.pMsg); } else if (pendingPartSources.containsKey(event.pMsg)) { messagePartsHandler.addMessagePart(event.pMsg); } diff --git a/sendium-core/src/main/java/gr/cytech/sendium/core/storage/PendingMessageStore.java b/sendium-core/src/main/java/gr/cytech/sendium/core/storage/PendingMessageStore.java index 7a5e4447..dd15e829 100644 --- a/sendium-core/src/main/java/gr/cytech/sendium/core/storage/PendingMessageStore.java +++ b/sendium-core/src/main/java/gr/cytech/sendium/core/storage/PendingMessageStore.java @@ -16,18 +16,18 @@ public interface PendingMessageStore extends Outbound */ void admit(SourceId source, M message); - /** Retains an accepted source without making it eligible for selection until publishReady succeeds. */ + /** Retains an accepted source without making it eligible for selection until makeHeldReady succeeds. */ void admitHeld(SourceId source, M message); /** - * Publishes one execution message referring to a nonempty set of admitted held sources. Original + * Makes held sources ready using one execution message referring to their nonempty source set. Original * source records remain authoritative. Publication must bind the whole set before exposing work * to selection; it does not bypass selection limits or create another accepted source. * While all sources remain retained, repeating the exact source set preserves the first published * payload and cannot duplicate work, including after selection/routing. Missing, ordinarily admitted, * or differently grouped sources fail explicitly. The caller owns assembly/expiry policy. */ - void publishReady(Set sources, M message); + void makeHeldReady(Set sources, M message); /** * Called only after all work and required handoffs are terminal. Missing sources are already diff --git a/sendium-core/src/main/java/gr/cytech/sendium/core/storage/SelectedRouterStore.java b/sendium-core/src/main/java/gr/cytech/sendium/core/storage/SelectedRouterStore.java index a5a817b5..e0185099 100644 --- a/sendium-core/src/main/java/gr/cytech/sendium/core/storage/SelectedRouterStore.java +++ b/sendium-core/src/main/java/gr/cytech/sendium/core/storage/SelectedRouterStore.java @@ -13,13 +13,13 @@ */ public interface SelectedRouterStore extends OutboundStage { /** - * Selects and stages at most limit eligible, priority-aware items, returning the newly staged count. + * Selects at most limit eligible, priority-aware items into the router backlog, returning the newly staged count. * Limit must be positive. Selected, taken, and routed sources are ineligible for another selection. * Source ownership is retained, including after partial publication failure. Stable selection IDs * allow retry/reconciliation without repeating already-committed preparation. No cross-store * transaction is implied. The implementation must not load the entire pending backlog. */ - int selectAndStage(int limit); + int selectToRouter(int limit); /** * Takes an execution projection without completing selected or pending ownership. diff --git a/sendium-core/src/main/java/gr/cytech/sendium/core/storage/memory/MemoryPendingMessageStore.java b/sendium-core/src/main/java/gr/cytech/sendium/core/storage/memory/MemoryPendingMessageStore.java index 4d61242e..f9e2abfb 100644 --- a/sendium-core/src/main/java/gr/cytech/sendium/core/storage/memory/MemoryPendingMessageStore.java +++ b/sendium-core/src/main/java/gr/cytech/sendium/core/storage/memory/MemoryPendingMessageStore.java @@ -118,7 +118,7 @@ public void admitHeld(SourceId source, M message) { } @Override - public void publishReady(Set sources, M message) { + public void makeHeldReady(Set sources, M message) { Set checked = Set.copyOf(sources); Objects.requireNonNull(message, "message"); if (checked.isEmpty()) { diff --git a/sendium-core/src/main/java/gr/cytech/sendium/core/storage/memory/MemorySelectedRouterStore.java b/sendium-core/src/main/java/gr/cytech/sendium/core/storage/memory/MemorySelectedRouterStore.java index e9946a9b..a461a6f8 100644 --- a/sendium-core/src/main/java/gr/cytech/sendium/core/storage/memory/MemorySelectedRouterStore.java +++ b/sendium-core/src/main/java/gr/cytech/sendium/core/storage/memory/MemorySelectedRouterStore.java @@ -74,7 +74,7 @@ public Status status() { } @Override - public int selectAndStage(int limit) { + public int selectToRouter(int limit) { if (limit <= 0) { throw new IllegalArgumentException("Selection limit must be positive"); } diff --git a/sendium-core/src/main/java/gr/cytech/sendium/routing/StandardOutboundDispatch.java b/sendium-core/src/main/java/gr/cytech/sendium/routing/StandardOutboundDispatch.java index 57a63804..33a7eaee 100644 --- a/sendium-core/src/main/java/gr/cytech/sendium/routing/StandardOutboundDispatch.java +++ b/sendium-core/src/main/java/gr/cytech/sendium/routing/StandardOutboundDispatch.java @@ -38,7 +38,7 @@ public final class StandardOutboundDispatch implements AutoCloseable { private final boolean ownsScheduler; private final ConcurrentMap, Semaphore> workerSlots = new ConcurrentHashMap<>(); private final ConcurrentMap> activeExecutions = new ConcurrentHashMap<>(); - private boolean quiescing; + private boolean shuttingDown; private final ConcurrentMap routingTransitions = new ConcurrentHashMap<>(); public StandardOutboundDispatch(OutboundCoordinator coordinator, StandardRoutingManager routing) { @@ -76,8 +76,8 @@ public synchronized Optional> routeSelected(Selected> routeSelected(Selected recordRouting(PendingRouting transition) { - var recorded = coordinator.route(transition.selected(), transition.destination()); + var recorded = coordinator.recordToRouted(transition.selected(), transition.destination()); routingTransitions.remove(transition.selected().id(), transition); return recorded; } /** Takes recorded work for provider processing; taking or enqueuing it is not terminal completion. */ public Optional> takeForProvider(String destination, Duration timeout) throws InterruptedException { - return coordinator.takeForDestination(destination, timeout); + return coordinator.takeFromRouted(destination, timeout); } /** Runs an explicitly taken SMPP work item and carries its ownership through replacement attempts. */ @@ -182,9 +182,9 @@ private SmppClientWorker retryDestination(StandardMessage messa } /** Stops new admission/takes and dispatch; existing provider retries and callbacks remain enabled. */ - public synchronized void quiesce() { - coordinator.quiesce(); - quiescing = true; + public synchronized void beginShutdown() { + coordinator.beginShutdown(); + shuttingDown = true; } /** Waits for terminal handoffs and cleanup. A timeout or failed handoff leaves ownership intact. */ @@ -195,8 +195,8 @@ public boolean awaitProviderDrain(Duration timeout) throws InterruptedException } CompletableFuture[] completions; synchronized (this) { - if (!quiescing) { - throw new IllegalStateException("Quiesce dispatch before awaiting provider drain"); + if (!shuttingDown) { + throw new IllegalStateException("Begin dispatch shutdown before awaiting provider drain"); } completions = activeExecutions.values().stream().map(CompletionStage::toCompletableFuture) .toArray(CompletableFuture[]::new); @@ -223,8 +223,8 @@ private CompletionStage track(WorkId id, CompletionStage completion) } private void requireDispatching() { - if (quiescing) { - throw new IllegalStateException("Outbound dispatch is quiescing"); + if (shuttingDown) { + throw new IllegalStateException("Outbound dispatch is shutting down"); } } @@ -238,7 +238,7 @@ private void removeSuccessfulExecutions() { /** Call after drain and execution-loop stop/join; parent stores remain application-owned. */ @Override public synchronized void close() { - quiesce(); + beginShutdown(); removeSuccessfulExecutions(); if (!activeExecutions.isEmpty()) { throw new IllegalStateException("Drain provider executions before closing outbound dispatch"); diff --git a/sendium-core/src/test/java/gr/cytech/sendium/core/http/KannelResourceTest.java b/sendium-core/src/test/java/gr/cytech/sendium/core/http/KannelResourceTest.java index acd9b1ed..d207cbee 100644 --- a/sendium-core/src/test/java/gr/cytech/sendium/core/http/KannelResourceTest.java +++ b/sendium-core/src/test/java/gr/cytech/sendium/core/http/KannelResourceTest.java @@ -119,7 +119,7 @@ void coordinatorAdmissionOwnsTheMessageBeforeHttpSuccess() { Response response = submit("https://callback.test/dlr"); var source = ArgumentCaptor.forClass(SourceId.class); var message = ArgumentCaptor.forClass(StandardMessage.class); - verify(lifecycle.coordinator).admit(source.capture(), message.capture()); + verify(lifecycle.coordinator).accept(source.capture(), message.capture()); assertThat(response.getStatus()).isEqualTo(202); assertThat(source.getValue().value()).isEqualTo(UUID.fromString((String) response.getEntity())); var retained = lifecycle.pending.find(source.getValue()).orElseThrow(); @@ -132,13 +132,13 @@ void coordinatorAdmissionOwnsTheMessageBeforeHttpSuccess() { } @Test - void fullOrQuiescingCoordinatorRejectsWithoutFallingBackToTheQueue() { + void fullOrShuttingDownCoordinatorRejectsWithoutFallingBackToTheQueue() { try (var lifecycle = new OutboundIngressFixture(1)) { when(resource.outboundCoordinators.isUnsatisfied()).thenReturn(false); when(resource.outboundCoordinators.get()).thenReturn(lifecycle.coordinator); assertThat(submit(null).getStatus()).isEqualTo(202); assertThat(submit(null).getStatus()).isEqualTo(503); - lifecycle.coordinator.quiesce(); + lifecycle.coordinator.beginShutdown(); assertThat(submit(null).getStatus()).isEqualTo(503); verifyNoInteractions(routerQueue); } @@ -152,7 +152,7 @@ void mapsStorageFailuresWithoutLeakingTheirDetails(OutboundStorageException.Reas when(resource.outboundCoordinators.isUnsatisfied()).thenReturn(false); when(resource.outboundCoordinators.get()).thenReturn(coordinator); doThrow(new OutboundStorageException(OutboundStage.Role.PENDING, reason, "private storage details")) - .when(coordinator).admit(any(), any()); + .when(coordinator).accept(any(), any()); Response response = submit(null); int expected = switch (reason) { case UNSUPPORTED -> 400; diff --git a/sendium-core/src/test/java/gr/cytech/sendium/core/outbound/DefaultOutboundCoordinatorTest.java b/sendium-core/src/test/java/gr/cytech/sendium/core/outbound/DefaultOutboundCoordinatorTest.java index fde4e6aa..28b460f7 100644 --- a/sendium-core/src/test/java/gr/cytech/sendium/core/outbound/DefaultOutboundCoordinatorTest.java +++ b/sendium-core/src/test/java/gr/cytech/sendium/core/outbound/DefaultOutboundCoordinatorTest.java @@ -54,14 +54,14 @@ class DefaultOutboundCoordinatorTest { void constructionIsPassiveAndStartupOwnsStageOrdering() { try (var h = new Harness()) { verifyNoInteractions(h.pending, h.router, h.routed); - fails(() -> h.coordinator.admit(source(), new Payload("one")), UNAVAILABLE); + fails(() -> h.coordinator.accept(source(), new Payload("one")), UNAVAILABLE); h.start(); h.coordinator.start(); var order = inOrder(h.pending, h.router, h.routed); order.verify(h.pending).open(); order.verify(h.router).open(); order.verify(h.routed).open(); - h.coordinator.quiesce(); + h.coordinator.beginShutdown(); fails(h.coordinator::start, INVALID_TRANSITION); } } @@ -114,7 +114,7 @@ void successfulHandoffCleansUpInOrderAndLateCompletionCannotRecreateWork() throw assertThat(result.toCompletableFuture()).isNotDone(); assertThat(h.pending.find(delivery.source())).isPresent(); assertThat((Object) h.coordinator.complete(delivery.work.id(), CompletableFuture.completedStage(null))).isSameAs(result); - fails(() -> h.coordinator.returnToDestination(delivery.work), INVALID_TRANSITION); + fails(() -> h.coordinator.returnToRouted(delivery.work), INVALID_TRANSITION); handoff.complete(null); result.toCompletableFuture().join(); var order = inOrder(h.routed, h.pending, h.router); @@ -125,8 +125,8 @@ void successfulHandoffCleansUpInOrderAndLateCompletionCannotRecreateWork() throw assertThat(h.pending.find(delivery.source())).isEmpty(); h.coordinator.complete(delivery.work.id(), new CompletableFuture<>()).toCompletableFuture().join(); verify(h.routed, times(1)).complete(delivery.work.id()); - fails(() -> h.coordinator.route(delivery.selected, destination("A", "stale")), INVALID_TRANSITION); - fails(() -> h.coordinator.returnToDestination(delivery.work), INVALID_TRANSITION); + fails(() -> h.coordinator.recordToRouted(delivery.selected, destination("A", "stale")), INVALID_TRANSITION); + fails(() -> h.coordinator.returnToRouted(delivery.work), INVALID_TRANSITION); } } @@ -145,8 +145,8 @@ void failedOrCancelledHandoffRetainsOwnershipForHandoffOnlyRetry(boolean cancel) assertThat(result.toCompletableFuture()).isCompletedExceptionally(); assertThat(h.pending.find(delivery.source())).isPresent(); verify(h.routed, never()).complete(any()); - assertThat(h.coordinator.takeForDestination("A", Duration.ZERO)).isEmpty(); - fails(() -> h.coordinator.returnToDestination(delivery.work), INVALID_TRANSITION); + assertThat(h.coordinator.takeFromRouted("A", Duration.ZERO)).isEmpty(); + fails(() -> h.coordinator.returnToRouted(delivery.work), INVALID_TRANSITION); h.coordinator.complete(delivery.work.id(), CompletableFuture.completedStage(null)).toCompletableFuture().join(); assertThat(h.pending.find(delivery.source())).isEmpty(); } @@ -196,7 +196,7 @@ void completionResultCannotRemoveUnrelatedSources() throws Exception { try (var h = new Harness().start()) { var delivery = h.delivery(); var unrelated = source(); - h.coordinator.admit(unrelated, new Payload("unrelated")); + h.coordinator.accept(unrelated, new Payload("unrelated")); doReturn(new Completed(delivery.selected.id(), Set.of(unrelated))) .when(h.routed).complete(delivery.work.id()); var completion = h.coordinator.complete(delivery.work.id(), CompletableFuture.completedStage(null)); @@ -220,7 +220,7 @@ void returnedFutureCannotCancelOwnedCleanupAndCallbacksRunOutsideTheLock() throw var nextSource = source(); var callback = completion.thenRun(() -> { try { - executor.submit(() -> h.coordinator.admit(nextSource, new Payload("next"))).get(1, TimeUnit.SECONDS); + executor.submit(() -> h.coordinator.accept(nextSource, new Payload("next"))).get(1, TimeUnit.SECONDS); } catch (Exception failure) { throw new IllegalStateException(failure); } @@ -249,14 +249,14 @@ void routingRetryReconcilesPublishedDestinationWithoutRoutingAgain(String bounda } else { doThrow(new IllegalStateException("mark failed")).when(h.router).markRouted(selected.id()); } - assertThatThrownBy(() -> h.coordinator.route(selected, destination("A", "chosen"))) + assertThatThrownBy(() -> h.coordinator.recordToRouted(selected, destination("A", "chosen"))) .isInstanceOf(IllegalStateException.class); - fails(() -> h.coordinator.returnToRouter(selected), INVALID_TRANSITION); - fails(() -> h.coordinator.route(selected, destination("B", "different")), INVALID_TRANSITION); - fails(() -> h.coordinator.takeForDestination("A", Duration.ZERO), UNAVAILABLE); + fails(() -> h.coordinator.requeueForRouting(selected), INVALID_TRANSITION); + fails(() -> h.coordinator.recordToRouted(selected, destination("B", "different")), INVALID_TRANSITION); + fails(() -> h.coordinator.takeFromRouted("A", Duration.ZERO), UNAVAILABLE); doCallRealMethod().when(h.router).markRouted(selected.id()); - var receipt = h.coordinator.route(selected, destination("A", "retry payload")); - var taken = h.coordinator.takeForDestination("A", Duration.ZERO).orElseThrow(); + var receipt = h.coordinator.recordToRouted(selected, destination("A", "retry payload")); + var taken = h.coordinator.takeFromRouted("A", Duration.ZERO).orElseThrow(); assertThat(taken.id()).isEqualTo(receipt.id()); assertThat(taken.message().body).isEqualTo("chosen"); h.coordinator.complete(taken.id(), CompletableFuture.completedStage(null)).toCompletableFuture().join(); @@ -267,14 +267,14 @@ void routingRetryReconcilesPublishedDestinationWithoutRoutingAgain(String bounda void destinationCapacityBackpressureDoesNotPreventExistingWorkFromCompleting() throws Exception { try (var h = new Harness(1).start()) { var first = h.select("first"); - h.coordinator.route(first, destination("A", "first")); + h.coordinator.recordToRouted(first, destination("A", "first")); var second = h.select("second"); - fails(() -> h.coordinator.route(second, destination("A", "second")), CAPACITY_EXCEEDED); - var taken = h.coordinator.takeForDestination("A", Duration.ZERO).orElseThrow(); + fails(() -> h.coordinator.recordToRouted(second, destination("A", "second")), CAPACITY_EXCEEDED); + var taken = h.coordinator.takeFromRouted("A", Duration.ZERO).orElseThrow(); assertThat(taken.selection()).isEqualTo(first.id()); h.coordinator.complete(taken.id(), CompletableFuture.completedStage(null)).toCompletableFuture().join(); - h.coordinator.route(second, destination("A", "retry")); - taken = h.coordinator.takeForDestination("A", Duration.ZERO).orElseThrow(); + h.coordinator.recordToRouted(second, destination("A", "retry")); + taken = h.coordinator.takeFromRouted("A", Duration.ZERO).orElseThrow(); assertThat(taken.message().body).isEqualTo("second"); h.coordinator.complete(taken.id(), CompletableFuture.completedStage(null)).toCompletableFuture().join(); } @@ -284,9 +284,9 @@ void destinationCapacityBackpressureDoesNotPreventExistingWorkFromCompleting() t void queuedReceiptsCannotCompleteBeforeTheyAreTaken() throws Exception { try (var h = new Harness().start()) { var selected = h.select("one"); - var receipt = h.coordinator.route(selected, destination("A", "one")); + var receipt = h.coordinator.recordToRouted(selected, destination("A", "one")); fails(() -> h.coordinator.complete(receipt.id(), CompletableFuture.completedStage(null)), INVALID_TRANSITION); - var taken = h.coordinator.takeForDestination("A", Duration.ZERO).orElseThrow(); + var taken = h.coordinator.takeFromRouted("A", Duration.ZERO).orElseThrow(); h.coordinator.complete(taken.id(), CompletableFuture.completedStage(null)).toCompletableFuture().join(); } } @@ -296,37 +296,37 @@ void unexposedTakeSurvivesFailureWhileReturningAnUnfinishedRoute() throws Except try (var h = new Harness().start()) { var selected = h.select("one"); doThrow(new IllegalStateException("mark failed")).when(h.router).markRouted(selected.id()); - assertThatThrownBy(() -> h.coordinator.route(selected, destination("A", "one"))).isInstanceOf(IllegalStateException.class); + assertThatThrownBy(() -> h.coordinator.recordToRouted(selected, destination("A", "one"))).isInstanceOf(IllegalStateException.class); doThrow(new IllegalStateException("return failed")).when(h.routed).release(any()); - assertThatThrownBy(() -> h.coordinator.takeForDestination("A", Duration.ZERO)).isInstanceOf(IllegalStateException.class); + assertThatThrownBy(() -> h.coordinator.takeFromRouted("A", Duration.ZERO)).isInstanceOf(IllegalStateException.class); doCallRealMethod().when(h.router).markRouted(selected.id()); doCallRealMethod().when(h.routed).release(any()); - h.coordinator.route(selected, destination("A", "one")); - var taken = h.coordinator.takeForDestination("A", Duration.ZERO).orElseThrow(); + h.coordinator.recordToRouted(selected, destination("A", "one")); + var taken = h.coordinator.takeFromRouted("A", Duration.ZERO).orElseThrow(); h.coordinator.complete(taken.id(), CompletableFuture.completedStage(null)).toCompletableFuture().join(); } } @Test - void routerAndDestinationReturnsPreserveMutationsAndRejectForgedTakes() throws Exception { + void routingRequeueAndRoutedReturnPreserveMutationsAndRejectForgedTakes() throws Exception { try (var h = new Harness().start()) { var selected = h.select("original"); selected.message().body = "filtered"; - h.coordinator.returnToRouter(selected); + h.coordinator.requeueForRouting(selected); var next = h.coordinator.takeForRouting(Duration.ZERO).orElseThrow(); - h.coordinator.returnToRouter(selected); + h.coordinator.requeueForRouting(selected); assertThat(next.message().body).isEqualTo("filtered"); assertThat(h.coordinator.takeForRouting(Duration.ZERO)).isEmpty(); - fails(() -> h.coordinator.route(new Selected<>(next.id(), next.sources(), next.message()), destination("A", "fake")), + fails(() -> h.coordinator.recordToRouted(new Selected<>(next.id(), next.sources(), next.message()), destination("A", "fake")), INVALID_TRANSITION); - h.coordinator.route(next, new Destination<>("A", next.message())); - var first = h.coordinator.takeForDestination("A", Duration.ZERO).orElseThrow(); + h.coordinator.recordToRouted(next, new Destination<>("A", next.message())); + var first = h.coordinator.takeFromRouted("A", Duration.ZERO).orElseThrow(); first.message().body = "retry"; - h.coordinator.returnToDestination(first); - var second = h.coordinator.takeForDestination("A", Duration.ZERO).orElseThrow(); - h.coordinator.returnToDestination(first); + h.coordinator.returnToRouted(first); + var second = h.coordinator.takeFromRouted("A", Duration.ZERO).orElseThrow(); + h.coordinator.returnToRouted(first); assertThat(second.message().body).isEqualTo("retry"); - assertThat(h.coordinator.takeForDestination("A", Duration.ZERO)).isEmpty(); + assertThat(h.coordinator.takeFromRouted("A", Duration.ZERO)).isEmpty(); h.coordinator.complete(second.id(), CompletableFuture.completedStage(null)).toCompletableFuture().join(); } } @@ -338,7 +338,7 @@ void discardWaitsForHandoffAndCannotBeUsedAfterAssignment() throws Exception { var handoff = new CompletableFuture(); var result = h.coordinator.discard(selected.id(), handoff); assertThat(h.pending.find(selected.sources().iterator().next())).isPresent(); - fails(() -> h.coordinator.returnToRouter(selected), INVALID_TRANSITION); + fails(() -> h.coordinator.requeueForRouting(selected), INVALID_TRANSITION); handoff.completeExceptionally(new IllegalStateException("handoff failed")); assertThat(result.toCompletableFuture()).isCompletedExceptionally(); h.coordinator.discard(selected.id(), CompletableFuture.completedStage(null)).toCompletableFuture().join(); @@ -358,13 +358,13 @@ void multiSourceCleanupCanRetryAfterPartialRemoval() throws Exception { var sources = Set.of(source(), source()); try (var coordinator = new DefaultOutboundCoordinator<>(pending, router, routed)) { coordinator.start(); - sources.forEach(source -> coordinator.admitHeld(source, new Payload("part"))); - coordinator.publishReady(sources, new Payload("aggregate")); - coordinator.selectAndStage(1); + sources.forEach(source -> coordinator.acceptHeld(source, new Payload("part"))); + coordinator.makeHeldReady(sources, new Payload("aggregate")); + coordinator.selectToRouter(1); var selected = coordinator.takeForRouting(Duration.ZERO).orElseThrow(); assertThat(selected.sources()).isEqualTo(sources); - coordinator.route(selected, destination("A", "aggregate")); - var taken = coordinator.takeForDestination("A", Duration.ZERO).orElseThrow(); + coordinator.recordToRouted(selected, destination("A", "aggregate")); + var taken = coordinator.takeFromRouted("A", Duration.ZERO).orElseThrow(); AtomicBoolean first = new AtomicBoolean(true); doAnswer(call -> { if (first.getAndSet(false)) { @@ -376,10 +376,10 @@ void multiSourceCleanupCanRetryAfterPartialRemoval() throws Exception { var result = coordinator.complete(taken.id(), CompletableFuture.completedStage(null)); assertThat(result.toCompletableFuture()).isCompletedExceptionally(); assertThat(sources.stream().filter(source -> pending.find(source).isPresent()).count()).isEqualTo(1); - sources.forEach(source -> coordinator.admit(source, new Payload("duplicate during cleanup"))); - sources.forEach(source -> coordinator.admitHeld(source, new Payload("held duplicate during cleanup"))); + sources.forEach(source -> coordinator.accept(source, new Payload("duplicate during cleanup"))); + sources.forEach(source -> coordinator.acceptHeld(source, new Payload("held duplicate during cleanup"))); assertThat(sources.stream().filter(source -> pending.find(source).isPresent()).count()).isEqualTo(1); - fails(() -> coordinator.publishReady(sources, new Payload("late publication")), INVALID_TRANSITION); + fails(() -> coordinator.makeHeldReady(sources, new Payload("late publication")), INVALID_TRANSITION); coordinator.complete(taken.id(), new CompletableFuture<>()).toCompletableFuture().join(); sources.forEach(source -> assertThat(pending.find(source)).isEmpty()); verify(routed, times(1)).complete(taken.id()); @@ -389,20 +389,20 @@ void multiSourceCleanupCanRetryAfterPartialRemoval() throws Exception { @ParameterizedTest @ValueSource(booleans = {false, true}) - void quiescenceWakesWaitingTakesAndKeepsCompletionAvailable(boolean destination) throws Exception { + void beginningShutdownWakesWaitingTakesAndKeepsCompletionAvailable(boolean destination) throws Exception { try (var h = new Harness().start(); var executor = Executors.newVirtualThreadPerTaskExecutor()) { var delivery = h.delivery(); AtomicReference thread = new AtomicReference<>(); var waiting = executor.submit(() -> { thread.set(Thread.currentThread()); - return destination ? h.coordinator.takeForDestination("empty", Duration.ofSeconds(3)) : + return destination ? h.coordinator.takeFromRouted("empty", Duration.ofSeconds(3)) : h.coordinator.takeForRouting(Duration.ofSeconds(3)); }); awaitWaiting(thread); - h.coordinator.quiesce(); + h.coordinator.beginShutdown(); assertThatThrownBy(() -> waiting.get(1, TimeUnit.SECONDS)).hasCauseInstanceOf(OutboundStorageException.class); - fails(() -> h.coordinator.admit(source(), new Payload("new")), UNAVAILABLE); - fails(() -> h.coordinator.selectAndStage(1), UNAVAILABLE); + fails(() -> h.coordinator.accept(source(), new Payload("new")), UNAVAILABLE); + fails(() -> h.coordinator.selectToRouter(1), UNAVAILABLE); h.coordinator.complete(delivery.work.id(), CompletableFuture.completedStage(null)).toCompletableFuture().join(); } } @@ -416,16 +416,16 @@ void publicationWakesCoordinatorTakes() throws Exception { return h.coordinator.takeForRouting(Duration.ofSeconds(3)); }); awaitWaiting(thread); - h.coordinator.admit(source(), new Payload("one")); - h.coordinator.selectAndStage(1); + h.coordinator.accept(source(), new Payload("one")); + h.coordinator.selectToRouter(1); var selected = waiting.get(1, TimeUnit.SECONDS).orElseThrow(); thread.set(null); var destination = executor.submit(() -> { thread.set(Thread.currentThread()); - return h.coordinator.takeForDestination("A", Duration.ofSeconds(3)); + return h.coordinator.takeFromRouted("A", Duration.ofSeconds(3)); }); awaitWaiting(thread); - h.coordinator.route(selected, destination("A", "one")); + h.coordinator.recordToRouted(selected, destination("A", "one")); var taken = destination.get(1, TimeUnit.SECONDS).orElseThrow(); h.coordinator.complete(taken.id(), CompletableFuture.completedStage(null)).toCompletableFuture().join(); } @@ -436,7 +436,7 @@ void closeDrainsWorkBackToItsStageWithoutCompletingSources() throws Exception { try (var h = new Harness().start()) { var delivery = h.delivery(); var routing = h.select("not routed"); - h.coordinator.quiesce(); + h.coordinator.beginShutdown(); h.coordinator.close(); var order = inOrder(h.router, h.routed, h.pending); order.verify(h.router).release(routing); @@ -570,15 +570,15 @@ private Harness start() { } private Selected select(String body) throws InterruptedException { - coordinator.admit(source(), new Payload(body)); - coordinator.selectAndStage(1); + coordinator.accept(source(), new Payload(body)); + coordinator.selectToRouter(1); return coordinator.takeForRouting(Duration.ZERO).orElseThrow(); } private Delivery delivery() throws InterruptedException { var selected = select("original"); - coordinator.route(selected, new Destination<>("A", selected.message())); - return new Delivery(selected, coordinator.takeForDestination("A", Duration.ZERO).orElseThrow()); + coordinator.recordToRouted(selected, new Destination<>("A", selected.message())); + return new Delivery(selected, coordinator.takeFromRouted("A", Duration.ZERO).orElseThrow()); } @Override diff --git a/sendium-core/src/test/java/gr/cytech/sendium/core/outbound/consumer/OutboundEmbeddedLifecycleTest.java b/sendium-core/src/test/java/gr/cytech/sendium/core/outbound/consumer/OutboundEmbeddedLifecycleTest.java index b65bb63b..415a52e1 100644 --- a/sendium-core/src/test/java/gr/cytech/sendium/core/outbound/consumer/OutboundEmbeddedLifecycleTest.java +++ b/sendium-core/src/test/java/gr/cytech/sendium/core/outbound/consumer/OutboundEmbeddedLifecycleTest.java @@ -41,21 +41,21 @@ void customMessagesSurviveApplicationSelectionPolicyReturnsAndAsynchronousHandof var firstSource = source(); var secondSource = source(); var first = message("custom-1"); - coordinator.admit(firstSource, first); - coordinator.admit(secondSource, message("custom-2")); + coordinator.accept(firstSource, first); + coordinator.accept(secondSource, message("custom-2")); first.labels.add("caller mutation"); // The application-owned port narrows the requested batch without a core configuration or CDI container. - assertThat(coordinator.selectAndStage(10)).isEqualTo(1); + assertThat(coordinator.selectToRouter(10)).isEqualTo(1); assertThat(selected.requestedLimit).isEqualTo(10); var selection = coordinator.takeForRouting(Duration.ZERO).orElseThrow(); assertThat(selection.message().externalId).isEqualTo("custom-1"); assertThat(selection.message().labels).containsExactly("accepted"); - coordinator.route(selection, new Destination<>("application-provider", selection.message())); - var work = coordinator.takeForDestination("application-provider", Duration.ZERO).orElseThrow(); + coordinator.recordToRouted(selection, new Destination<>("application-provider", selection.message())); + var work = coordinator.takeFromRouted("application-provider", Duration.ZERO).orElseThrow(); work.message().labels.add("worker mutation"); - coordinator.returnToDestination(work); - var returned = coordinator.takeForDestination("application-provider", Duration.ZERO).orElseThrow(); + coordinator.returnToRouted(work); + var returned = coordinator.takeFromRouted("application-provider", Duration.ZERO).orElseThrow(); assertThat(returned.id()).isEqualTo(work.id()); assertThat(returned.message().externalId).isEqualTo("custom-1"); assertThat(returned.message().labels).containsExactly("accepted", "worker mutation"); @@ -67,7 +67,7 @@ void customMessagesSurviveApplicationSelectionPolicyReturnsAndAsynchronousHandof handoff.completeExceptionally(new IllegalStateException("application handoff failed")); assertThatThrownBy(() -> completion.toCompletableFuture().join()).hasCauseInstanceOf(IllegalStateException.class); assertThat(pending.find(firstSource)).isPresent(); - assertThat(coordinator.takeForDestination("application-provider", Duration.ZERO)).isEmpty(); + assertThat(coordinator.takeFromRouted("application-provider", Duration.ZERO)).isEmpty(); coordinator.complete(returned.id(), CompletableFuture.completedStage(null)).toCompletableFuture().join(); assertThat(pending.find(firstSource)).isEmpty(); assertThat(pending.find(secondSource).orElseThrow().externalId).isEqualTo("custom-2"); @@ -75,7 +75,7 @@ void customMessagesSurviveApplicationSelectionPolicyReturnsAndAsynchronousHandof // A late terminal callback cannot remove the other accepted source. coordinator.complete(returned.id(), CompletableFuture.completedStage(null)).toCompletableFuture().join(); assertThat(pending.find(secondSource)).isPresent(); - assertThat(coordinator.selectAndStage(10)).isEqualTo(1); + assertThat(coordinator.selectToRouter(10)).isEqualTo(1); var second = coordinator.takeForRouting(Duration.ZERO).orElseThrow(); assertThat(second.message().externalId).isEqualTo("custom-2"); coordinator.discard(second.id(), CompletableFuture.completedStage(null)).toCompletableFuture().join(); @@ -90,20 +90,20 @@ void applicationOwnedShutdownReturnsTakenCustomWorkWithoutDeclaringItCompleted() var selected = new ApplicationSelectedStore(router); var routed = spy(new MemoryRoutedWorkStore(2, OutboundEmbeddedLifecycleTest::snapshot)); try (var coordinator = new DefaultOutboundCoordinator<>(pending, selected, routed)) { - assertThatThrownBy(() -> coordinator.admit(source(), message("before-start"))) + assertThatThrownBy(() -> coordinator.accept(source(), message("before-start"))) .isInstanceOf(OutboundStorageException.class); coordinator.start(); - coordinator.admit(source(), message("routed")); - coordinator.selectAndStage(1); + coordinator.accept(source(), message("routed")); + coordinator.selectToRouter(1); var initial = coordinator.takeForRouting(Duration.ZERO).orElseThrow(); - coordinator.route(initial, new Destination<>("custom-provider", initial.message())); - var destinationTake = coordinator.takeForDestination("custom-provider", Duration.ZERO).orElseThrow(); - coordinator.admit(source(), message("unassigned")); - coordinator.selectAndStage(1); + coordinator.recordToRouted(initial, new Destination<>("custom-provider", initial.message())); + var destinationTake = coordinator.takeFromRouted("custom-provider", Duration.ZERO).orElseThrow(); + coordinator.accept(source(), message("unassigned")); + coordinator.selectToRouter(1); var routerTake = coordinator.takeForRouting(Duration.ZERO).orElseThrow(); - coordinator.quiesce(); - assertThatThrownBy(() -> coordinator.admit(source(), message("after-quiesce"))) + coordinator.beginShutdown(); + assertThatThrownBy(() -> coordinator.accept(source(), message("after-shutdown-start"))) .isInstanceOf(OutboundStorageException.class); coordinator.close(); verify(router).release(routerTake); @@ -150,9 +150,9 @@ private ApplicationSelectedStore(SelectedRouterStore delegate) { } @Override - public int selectAndStage(int limit) { + public int selectToRouter(int limit) { requestedLimit = limit; - return delegate.selectAndStage(Math.min(limit, 1)); + return delegate.selectToRouter(Math.min(limit, 1)); } @Override diff --git a/sendium-core/src/test/java/gr/cytech/sendium/core/smpp/client/SmppClientWorkerTest.java b/sendium-core/src/test/java/gr/cytech/sendium/core/smpp/client/SmppClientWorkerTest.java index 6df0a616..bd4e4a49 100644 --- a/sendium-core/src/test/java/gr/cytech/sendium/core/smpp/client/SmppClientWorkerTest.java +++ b/sendium-core/src/test/java/gr/cytech/sendium/core/smpp/client/SmppClientWorkerTest.java @@ -64,15 +64,15 @@ void heldSourcesCopiedRouteRejectionMultipartReroutingAndHandoffRetryRemainOwned var secondPart = messageWithNetwork(); secondPart.serial = secondSource.value().toString(); secondPart.body = "a".repeat(100); - fixture.coordinator.admitHeld(firstSource, firstPart); - fixture.coordinator.admitHeld(secondSource, secondPart); - assertThat(fixture.coordinator.selectAndStage(2)).isZero(); + fixture.coordinator.acceptHeld(firstSource, firstPart); + fixture.coordinator.acceptHeld(secondSource, secondPart); + assertThat(fixture.coordinator.selectToRouter(2)).isZero(); var prepared = messageWithNetwork(); prepared.serial = firstPart.serial; prepared.body = firstPart.body + secondPart.body; prepared.reassembledParts = new java.util.ArrayList<>(java.util.List.of(firstPart.serial, secondPart.serial)); - fixture.coordinator.publishReady(Set.of(firstSource, secondSource), prepared); - assertThat(fixture.coordinator.selectAndStage(2)).isEqualTo(1); + fixture.coordinator.makeHeldReady(Set.of(firstSource, secondSource), prepared); + assertThat(fixture.coordinator.selectToRouter(2)).isEqualTo(1); var selected = fixture.coordinator.takeForRouting(Duration.ZERO).orElseThrow(); assertThatThrownBy(() -> dispatch.routeSelected(selected)) .isInstanceOfSatisfying(OutboundStorageException.class, failure -> @@ -88,7 +88,7 @@ void heldSourcesCopiedRouteRejectionMultipartReroutingAndHandoffRetryRemainOwned var execution = dispatch.submitToProvider(parent, first); first.awaitRequests(2); routing.use(second.getFullName() + "::default:"); - dispatch.quiesce(); + dispatch.beginShutdown(); assertThat(dispatch.awaitProviderDrain(Duration.ZERO)).isFalse(); respond(handler(first), first.coordinatedRequests.getFirst(), SmppConstants.STATUS_INVDSTADR); second.awaitRequests(2); @@ -134,14 +134,14 @@ void workerSlotAndRateGatePreventConcurrentPreparationAndEarlySending() throws E for (int index = 0; index < 2; index++) { var message = messageWithNetwork(); message.body = "hello"; - fixture.coordinator.admit(new SourceId(UUID.randomUUID()), message); + fixture.coordinator.accept(new SourceId(UUID.randomUUID()), message); } - fixture.coordinator.selectAndStage(2); + fixture.coordinator.selectToRouter(2); for (int index = 0; index < 2; index++) { var selected = fixture.coordinator.takeForRouting(Duration.ZERO).orElseThrow(); - fixture.coordinator.route(selected, new Destination<>(worker.getFullName(), selected.message())); + fixture.coordinator.recordToRouted(selected, new Destination<>(worker.getFullName(), selected.message())); executions.add(dispatch.submitToProvider( - fixture.coordinator.takeForDestination(worker.getFullName(), Duration.ZERO).orElseThrow(), worker)); + fixture.coordinator.takeFromRouted(worker.getFullName(), Duration.ZERO).orElseThrow(), worker)); } assertThat(worker.rateEntered.await(5, TimeUnit.SECONDS)).isTrue(); scheduler.submit(() -> { }).get(5, TimeUnit.SECONDS); @@ -178,14 +178,14 @@ void failedRetryLookupCanResumeWithoutLosingParentOrRepeatingOriginalSubmission( var message = messageWithNetwork(); message.body = "hello"; SourceId source = new SourceId(UUID.randomUUID()); - fixture.coordinator.admit(source, message); - fixture.coordinator.selectAndStage(1); + fixture.coordinator.accept(source, message); + fixture.coordinator.selectToRouter(1); var selected = fixture.coordinator.takeForRouting(Duration.ZERO).orElseThrow(); - fixture.coordinator.route(selected, new Destination<>(worker.getFullName(), selected.message())); - var work = fixture.coordinator.takeForDestination(worker.getFullName(), Duration.ZERO).orElseThrow(); + fixture.coordinator.recordToRouted(selected, new Destination<>(worker.getFullName(), selected.message())); + var work = fixture.coordinator.takeFromRouted(worker.getFullName(), Duration.ZERO).orElseThrow(); var execution = dispatch.submitToProvider(work, worker); worker.awaitRequests(1); - dispatch.quiesce(); + dispatch.beginShutdown(); assertThat(dispatch.awaitProviderDrain(Duration.ZERO)).isFalse(); assertThatThrownBy(() -> dispatch.submitToProvider(work, worker)).isInstanceOf(IllegalStateException.class); respond(handler(worker), worker.coordinatedRequests.getFirst(), SmppConstants.STATUS_INVDSTADR); @@ -271,11 +271,11 @@ void routerRetryTracksAnotherProviderWhileOriginalMultipartRequestIsOutstanding( SourceId source = new SourceId(UUID.randomUUID()); var message = messageWithNetwork(); message.body = "a".repeat(200); - fixture.coordinator.admit(source, message); - fixture.coordinator.selectAndStage(1); + fixture.coordinator.accept(source, message); + fixture.coordinator.selectToRouter(1); var selected = fixture.coordinator.takeForRouting(Duration.ZERO).orElseThrow(); - fixture.coordinator.route(selected, new Destination<>(first.getFullName(), selected.message())); - var parent = fixture.coordinator.takeForDestination(first.getFullName(), Duration.ZERO).orElseThrow(); + fixture.coordinator.recordToRouted(selected, new Destination<>(first.getFullName(), selected.message())); + var parent = fixture.coordinator.takeFromRouted(first.getFullName(), Duration.ZERO).orElseThrow(); var execution = retryDispatch.submitToProvider(parent, first); first.awaitRequests(2); respond(handler(first), first.coordinatedRequests.getFirst(), SmppConstants.STATUS_INVDSTADR); @@ -306,12 +306,12 @@ void pausedAttemptWaitsWithoutSendingAndPreparesAfterResume() throws Exception { var worker = new TestSmppClientWorker(new TestConfigurationProvider(), new Queue<>(), new CapturingTracker()); worker.testPaused = true; SourceId source = new SourceId(UUID.randomUUID()); - fixture.coordinator.admit(source, messageWithNetwork()); - fixture.coordinator.selectAndStage(1); + fixture.coordinator.accept(source, messageWithNetwork()); + fixture.coordinator.selectToRouter(1); var selected = fixture.coordinator.takeForRouting(Duration.ZERO).orElseThrow(); selected.message().body = "hello"; - fixture.coordinator.route(selected, new Destination<>(worker.getFullName(), selected.message())); - var work = fixture.coordinator.takeForDestination(worker.getFullName(), Duration.ZERO).orElseThrow(); + fixture.coordinator.recordToRouted(selected, new Destination<>(worker.getFullName(), selected.message())); + var work = fixture.coordinator.takeFromRouted(worker.getFullName(), Duration.ZERO).orElseThrow(); var execution = dispatch.submitToProvider(work, worker); assertThat(worker.pauseChecked.await(5, TimeUnit.SECONDS)).isTrue(); assertThat(worker.coordinatedRequests).isEmpty(); @@ -395,11 +395,11 @@ void firstPartRouterRetryKeepsParentSourcesThroughOriginalAndReplacementParts() var message = messageWithNetwork(); message.body = "a".repeat(200); SourceId source = new SourceId(UUID.randomUUID()); - fixture.coordinator.admit(source, message); - fixture.coordinator.selectAndStage(1); + fixture.coordinator.accept(source, message); + fixture.coordinator.selectToRouter(1); var selected = fixture.coordinator.takeForRouting(Duration.ZERO).orElseThrow(); - fixture.coordinator.route(selected, new Destination<>(worker.getFullName(), selected.message())); - var work = fixture.coordinator.takeForDestination(worker.getFullName(), Duration.ZERO).orElseThrow(); + fixture.coordinator.recordToRouted(selected, new Destination<>(worker.getFullName(), selected.message())); + var work = fixture.coordinator.takeFromRouted(worker.getFullName(), Duration.ZERO).orElseThrow(); var routing = mock(StandardRoutingManager.class); when(routing.lookupForLifecycle(org.mockito.ArgumentMatchers.any())).thenReturn( new RoutingLookupResult(java.util.List.of(worker), true)); @@ -428,11 +428,11 @@ void coordinatedSessionCallbacksCompleteRealLifecycleOnlyAfterEveryPart() throws var message = messageWithNetwork(); message.body = "a".repeat(200); SourceId source = new SourceId(UUID.randomUUID()); - fixture.coordinator.admit(source, message); - fixture.coordinator.selectAndStage(1); + fixture.coordinator.accept(source, message); + fixture.coordinator.selectToRouter(1); var selected = fixture.coordinator.takeForRouting(Duration.ZERO).orElseThrow(); - fixture.coordinator.route(selected, new Destination<>(worker.getFullName(), selected.message())); - var work = fixture.coordinator.takeForDestination(worker.getFullName(), Duration.ZERO).orElseThrow(); + fixture.coordinator.recordToRouted(selected, new Destination<>(worker.getFullName(), selected.message())); + var work = fixture.coordinator.takeFromRouted(worker.getFullName(), Duration.ZERO).orElseThrow(); var dispatch = new StandardOutboundDispatch(fixture.coordinator, mock(StandardRoutingManager.class)); var execution = dispatch.submitToProvider(work, worker); worker.awaitRequests(2); diff --git a/sendium-core/src/test/java/gr/cytech/sendium/core/smpp/server/SmppLifecycleAdmissionTest.java b/sendium-core/src/test/java/gr/cytech/sendium/core/smpp/server/SmppLifecycleAdmissionTest.java index e7b607f3..0a66c3ab 100644 --- a/sendium-core/src/test/java/gr/cytech/sendium/core/smpp/server/SmppLifecycleAdmissionTest.java +++ b/sendium-core/src/test/java/gr/cytech/sendium/core/smpp/server/SmppLifecycleAdmissionTest.java @@ -45,7 +45,7 @@ void completeSubmissionIsOwnedBeforeStatusOkAndDoesNotEnterLegacyQueues() throws var event = submission(null, "hello", 42); worker.onResponse = response -> { var source = ArgumentCaptor.forClass(SourceId.class); - verify(lifecycle.coordinator).admit(source.capture(), eq(event.pMsg)); + verify(lifecycle.coordinator).accept(source.capture(), eq(event.pMsg)); assertThat(source.getValue().value()).isEqualTo(UUID.fromString(response.getMessageId())); assertThat(lifecycle.pending.find(source.getValue()).orElseThrow().serial).isEqualTo(response.getMessageId()); }; @@ -58,7 +58,7 @@ void completeSubmissionIsOwnedBeforeStatusOkAndDoesNotEnterLegacyQueues() throws assertThat(event.waitingForResponse).isFalse(); assertThat(worker.getInEventQueue()).isEmpty(); assertThat(worker.legacyQueue.isEmpty()).isTrue(); - assertThat(lifecycle.coordinator.selectAndStage(1)).isEqualTo(1); + assertThat(lifecycle.coordinator.selectToRouter(1)).isEqualTo(1); var selected = lifecycle.coordinator.takeForRouting(Duration.ZERO).orElseThrow(); assertThat(selected.message().dlrReturnMetadata).isEqualTo(event.pMsg.dlrReturnMetadata); assertThat(selected.message().ctstamp).isEqualTo(event.localTimestamp.getTime()); @@ -76,14 +76,14 @@ void multipartPartsAreAcknowledgedIndividuallyThenPublishedWithAllSources() thro assertThat(accepted.sourceIds.iterator().next().value()).isEqualTo( UUID.fromString(response.getMessageId())); accepted.sourceIds.forEach(source -> assertThat(lifecycle.pending.find(source)).isPresent()); - assertThat(lifecycle.coordinator.selectAndStage(2)).isZero(); + assertThat(lifecycle.coordinator.selectToRouter(2)).isZero(); }; var first = submission("0500037F0201", "Hello ", 1); worker.enqueueIn(first); var firstAccepted = worker.getInEventQueue().remove(); worker.handleIngressMessages(List.of(firstAccepted)); assertThat(worker.responses).hasSize(1); - assertThat(lifecycle.coordinator.selectAndStage(2)).isZero(); + assertThat(lifecycle.coordinator.selectToRouter(2)).isZero(); var second = submission("0500037F0202", "world", 2); worker.enqueueIn(second); var secondAccepted = worker.getInEventQueue().remove(); @@ -100,12 +100,12 @@ void multipartPartsAreAcknowledgedIndividuallyThenPublishedWithAllSources() thro assertThat(ready.pMsg.reassembledParts).containsExactly(first.pMsg.serial, second.pMsg.serial); assertThat(lifecycle.pending.find(firstAccepted.sourceIds.iterator().next()).orElseThrow().body).isEqualTo("Hello "); worker.handleIngressMessages(List.of(ready)); - assertThat(lifecycle.coordinator.selectAndStage(2)).isEqualTo(1); + assertThat(lifecycle.coordinator.selectToRouter(2)).isEqualTo(1); var selected = lifecycle.coordinator.takeForRouting(Duration.ZERO).orElseThrow(); assertThat(selected.sources()).isEqualTo(sources); assertThat(selected.message().body).isEqualTo("Hello world"); - lifecycle.coordinator.route(selected, new Destination<>("provider", selected.message())); - var work = lifecycle.coordinator.takeForDestination("provider", Duration.ZERO).orElseThrow(); + lifecycle.coordinator.recordToRouted(selected, new Destination<>("provider", selected.message())); + var work = lifecycle.coordinator.takeFromRouted("provider", Duration.ZERO).orElseThrow(); lifecycle.coordinator.complete(work.id(), CompletableFuture.completedStage(null)).toCompletableFuture().join(); sources.forEach(source -> assertThat(lifecycle.pending.find(source)).isEmpty()); assertThat(worker.responses).hasSize(2).allSatisfy(response -> @@ -124,14 +124,14 @@ void publicationFailureRetriesPreparedWorkWithoutAnotherAcknowledgementOrReassem var ready = worker.getInEventQueue().remove(); doThrow(new OutboundStorageException(OutboundStage.Role.PENDING, OutboundStorageException.Reason.UNAVAILABLE, "publication unavailable")) - .when(lifecycle.coordinator).publishReady(any(), any()); + .when(lifecycle.coordinator).makeHeldReady(any(), any()); worker.handleIngressMessages(List.of(ready)); assertThat(worker.getInEventQueue()).containsExactly(ready); - assertThat(lifecycle.coordinator.selectAndStage(2)).isZero(); + assertThat(lifecycle.coordinator.selectToRouter(2)).isZero(); ready.sourceIds.forEach(source -> assertThat(lifecycle.pending.find(source)).isPresent()); - doCallRealMethod().when(lifecycle.coordinator).publishReady(any(), any()); + doCallRealMethod().when(lifecycle.coordinator).makeHeldReady(any(), any()); worker.processQueued(); - assertThat(lifecycle.coordinator.selectAndStage(2)).isEqualTo(1); + assertThat(lifecycle.coordinator.selectToRouter(2)).isEqualTo(1); assertThat(lifecycle.coordinator.takeForRouting(Duration.ZERO).orElseThrow().message().body).isEqualTo("onetwo"); assertThat(worker.responses).hasSize(2); } @@ -153,7 +153,7 @@ void expiryPublishesEachAcceptedPartIndependently() throws Exception { assertThat(event.submitSm).isNull(); }); worker.handleIngressMessages(events); - assertThat(lifecycle.coordinator.selectAndStage(2)).isEqualTo(2); + assertThat(lifecycle.coordinator.selectToRouter(2)).isEqualTo(2); var one = lifecycle.coordinator.takeForRouting(Duration.ZERO).orElseThrow(); var three = lifecycle.coordinator.takeForRouting(Duration.ZERO).orElseThrow(); assertThat(one.message().body).isEqualTo("one"); @@ -176,7 +176,7 @@ void ignoredDuplicateOrdinalStillReleasesItsAcceptedSourceWithTheOriginalGroup() assertThat(ready.pMsg.body).isEqualTo("firstsecond"); assertThat(ready.pMsg.reassembledParts).hasSize(2); worker.handleIngressMessages(List.of(ready)); - lifecycle.coordinator.selectAndStage(1); + lifecycle.coordinator.selectToRouter(1); var selected = lifecycle.coordinator.takeForRouting(Duration.ZERO).orElseThrow(); assertThat(selected.sources()).hasSize(3); lifecycle.coordinator.discard(selected.id(), CompletableFuture.completedStage(null)).toCompletableFuture().join(); @@ -196,7 +196,7 @@ void replayingProcessedRawEventsDoesNotReassembleOrAcknowledgeThemAgain() throws worker.handleIngressMessages(raw); assertThat(worker.getInEventQueue()).hasSize(1); worker.processQueued(); - assertThat(lifecycle.coordinator.selectAndStage(2)).isEqualTo(1); + assertThat(lifecycle.coordinator.selectToRouter(2)).isEqualTo(1); var selected = lifecycle.coordinator.takeForRouting(Duration.ZERO).orElseThrow(); assertThat(selected.message().body).isEqualTo("onetwo"); assertThat(selected.sources()).hasSize(2); @@ -219,25 +219,25 @@ void expiryDoesNotClaimAnAcceptedPartStillWaitingInTheIngressQueue() throws Exce var expired = events.stream().filter(event -> event.submitSm == null).findFirst().orElseThrow(); assertThat(expired.sourceIds).doesNotContainAnyElementsOf(unprocessed.sourceIds); worker.handleIngressMessages(List.of(expired)); - assertThat(lifecycle.coordinator.selectAndStage(2)).isEqualTo(1); + assertThat(lifecycle.coordinator.selectToRouter(2)).isEqualTo(1); assertThat(lifecycle.coordinator.takeForRouting(Duration.ZERO).orElseThrow().message().body).isEqualTo("first"); worker.handleIngressMessages(List.of(unprocessed)); - assertThat(lifecycle.coordinator.selectAndStage(2)).isZero(); + assertThat(lifecycle.coordinator.selectToRouter(2)).isZero(); worker.expire(second.pMsg); worker.processQueued(); - assertThat(lifecycle.coordinator.selectAndStage(2)).isEqualTo(1); + assertThat(lifecycle.coordinator.selectToRouter(2)).isEqualTo(1); assertThat(lifecycle.coordinator.takeForRouting(Duration.ZERO).orElseThrow().message().body).isEqualTo("second"); assertThat(worker.responses).hasSize(2); } } @Test - void capacityAndQuiescenceRejectBeforeSuccessAndNeverFallBackToLegacyQueue() { + void capacityAndShutdownStartRejectBeforeSuccessAndNeverFallBackToLegacyQueue() { try (var lifecycle = new OutboundIngressFixture(1); var worker = new TestWorker()) { worker.setIngressCoordinator(lifecycle.coordinator); worker.enqueueIn(submission("0500037F0201", "held", 1)); worker.enqueueIn(submission("0500037F0202", "full", 2)); - lifecycle.coordinator.quiesce(); + lifecycle.coordinator.beginShutdown(); worker.enqueueIn(submission(null, "unavailable", 3)); assertThat(worker.responses).extracting(SubmitSmResp::getCommandStatus) .containsExactly(SmppConstants.STATUS_OK, SmppConstants.STATUS_THROTTLED, SmppConstants.STATUS_SYSERR); @@ -254,7 +254,7 @@ void unsupportedHeaderIsRejectedUsingTheExistingReassemblyValidator() { assertThat(worker.responses).singleElement().satisfies(response -> assertThat(response.getCommandStatus()).isEqualTo(SmppConstants.STATUS_SUBMITFAIL)); assertThat(worker.getInEventQueue()).isEmpty(); - assertThat(lifecycle.coordinator.selectAndStage(1)).isZero(); + assertThat(lifecycle.coordinator.selectToRouter(1)).isZero(); } } diff --git a/sendium-core/src/test/java/gr/cytech/sendium/core/storage/memory/HeldAdmissionTest.java b/sendium-core/src/test/java/gr/cytech/sendium/core/storage/memory/HeldAdmissionTest.java index 7d7e8502..a9a319ab 100644 --- a/sendium-core/src/test/java/gr/cytech/sendium/core/storage/memory/HeldAdmissionTest.java +++ b/sendium-core/src/test/java/gr/cytech/sendium/core/storage/memory/HeldAdmissionTest.java @@ -52,11 +52,11 @@ void heldSourcesAreRetainedButNeverIndividuallySelected() throws Exception { part.tags.clear(); assertThat(stores.pending.find(first).orElseThrow().body).isEqualTo("first"); assertThat(stores.pending.find(first).orElseThrow().tags).containsExactly("original"); - assertThat(stores.router.selectAndStage(3)).isZero(); + assertThat(stores.router.selectToRouter(3)).isZero(); assertThat(stores.router.take(Duration.ZERO)).isEmpty(); SourceId ordinary = source(); stores.pending.admit(ordinary, payload("ordinary", 1)); - assertThat(stores.router.selectAndStage(3)).isEqualTo(1); + assertThat(stores.router.selectToRouter(3)).isEqualTo(1); assertThat(stores.router.take(Duration.ZERO).orElseThrow().sources()).containsExactly(ordinary); } } @@ -72,18 +72,18 @@ void publicationAtFullCapacitySelectsOneAggregateWithAllOriginalSources() throws fails(() -> stores.pending.admitHeld(source(), payload("full", 2)), CAPACITY_EXCEEDED); Payload aggregate = payload("firstsecond", 2); aggregate.tags.add("assembled"); - stores.pending.publishReady(sources, aggregate); + stores.pending.makeHeldReady(sources, aggregate); sources.clear(); aggregate.body = "changed"; aggregate.tags.clear(); - assertThat(stores.router.selectAndStage(10)).isEqualTo(1); + assertThat(stores.router.selectToRouter(10)).isEqualTo(1); var selected = stores.router.take(Duration.ZERO).orElseThrow(); assertThat(selected.sources()).containsExactlyInAnyOrder(first, second); assertThat(selected.message().body).isEqualTo("firstsecond"); assertThat(selected.message().tags).containsExactly("assembled"); assertThat(stores.pending.find(first).orElseThrow().body).isEqualTo("first"); assertThat(stores.pending.find(second).orElseThrow().body).isEqualTo("second"); - assertThat(stores.router.selectAndStage(10)).isZero(); + assertThat(stores.router.selectToRouter(10)).isZero(); assertThat(stores.router.take(Duration.ZERO)).isEmpty(); stores.router.markRouted(selected.id()); fails(() -> stores.pending.admit(source(), payload("still full", 2)), CAPACITY_EXCEEDED); @@ -102,23 +102,23 @@ void publicationIsFirstWinsBeforeAndAfterSelectionAndRouting() throws Exception })) { var sources = Set.of(source(), source()); sources.forEach(source -> stores.pending.admitHeld(source, payload("part", 2))); - stores.pending.publishReady(sources, payload("first publication", 2)); + stores.pending.makeHeldReady(sources, payload("first publication", 2)); int count = snapshots.get(); - stores.pending.publishReady(new HashSet<>(sources), payload("replacement", 3)); + stores.pending.makeHeldReady(new HashSet<>(sources), payload("replacement", 3)); assertThat(snapshots).hasValue(count); - stores.router.selectAndStage(1); - stores.pending.publishReady(sources, payload("after selection", 3)); + stores.router.selectToRouter(1); + stores.pending.makeHeldReady(sources, payload("after selection", 3)); var selected = stores.router.take(Duration.ZERO).orElseThrow(); assertThat(selected.message().body).isEqualTo("first publication"); - stores.pending.publishReady(sources, payload("while taken", 3)); + stores.pending.makeHeldReady(sources, payload("while taken", 3)); stores.router.markRouted(selected.id()); - stores.pending.publishReady(sources, payload("after routing", 3)); - assertThat(stores.router.selectAndStage(1)).isZero(); + stores.pending.makeHeldReady(sources, payload("after routing", 3)); + assertThat(stores.router.selectToRouter(1)).isZero(); assertThat(stores.router.take(Duration.ZERO)).isEmpty(); stores.pending.complete(sources); stores.router.complete(selected.id()); - fails(() -> stores.pending.publishReady(sources, payload("after cleanup", 2)), INVALID_TRANSITION); - assertThat(stores.router.selectAndStage(1)).isZero(); + fails(() -> stores.pending.makeHeldReady(sources, payload("after cleanup", 2)), INVALID_TRANSITION); + assertThat(stores.router.selectToRouter(1)).isZero(); } } @@ -132,10 +132,10 @@ void duplicateAdmissionCannotPromoteHeldWorkOrHoldOrdinaryWork() throws Exceptio stores.pending.admit(held, payload("promote", 3)); stores.pending.admitHeld(held, payload("overwrite", 3)); stores.pending.admitHeld(ordinary, payload("hold", 3)); - assertThat(stores.router.selectAndStage(2)).isEqualTo(1); + assertThat(stores.router.selectToRouter(2)).isEqualTo(1); assertThat(stores.router.take(Duration.ZERO).orElseThrow().sources()).containsExactly(ordinary); assertThat(stores.pending.find(held).orElseThrow().body).isEqualTo("held"); - fails(() -> stores.pending.publishReady(Set.of(ordinary), payload("ordinary replacement", 2)), INVALID_TRANSITION); + fails(() -> stores.pending.makeHeldReady(Set.of(ordinary), payload("ordinary replacement", 2)), INVALID_TRANSITION); } } @@ -146,16 +146,16 @@ void callerCanPublishAnExpiredPartIndividuallyWithoutReleasingOtherHeldParts() t SourceId second = source(); stores.pending.admitHeld(first, payload("first", 2)); stores.pending.admitHeld(second, payload("second", 2)); - stores.pending.publishReady(Set.of(first), stores.pending.find(first).orElseThrow()); - assertThat(stores.router.selectAndStage(2)).isEqualTo(1); + stores.pending.makeHeldReady(Set.of(first), stores.pending.find(first).orElseThrow()); + assertThat(stores.router.selectToRouter(2)).isEqualTo(1); var selected = stores.router.take(Duration.ZERO).orElseThrow(); assertThat(selected.sources()).containsExactly(first); assertThat(selected.message().body).isEqualTo("first"); assertThat(stores.pending.find(second)).isPresent(); - assertThat(stores.router.selectAndStage(2)).isZero(); - fails(() -> stores.pending.publishReady(Set.of(first, second), payload("late aggregate", 2)), INVALID_TRANSITION); - stores.pending.publishReady(Set.of(second), stores.pending.find(second).orElseThrow()); - assertThat(stores.router.selectAndStage(2)).isEqualTo(1); + assertThat(stores.router.selectToRouter(2)).isZero(); + fails(() -> stores.pending.makeHeldReady(Set.of(first, second), payload("late aggregate", 2)), INVALID_TRANSITION); + stores.pending.makeHeldReady(Set.of(second), stores.pending.find(second).orElseThrow()); + assertThat(stores.router.selectToRouter(2)).isEqualTo(1); } } @@ -168,14 +168,14 @@ void missingOrOverlappingSourceSetsFailWithoutBindingUnpublishedSources() throws for (var source : Set.of(first, second, third)) { stores.pending.admitHeld(source, payload("part", 2)); } - fails(() -> stores.pending.publishReady(Set.of(first, source()), payload("unknown", 2)), INVALID_TRANSITION); - assertThat(stores.router.selectAndStage(3)).isZero(); - stores.pending.publishReady(Set.of(first, second), payload("aggregate", 2)); - fails(() -> stores.pending.publishReady(Set.of(first), payload("subset", 2)), INVALID_TRANSITION); - fails(() -> stores.pending.publishReady(Set.of(first, second, third), payload("superset", 2)), INVALID_TRANSITION); - fails(() -> stores.pending.publishReady(Set.of(first, third), payload("overlap", 2)), INVALID_TRANSITION); - stores.pending.publishReady(Set.of(third), payload("third", 2)); - assertThat(stores.router.selectAndStage(3)).isEqualTo(2); + fails(() -> stores.pending.makeHeldReady(Set.of(first, source()), payload("unknown", 2)), INVALID_TRANSITION); + assertThat(stores.router.selectToRouter(3)).isZero(); + stores.pending.makeHeldReady(Set.of(first, second), payload("aggregate", 2)); + fails(() -> stores.pending.makeHeldReady(Set.of(first), payload("subset", 2)), INVALID_TRANSITION); + fails(() -> stores.pending.makeHeldReady(Set.of(first, second, third), payload("superset", 2)), INVALID_TRANSITION); + fails(() -> stores.pending.makeHeldReady(Set.of(first, third), payload("overlap", 2)), INVALID_TRANSITION); + stores.pending.makeHeldReady(Set.of(third), payload("third", 2)); + assertThat(stores.router.selectToRouter(3)).isEqualTo(2); assertThat(stores.router.take(Duration.ZERO).orElseThrow().sources()).containsExactlyInAnyOrder(first, second); assertThat(stores.router.take(Duration.ZERO).orElseThrow().sources()).containsExactly(third); } @@ -193,12 +193,12 @@ void snapshotFailureLeavesHeldSourcesAvailableForACompletePublicationRetry() thr var sources = Set.of(source(), source()); sources.forEach(source -> stores.pending.admitHeld(source, payload("part", 2))); fail.set(true); - fails(() -> stores.pending.publishReady(sources, payload("aggregate", 2)), UNAVAILABLE); + fails(() -> stores.pending.makeHeldReady(sources, payload("aggregate", 2)), UNAVAILABLE); fail.set(false); sources.forEach(source -> assertThat(stores.pending.find(source)).isPresent()); - assertThat(stores.router.selectAndStage(2)).isZero(); - stores.pending.publishReady(sources, payload("aggregate", 2)); - assertThat(stores.router.selectAndStage(2)).isEqualTo(1); + assertThat(stores.router.selectToRouter(2)).isZero(); + stores.pending.makeHeldReady(sources, payload("aggregate", 2)); + assertThat(stores.router.selectToRouter(2)).isEqualTo(1); assertThat(stores.router.take(Duration.ZERO).orElseThrow().sources()).isEqualTo(sources); } } @@ -219,11 +219,11 @@ void eligibilityIsEvaluatedAtPublicationAndFailureDoesNotPublishPartOfAGroup() t var sources = Set.of(source(), source()); sources.forEach(source -> pending.admitHeld(source, new Payload("part", 3, null))); assertThat(evaluations).hasValue(0); - fails(() -> pending.publishReady(sources, payload("aggregate", 2)), UNSUPPORTED); - assertThat(router.selectAndStage(1)).isZero(); + fails(() -> pending.makeHeldReady(sources, payload("aggregate", 2)), UNSUPPORTED); + assertThat(router.selectToRouter(1)).isZero(); fail.set(false); - pending.publishReady(sources, payload("aggregate", 2)); - assertThat(router.selectAndStage(1)).isEqualTo(1); + pending.makeHeldReady(sources, payload("aggregate", 2)); + assertThat(router.selectToRouter(1)).isEqualTo(1); assertThat(router.take(Duration.ZERO).orElseThrow().message().body).isEqualTo("aggregate"); assertThat(evaluations).hasValue(2); } @@ -240,24 +240,24 @@ void preparedPriorityAndEligibilityStillPassThroughBoundedSelection() throws Exc for (var source : Set.of(low, due, future)) { stores.pending.admitHeld(source, payload("raw high priority", 5)); } - stores.pending.publishReady(Set.of(low), payload("prepared low", 1)); - stores.pending.publishReady(Set.of(due), payload("prepared high", 3)); - stores.pending.publishReady(Set.of(future), new Payload("future", 9, NOW.plusSeconds(1))); + stores.pending.makeHeldReady(Set.of(low), payload("prepared low", 1)); + stores.pending.makeHeldReady(Set.of(due), payload("prepared high", 3)); + stores.pending.makeHeldReady(Set.of(future), new Payload("future", 9, NOW.plusSeconds(1))); assertThat(stores.router.take(Duration.ZERO)).isEmpty(); - assertThat(stores.router.selectAndStage(10)).isEqualTo(1); + assertThat(stores.router.selectToRouter(10)).isEqualTo(1); var selected = stores.router.take(Duration.ZERO).orElseThrow(); assertThat(selected.message().body).isEqualTo("prepared high"); - assertThat(stores.router.selectAndStage(10)).isZero(); + assertThat(stores.router.selectToRouter(10)).isZero(); stores.router.markRouted(selected.id()); - assertThat(stores.router.selectAndStage(10)).isEqualTo(1); + assertThat(stores.router.selectToRouter(10)).isEqualTo(1); selected = stores.router.take(Duration.ZERO).orElseThrow(); assertThat(selected.message().body).isEqualTo("ordinary"); stores.router.markRouted(selected.id()); - stores.router.selectAndStage(10); + stores.router.selectToRouter(10); selected = stores.router.take(Duration.ZERO).orElseThrow(); assertThat(selected.message().body).isEqualTo("prepared low"); stores.router.markRouted(selected.id()); - assertThat(stores.router.selectAndStage(10)).isZero(); + assertThat(stores.router.selectToRouter(10)).isZero(); } } @@ -270,8 +270,8 @@ void concurrentPublicationAndSelectionYieldOneWholeGroup() throws Exception { for (int i = 0; i < 16; i++) { final int index = i; jobs.add(executor.submit(() -> { - stores.pending.publishReady(sources, payload("aggregate-" + index, 2)); - stores.router.selectAndStage(2); + stores.pending.makeHeldReady(sources, payload("aggregate-" + index, 2)); + stores.router.selectToRouter(2); })); } for (var job : jobs) { @@ -281,7 +281,7 @@ void concurrentPublicationAndSelectionYieldOneWholeGroup() throws Exception { assertThat(selected.sources()).isEqualTo(sources); assertThat(selected.message().body).startsWith("aggregate-"); assertThat(stores.router.take(Duration.ZERO)).isEmpty(); - assertThat(stores.router.selectAndStage(2)).isZero(); + assertThat(stores.router.selectToRouter(2)).isZero(); } } @@ -299,7 +299,7 @@ void competingGroupsCannotBothOwnAnOverlappingSource() throws Exception { for (var group : groups) { jobs.add(executor.submit(() -> { try { - stores.pending.publishReady(group, payload("aggregate", 2)); + stores.pending.makeHeldReady(group, payload("aggregate", 2)); return true; } catch (OutboundStorageException failure) { assertThat(failure.reason()).isEqualTo(INVALID_TRANSITION); @@ -309,12 +309,12 @@ void competingGroupsCannotBothOwnAnOverlappingSource() throws Exception { } boolean leftWon = jobs.getFirst().get(2, TimeUnit.SECONDS); assertThat(jobs.getLast().get(2, TimeUnit.SECONDS)).isNotEqualTo(leftWon); - assertThat(stores.router.selectAndStage(3)).isEqualTo(1); + assertThat(stores.router.selectToRouter(3)).isEqualTo(1); var selected = stores.router.take(Duration.ZERO).orElseThrow(); assertThat(selected.sources()).isEqualTo(groups.get(leftWon ? 0 : 1)); SourceId unbound = leftWon ? right : left; - stores.pending.publishReady(Set.of(unbound), payload("unbound", 2)); - assertThat(stores.router.selectAndStage(3)).isEqualTo(1); + stores.pending.makeHeldReady(Set.of(unbound), payload("unbound", 2)); + assertThat(stores.router.selectToRouter(3)).isEqualTo(1); } } @@ -327,11 +327,11 @@ void coordinatorRetainsHeldSourcesThroughHandoffOrUnroutedDiscard(boolean discar var coordinator = new DefaultOutboundCoordinator<>(pending, router, routed)) { coordinator.start(); var sources = Set.of(source(), source()); - sources.forEach(source -> coordinator.admitHeld(source, payload("part", 2))); - assertThat(coordinator.selectAndStage(1)).isZero(); - coordinator.publishReady(sources, payload("aggregate", 2)); + sources.forEach(source -> coordinator.acceptHeld(source, payload("part", 2))); + assertThat(coordinator.selectToRouter(1)).isZero(); + coordinator.makeHeldReady(sources, payload("aggregate", 2)); assertThat(coordinator.takeForRouting(Duration.ZERO)).isEmpty(); - coordinator.selectAndStage(1); + coordinator.selectToRouter(1); var selected = coordinator.takeForRouting(Duration.ZERO).orElseThrow(); assertThat(selected.sources()).isEqualTo(sources); var handoff = new CompletableFuture(); @@ -339,35 +339,35 @@ void coordinatorRetainsHeldSourcesThroughHandoffOrUnroutedDiscard(boolean discar if (discard) { completion = coordinator.discard(selected.id(), handoff); } else { - coordinator.route(selected, new Destination<>("A", selected.message())); - var work = coordinator.takeForDestination("A", Duration.ZERO).orElseThrow(); + coordinator.recordToRouted(selected, new Destination<>("A", selected.message())); + var work = coordinator.takeFromRouted("A", Duration.ZERO).orElseThrow(); completion = coordinator.complete(work.id(), handoff); } - coordinator.publishReady(sources, payload("late duplicate", 2)); + coordinator.makeHeldReady(sources, payload("late duplicate", 2)); sources.forEach(source -> assertThat(pending.find(source).orElseThrow().body).isEqualTo("part")); - assertThat(coordinator.selectAndStage(1)).isZero(); + assertThat(coordinator.selectToRouter(1)).isZero(); handoff.complete(null); completion.toCompletableFuture().join(); sources.forEach(source -> assertThat(pending.find(source)).isEmpty()); - fails(() -> coordinator.publishReady(sources, payload("after cleanup", 2)), INVALID_TRANSITION); + fails(() -> coordinator.makeHeldReady(sources, payload("after cleanup", 2)), INVALID_TRANSITION); } } @Test - void quiescenceAllowsExistingHeldWorkToBecomeReadyButRejectsNewAdmissionAndSelection() { + void shutdownStartAllowsExistingHeldWorkToBecomeReadyButRejectsNewAdmissionAndSelection() { try (var pending = new MemoryPendingMessageStore(1, Payload::copy); var router = new MemorySelectedRouterStore<>(pending, 1); var routed = new MemoryRoutedWorkStore(1, Payload::copy); var coordinator = new DefaultOutboundCoordinator<>(pending, router, routed)) { SourceId source = source(); - fails(() -> coordinator.admitHeld(source, payload("part", 2)), UNAVAILABLE); + fails(() -> coordinator.acceptHeld(source, payload("part", 2)), UNAVAILABLE); coordinator.start(); - coordinator.admitHeld(source, payload("part", 2)); - coordinator.quiesce(); - coordinator.publishReady(Set.of(source), payload("released at shutdown", 2)); + coordinator.acceptHeld(source, payload("part", 2)); + coordinator.beginShutdown(); + coordinator.makeHeldReady(Set.of(source), payload("released at shutdown", 2)); assertThat(pending.find(source)).isPresent(); - fails(() -> coordinator.admitHeld(source(), payload("new", 2)), UNAVAILABLE); - fails(() -> coordinator.selectAndStage(1), UNAVAILABLE); + fails(() -> coordinator.acceptHeld(source(), payload("new", 2)), UNAVAILABLE); + fails(() -> coordinator.selectToRouter(1), UNAVAILABLE); } } @@ -375,18 +375,18 @@ void quiescenceAllowsExistingHeldWorkToBecomeReadyButRejectsNewAdmissionAndSelec void invalidPublicationCannotCreateSourcesAndNewInstancesRecoverNoHeldOrReadyState() { var sources = Set.of(source()); try (var stores = new Stores(1, 1)) { - assertThatThrownBy(() -> stores.pending.publishReady(Set.of(), payload("empty", 2))) + assertThatThrownBy(() -> stores.pending.makeHeldReady(Set.of(), payload("empty", 2))) .isInstanceOf(IllegalArgumentException.class); - fails(() -> stores.pending.publishReady(sources, payload("unknown", 2)), INVALID_TRANSITION); + fails(() -> stores.pending.makeHeldReady(sources, payload("unknown", 2)), INVALID_TRANSITION); stores.pending.admitHeld(sources.iterator().next(), payload("held", 2)); - assertThatThrownBy(() -> stores.pending.publishReady(sources, null)).isInstanceOf(NullPointerException.class); + assertThatThrownBy(() -> stores.pending.makeHeldReady(sources, null)).isInstanceOf(NullPointerException.class); } try (var fresh = new Stores(1, 1)) { assertThat(fresh.pending.find(sources.iterator().next())).isEmpty(); - fails(() -> fresh.pending.publishReady(sources, payload("unknown", 2)), INVALID_TRANSITION); + fails(() -> fresh.pending.makeHeldReady(sources, payload("unknown", 2)), INVALID_TRANSITION); fresh.pending.close(); fails(() -> fresh.pending.admitHeld(source(), payload("closed", 2)), UNAVAILABLE); - fails(() -> fresh.pending.publishReady(sources, payload("closed", 2)), UNAVAILABLE); + fails(() -> fresh.pending.makeHeldReady(sources, payload("closed", 2)), UNAVAILABLE); } } diff --git a/sendium-core/src/test/java/gr/cytech/sendium/core/storage/memory/MemoryStorageTest.java b/sendium-core/src/test/java/gr/cytech/sendium/core/storage/memory/MemoryStorageTest.java index 71f5a016..bd96bfe5 100644 --- a/sendium-core/src/test/java/gr/cytech/sendium/core/storage/memory/MemoryStorageTest.java +++ b/sendium-core/src/test/java/gr/cytech/sendium/core/storage/memory/MemoryStorageTest.java @@ -101,13 +101,13 @@ void selectionHonorsDueTimesAndPriorityWithoutFutureWorkBlockingReadyMessages() stores.pending.admit(source(), message("high-first", 3)); stores.pending.admit(source(), message("normal", 2)); stores.pending.admit(source(), message("high-second", 3)); - assertThat(stores.router.selectAndStage(5)).isEqualTo(4); + assertThat(stores.router.selectToRouter(5)).isEqualTo(4); List bodies = new ArrayList<>(); for (int i = 0; i < 4; i++) { bodies.add(stores.router.take(Duration.ZERO).orElseThrow().message().body); } assertThat(bodies).containsExactly("high-first", "high-second", "normal", "low"); - assertThat(stores.router.selectAndStage(5)).isZero(); + assertThat(stores.router.selectToRouter(5)).isZero(); } } @@ -126,9 +126,9 @@ void advancingTheClockMakesDeferredWorkEligible() throws Exception { TestMessage delayed = new TestMessage("delayed", 2, NOW.plusSeconds(1)); pending.admit(source(), delayed); delayed.available = NOW; - assertThat(router.selectAndStage(1)).isZero(); + assertThat(router.selectToRouter(1)).isZero(); time.set(NOW.plusSeconds(1)); - assertThat(router.selectAndStage(1)).isEqualTo(1); + assertThat(router.selectToRouter(1)).isEqualTo(1); assertThat(router.take(Duration.ZERO).orElseThrow().message().body).isEqualTo("delayed"); } } @@ -144,20 +144,20 @@ void refillCopiesOnlyItsBudgetAndReservesCapacityForTakenWork() throws Exception stores.pending.admit(source(), message("item-" + i, 2)); } copies.set(0); - assertThat(stores.router.selectAndStage(1)).isEqualTo(1); + assertThat(stores.router.selectToRouter(1)).isEqualTo(1); assertThat(copies).hasValue(1); - assertThat(stores.router.selectAndStage(50)).isEqualTo(1); + assertThat(stores.router.selectToRouter(50)).isEqualTo(1); assertThat(copies).hasValue(2); Selected first = stores.router.take(Duration.ZERO).orElseThrow(); Selected second = stores.router.take(Duration.ZERO).orElseThrow(); - assertThat(stores.router.selectAndStage(50)).isZero(); + assertThat(stores.router.selectToRouter(50)).isZero(); assertThat(copies).hasValue(4); stores.router.release(first); stores.router.release(first); assertThat(stores.router.take(Duration.ZERO).orElseThrow().id()).isEqualTo(first.id()); assertThat(stores.router.take(Duration.ZERO)).isEmpty(); stores.router.markRouted(second.id()); - assertThat(stores.router.selectAndStage(50)).isEqualTo(1); + assertThat(stores.router.selectToRouter(50)).isEqualTo(1); } } @@ -166,7 +166,7 @@ void releasePreservesExecutionChangesWithoutMutatingPendingOrReactivatingAStaleT try (var stores = new Stores(1, 1)) { SourceId source = source(); stores.pending.admit(source, message("accepted", 2)); - stores.router.selectAndStage(1); + stores.router.selectToRouter(1); Selected first = stores.router.take(Duration.ZERO).orElseThrow(); first.message().body = "filtered"; first.message().tags.add("filter-tag"); @@ -193,12 +193,12 @@ void routingDoesNotCompleteTheSourceAndCleanupIsRetrySafe() throws Exception { try (var stores = new Stores(1, 1)) { SourceId source = source(); stores.pending.admit(source, message("one", 2)); - stores.router.selectAndStage(1); + stores.router.selectToRouter(1); Selected selected = stores.router.take(Duration.ZERO).orElseThrow(); fails(() -> stores.router.complete(selected.id()), ROUTER_QUEUE, INVALID_TRANSITION); stores.router.markRouted(selected.id()); stores.router.markRouted(selected.id()); - assertThat(stores.router.selectAndStage(1)).isZero(); + assertThat(stores.router.selectToRouter(1)).isZero(); assertThat(stores.pending.find(source)).isPresent(); fails(() -> stores.pending.admit(source(), message("two", 2)), PENDING, CAPACITY_EXCEEDED); stores.pending.complete(Set.of(source, source())); @@ -207,7 +207,7 @@ void routingDoesNotCompleteTheSourceAndCleanupIsRetrySafe() throws Exception { stores.router.complete(selected.id()); assertThat(stores.pending.find(source)).isEmpty(); stores.pending.admit(source(), message("two", 2)); - assertThat(stores.router.selectAndStage(1)).isEqualTo(1); + assertThat(stores.router.selectToRouter(1)).isEqualTo(1); } } @@ -257,12 +257,12 @@ void failureDuringBatchPublicationRetainsPublishedPrefixAndUnselectedSource() th stores.pending.admit(first, message("first", 2)); stores.pending.admit(second, message("second", 2)); fail.set(true); - fails(() -> stores.router.selectAndStage(2), ROUTER_QUEUE, UNAVAILABLE); + fails(() -> stores.router.selectToRouter(2), ROUTER_QUEUE, UNAVAILABLE); assertThat(stores.router.take(Duration.ZERO).orElseThrow().sources()).containsExactly(first); fail.set(false); assertThat(stores.pending.find(first)).isPresent(); assertThat(stores.pending.find(second)).isPresent(); - assertThat(stores.router.selectAndStage(2)).isEqualTo(1); + assertThat(stores.router.selectToRouter(2)).isEqualTo(1); assertThat(stores.router.take(Duration.ZERO).orElseThrow().sources()).containsExactly(second); assertThat(stores.router.take(Duration.ZERO)).isEmpty(); } @@ -278,7 +278,7 @@ void snapshotFailuresDuringTakeAndReturnLeaveOwnershipRetryable() throws Excepti return msg.copy(); })) { stores.pending.admit(source(), message("one", 2)); - stores.router.selectAndStage(1); + stores.router.selectToRouter(1); fail.set(true); fails(() -> stores.router.take(Duration.ZERO), ROUTER_QUEUE, UNAVAILABLE); fail.set(false); @@ -307,7 +307,7 @@ void concurrentSelectorsAndTakersNeverSelectTheSameSourceTwice() throws Exceptio jobs.add(executor.submit(() -> { long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(5); while (seen.size() < count && System.nanoTime() < deadline) { - stores.router.selectAndStage(3); + stores.router.selectToRouter(3); var selected = stores.router.take(Duration.ZERO); if (selected.isPresent()) { var work = selected.orElseThrow(); @@ -323,7 +323,7 @@ void concurrentSelectorsAndTakersNeverSelectTheSameSourceTwice() throws Exceptio job.get(6, TimeUnit.SECONDS); } assertThat(seen).hasSize(count); - assertThat(stores.router.selectAndStage(count)).isZero(); + assertThat(stores.router.selectToRouter(count)).isZero(); for (SourceId source : seen) { assertThat(stores.pending.find(source)).isPresent(); } @@ -342,7 +342,7 @@ void aWaitingTakeWakesWhenWorkIsStaged() throws Exception { }); awaitWaiting(waiter); stores.pending.admit(source(), message("one", 2)); - stores.router.selectAndStage(1); + stores.router.selectToRouter(1); assertThat(result.get(1, TimeUnit.SECONDS).orElseThrow().message().body).isEqualTo("one"); } } @@ -388,11 +388,11 @@ void takeIsInterruptibleAndFreshInstancesRecoverNothing() throws Exception { assertThat(taker.isAlive()).isFalse(); assertThat(failure.get()).isInstanceOf(InterruptedException.class); stores.pending.admit(source, message("will be lost", 2)); - stores.router.selectAndStage(1); + stores.router.selectToRouter(1); } try (var fresh = new Stores(1, 1)) { assertThat(fresh.pending.find(source)).isEmpty(); - assertThat(fresh.router.selectAndStage(1)).isZero(); + assertThat(fresh.router.selectToRouter(1)).isZero(); assertThat(fresh.router.take(Duration.ZERO)).isEmpty(); } } @@ -401,9 +401,9 @@ void takeIsInterruptibleAndFreshInstancesRecoverNothing() throws Exception { void invalidLimitsAndTimeoutsDoNotConsumeWork() throws Exception { try (var stores = new Stores(1, 1)) { stores.pending.admit(source(), message("one", 2)); - assertThatThrownBy(() -> stores.router.selectAndStage(0)).isInstanceOf(IllegalArgumentException.class); + assertThatThrownBy(() -> stores.router.selectToRouter(0)).isInstanceOf(IllegalArgumentException.class); assertThatThrownBy(() -> stores.router.take(Duration.ofNanos(-1))).isInstanceOf(IllegalArgumentException.class); - assertThat(stores.router.selectAndStage(1)).isEqualTo(1); + assertThat(stores.router.selectToRouter(1)).isEqualTo(1); assertThat(stores.router.take(Duration.ofSeconds(Long.MAX_VALUE))).isPresent(); assertThat(stores.router.take(Duration.ofMillis(1))).isEmpty(); } diff --git a/sendium-core/src/test/java/gr/cytech/sendium/routing/StandardOutboundDispatchTest.java b/sendium-core/src/test/java/gr/cytech/sendium/routing/StandardOutboundDispatchTest.java index de1ceda8..263ec0ef 100644 --- a/sendium-core/src/test/java/gr/cytech/sendium/routing/StandardOutboundDispatchTest.java +++ b/sendium-core/src/test/java/gr/cytech/sendium/routing/StandardOutboundDispatchTest.java @@ -34,11 +34,11 @@ void interruptedDestinationRecordingRetriesTheOriginalIntentWithoutRepeatingLook var routing = mock(StandardRoutingManager.class); when(routing.lookupForLifecycle(any())).thenReturn(new RoutingLookupResult(List.of(provider), true)); var dispatch = new StandardOutboundDispatch(fixture.coordinator, routing); - fixture.coordinator.admit(new SourceId(UUID.randomUUID()), message("hello")); - fixture.coordinator.selectAndStage(1); + fixture.coordinator.accept(new SourceId(UUID.randomUUID()), message("hello")); + fixture.coordinator.selectToRouter(1); var selected = fixture.coordinator.takeForRouting(Duration.ZERO).orElseThrow(); doThrow(new IllegalStateException("recording interrupted")).doCallRealMethod() - .when(fixture.coordinator).route(any(), any()); + .when(fixture.coordinator).recordToRouted(any(), any()); assertThatThrownBy(() -> dispatch.routeSelected(selected)).isInstanceOf(IllegalStateException.class); dispatch.retryRoutingTransitions(); verify(routing, times(1)).lookupForLifecycle(any()); @@ -53,8 +53,8 @@ void routingFilterDropCompletesTheSourceWithoutProviderDispatch() throws Excepti var routing = mock(StandardRoutingManager.class); var dispatch = new StandardOutboundDispatch(fixture.coordinator, routing); SourceId source = new SourceId(UUID.randomUUID()); - fixture.coordinator.admit(source, message("hello")); - fixture.coordinator.selectAndStage(1); + fixture.coordinator.accept(source, message("hello")); + fixture.coordinator.selectToRouter(1); var selected = fixture.coordinator.takeForRouting(Duration.ZERO).orElseThrow(); when(routing.lookupForLifecycle(any())).thenThrow(new FilterException(null, FilterStatusCodes.DROP, selected.message(), "dropped")); @@ -71,13 +71,13 @@ void shutdownDrainsAllHandoffsAndRetainsOwnershipOnTimeoutOrFailure() throws Exc var dispatch = new StandardOutboundDispatch(fixture.coordinator, routing(List.of(worker("smpp.provider", "provider")), "provider::default:")); SourceId source = new SourceId(UUID.randomUUID()); - fixture.coordinator.admit(source, message("hello")); + fixture.coordinator.accept(source, message("hello")); routeNext(dispatch, fixture); var taken = dispatch.takeForProvider("smpp.provider", Duration.ZERO).orElseThrow(); var first = new CompletableFuture(); var second = new CompletableFuture(); dispatch.finishProviderParts(taken, List.of(first, second)); - dispatch.quiesce(); + dispatch.beginShutdown(); first.complete(null); assertThat(dispatch.awaitProviderDrain(Duration.ZERO)).isFalse(); assertThatThrownBy(dispatch::close).isInstanceOf(IllegalStateException.class); @@ -98,16 +98,16 @@ void shutdownReturnsUnassignedWorkAndLeavesUnsubmittedWorkAtItsRecordedDestinati try (var fixture = new OutboundIngressFixture(2)) { var dispatch = new StandardOutboundDispatch(fixture.coordinator, routing(List.of(worker("smpp.provider", "provider")), "provider::default:")); - fixture.coordinator.admit(new SourceId(UUID.randomUUID()), message("assigned")); + fixture.coordinator.accept(new SourceId(UUID.randomUUID()), message("assigned")); routeNext(dispatch, fixture); var assigned = dispatch.takeForProvider("smpp.provider", Duration.ZERO).orElseThrow(); - fixture.coordinator.admit(new SourceId(UUID.randomUUID()), message("unassigned")); - fixture.coordinator.selectAndStage(1); + fixture.coordinator.accept(new SourceId(UUID.randomUUID()), message("unassigned")); + fixture.coordinator.selectToRouter(1); var selected = fixture.coordinator.takeForRouting(Duration.ZERO).orElseThrow(); - dispatch.quiesce(); + dispatch.beginShutdown(); assertThat(dispatch.routeSelected(selected)).isEmpty(); assertThat(fixture.router.take(Duration.ZERO).orElseThrow().id()).isEqualTo(selected.id()); - fixture.coordinator.returnToDestination(assigned); + fixture.coordinator.returnToRouted(assigned); assertThat(fixture.routed.take("smpp.provider", Duration.ZERO).orElseThrow().id()).isEqualTo(assigned.id()); assertThat(dispatch.awaitProviderDrain(Duration.ZERO)).isTrue(); dispatch.close(); @@ -121,7 +121,7 @@ void routesToOneRecordedDestinationAndRetainsSourcesThroughAllProviderPartHandof var routing = routing(List.of(worker), "provider::default:"); var dispatch = new StandardOutboundDispatch(fixture.coordinator, routing); SourceId source = new SourceId(UUID.randomUUID()); - fixture.coordinator.admit(source, message("hello")); + fixture.coordinator.accept(source, message("hello")); var routed = routeNext(dispatch, fixture).orElseThrow(); assertThat(routed.destination()).isEqualTo(worker.getFullName()); @@ -151,7 +151,7 @@ void failedPartHandoffRetainsWorkForCompletionOnlyRetryWithoutRedispatch() throw var dispatch = new StandardOutboundDispatch(fixture.coordinator, routing(List.of(worker("smpp.provider", "provider")), "provider::default:")); SourceId source = new SourceId(UUID.randomUUID()); - fixture.coordinator.admit(source, message("hello")); + fixture.coordinator.accept(source, message("hello")); routeNext(dispatch, fixture).orElseThrow(); var taken = dispatch.takeForProvider("smpp.provider", Duration.ZERO).orElseThrow(); var handoff = new CompletableFuture(); @@ -174,7 +174,7 @@ void copiedRouteIsRejectedBeforeRecordingWorkEvenWhenItFindsOnlyOneDestination() var routing = routing(List.of(provider), "+provider::default:"); var dispatch = new StandardOutboundDispatch(fixture.coordinator, routing); SourceId source = new SourceId(UUID.randomUUID()); - fixture.coordinator.admit(source, message("hello")); + fixture.coordinator.accept(source, message("hello")); assertThat(routing.lookupRoutingForMessage(message("legacy"), routing.getTargets().defaultTable) .getDestinations()).containsExactly(provider); @@ -196,7 +196,7 @@ void copiedRouteInsideNestedTableIsRejectedBeforeAssignment() throws Exception { var provider = worker("smpp.provider", "provider"); var routing = routing(List.of(provider), "second::default:", "[second]", "+provider::default:"); var dispatch = new StandardOutboundDispatch(fixture.coordinator, routing); - fixture.coordinator.admit(new SourceId(UUID.randomUUID()), message("hello")); + fixture.coordinator.accept(new SourceId(UUID.randomUUID()), message("hello")); assertThatThrownBy(() -> routeNext(dispatch, fixture)) .isInstanceOfSatisfying(OutboundStorageException.class, failure -> assertThat(failure.reason()).isEqualTo(OutboundStorageException.Reason.UNSUPPORTED)); @@ -211,7 +211,7 @@ void routingMissReturnsTheUnassignedSelectionForLaterRetry() throws Exception { var routing = routing(List.of(provider), "unknown::default:"); var dispatch = new StandardOutboundDispatch(fixture.coordinator, routing); SourceId source = new SourceId(UUID.randomUUID()); - fixture.coordinator.admit(source, message("hello")); + fixture.coordinator.accept(source, message("hello")); assertThat(routeNext(dispatch, fixture)).isEmpty(); assertThat(fixture.pending.find(source)).isPresent(); routing.parseNewRoutingTable(RoutingFileParser.parseRoutingTable(List.of("provider::default:")), List.of(provider)); @@ -227,7 +227,7 @@ private static StandardMessage message(String body) { private static Optional> routeNext(StandardOutboundDispatch dispatch, OutboundIngressFixture fixture) throws Exception { - fixture.coordinator.selectAndStage(1); + fixture.coordinator.selectToRouter(1); return dispatch.routeSelected(fixture.coordinator.takeForRouting(Duration.ZERO).orElseThrow()); } From 38f93b3bf9d98ff6da3e3e38a7e10181f1043d50 Mon Sep 17 00:00:00 2001 From: pavlos Date: Thu, 8 Oct 2026 15:53:23 +0300 Subject: [PATCH 14/20] test(storage): exclude routing fixture from CDI discovery Signed-off-by: pavlos --- .../cytech/sendium/core/smpp/client/SmppClientWorkerTest.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/sendium-core/src/test/java/gr/cytech/sendium/core/smpp/client/SmppClientWorkerTest.java b/sendium-core/src/test/java/gr/cytech/sendium/core/smpp/client/SmppClientWorkerTest.java index bd4e4a49..aea3df17 100644 --- a/sendium-core/src/test/java/gr/cytech/sendium/core/smpp/client/SmppClientWorkerTest.java +++ b/sendium-core/src/test/java/gr/cytech/sendium/core/smpp/client/SmppClientWorkerTest.java @@ -24,6 +24,7 @@ import gr.cytech.sendium.routing.StandardRoutingManager; import gr.cytech.sendium.routing.RoutingLookupResult; import gr.cytech.sendium.routing.RoutingFileParser; +import jakarta.enterprise.inject.Vetoed; import org.junit.jupiter.api.Test; import utils.OutboundIngressFixture; @@ -812,6 +813,7 @@ private static StandardMessage messageWithNetwork() { return msg; } + @Vetoed private static final class LifecycleRouting extends StandardRoutingManager { private final java.util.List providers; From ec81336adbe92ef880b7b45a20260a5e94a4b5cc Mon Sep 17 00:00:00 2001 From: pavlos Date: Thu, 8 Oct 2026 16:21:26 +0300 Subject: [PATCH 15/20] fix(storage): reserve worker TPS permits only for queued messages Empty legacy worker loops acquired rate-limit permits before checking their queues. With lifecycle traffic sharing the same worker limiter, idle polling consumed send capacity and delayed actual submissions. Acquire a permit only after obtaining a message, preserving one shared per-worker TPS budget for legacy and coordinated attempts. Avoid rate waiting after shutdown and retain the post-wait check that restores taken work. Verify idle polling, shutdown during dequeue and rate acquisition, and real shared-limiter pacing across legacy and lifecycle sends. The focused SMPP, dispatch and shutdown suite passes 48 tests. Signed-off-by: pavlos --- .../sendium/core/AbstractOutWorker.java | 4 +- .../smpp/client/SmppClientWorkerTest.java | 143 ++++++++++++++++++ 2 files changed, 146 insertions(+), 1 deletion(-) diff --git a/sendium-core/src/main/java/gr/cytech/sendium/core/AbstractOutWorker.java b/sendium-core/src/main/java/gr/cytech/sendium/core/AbstractOutWorker.java index 6472b243..8cfd0626 100644 --- a/sendium-core/src/main/java/gr/cytech/sendium/core/AbstractOutWorker.java +++ b/sendium-core/src/main/java/gr/cytech/sendium/core/AbstractOutWorker.java @@ -1478,7 +1478,6 @@ public void run() { sleeper.sleep(configurationProvider.getLongPrpt(_pauseSleepMs), TimeUnit.MILLISECONDS); continue; } - applyRateLimit(); if (msg == null) { // wait up to 200ms for the next Message msg = msgQ.dequeue(200); @@ -1487,6 +1486,9 @@ public void run() { if (msg == null) { continue; } + if (keepOnRunning) { + applyRateLimit(); + } if (!keepOnRunning) { logger.info("enqueued message msg:{} back to queue since worker stopping, and keepOnRunning:{}", msg.msgId, keepOnRunning); msgQ.enqueue(msg); diff --git a/sendium-core/src/test/java/gr/cytech/sendium/core/smpp/client/SmppClientWorkerTest.java b/sendium-core/src/test/java/gr/cytech/sendium/core/smpp/client/SmppClientWorkerTest.java index aea3df17..b2143207 100644 --- a/sendium-core/src/test/java/gr/cytech/sendium/core/smpp/client/SmppClientWorkerTest.java +++ b/sendium-core/src/test/java/gr/cytech/sendium/core/smpp/client/SmppClientWorkerTest.java @@ -13,6 +13,7 @@ import gr.cytech.sendium.core.AbstractOutWorker; import gr.cytech.sendium.core.message.StandardMessage; import gr.cytech.sendium.core.queue.Queue; +import gr.cytech.sendium.core.queue.QueueProvider; import gr.cytech.sendium.core.outbound.OutboundWork.Destination; import gr.cytech.sendium.core.outbound.OutboundWork.SourceId; import gr.cytech.sendium.core.storage.OutboundStorageException; @@ -25,6 +26,9 @@ import gr.cytech.sendium.routing.RoutingLookupResult; import gr.cytech.sendium.routing.RoutingFileParser; import jakarta.enterprise.inject.Vetoed; +import io.quarkus.arc.Arc; +import io.quarkus.arc.ArcContainer; +import io.quarkus.arc.ManagedContext; import org.junit.jupiter.api.Test; import utils.OutboundIngressFixture; @@ -42,11 +46,103 @@ import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.mockStatic; import static org.mockito.Mockito.spy; import static org.mockito.Mockito.when; class SmppClientWorkerTest { + @Test + void idleLegacyPollingDoesNotAcquireSendPermits() { + var worker = new LegacyRateWorker(); + int[] polls = {0}; + var queue = new Queue() { + @Override + public StandardMessage dequeue(long timeout) { + if (++polls[0] == 3) { + worker.keepOnRunning = false; + } + return null; + } + }; + worker.useQueue(queue); + + worker.new Worker(0).run(); + + assertThat(polls[0]).isEqualTo(3); + assertThat(worker.permits).isZero(); + assertThat(worker.legacySends).isZero(); + } + + @Test + void shutdownDuringRateAcquisitionReturnsTheTakenLegacyMessage() throws Exception { + var worker = new LegacyRateWorker(); + var queue = new Queue(); + var message = messageWithNetwork(); + queue.enqueue(message); + worker.useQueue(queue); + worker.afterPermit = () -> worker.keepOnRunning = false; + + worker.new Worker(0).run(); + + assertThat(worker.permits).isEqualTo(1); + assertThat(worker.legacySends).isZero(); + assertThat(queue.dequeue(0)).isSameAs(message); + assertThat(queue.isEmpty()).isTrue(); + } + + @Test + void shutdownDuringDequeueReturnsTheMessageWithoutAcquiringAPermit() throws Exception { + var worker = new LegacyRateWorker(); + var queue = new Queue() { + @Override + public StandardMessage dequeue(long timeout) throws InterruptedException { + var message = super.dequeue(timeout); + worker.keepOnRunning = false; + return message; + } + }; + var message = messageWithNetwork(); + queue.enqueue(message); + worker.useQueue(queue); + + worker.new Worker(0).run(); + + assertThat(worker.permits).isZero(); + assertThat(worker.legacySends).isZero(); + assertThat(queue.dequeue(0)).isSameAs(message); + assertThat(queue.isEmpty()).isTrue(); + } + + @Test + void legacyAndCoordinatedMessagesShareTheConfiguredWorkerRateLimit() throws Exception { + var worker = new LegacyRateWorker(); + worker.configureRate(2); + var queue = new Queue(); + queue.enqueue(messageWithNetwork()); + worker.useQueue(queue); + var container = mock(ArcContainer.class); + when(container.requestContext()).thenReturn(mock(ManagedContext.class)); + try (var arc = mockStatic(Arc.class)) { + arc.when(Arc::container).thenReturn(container); + worker.new Worker(0).run(); + } + assertThat(worker.legacySends).isEqualTo(1); + assertThat(worker.permits).isEqualTo(1); + worker.keepOnRunning = true; + + for (int i = 0; i < 2; i++) { + worker.submitPreparedCoordinated(messageWithNetwork(), (payload, policy) -> { + throw new AssertionError("Successful submission must not schedule a retry"); + }); + } + + assertThat(worker.permits).isEqualTo(3); + assertThat(worker.coordinatedSendTime - worker.legacySendTime) + .isGreaterThanOrEqualTo(Duration.ofMillis(800).toNanos()); + assertThat(worker.getTransactionsPerSecond()).isEqualTo(2); + } + @Test void heldSourcesCopiedRouteRejectionMultipartReroutingAndHandoffRetryRemainOwnedThroughShutdown() throws Exception { try (var fixture = new OutboundIngressFixture(2)) { @@ -813,6 +909,53 @@ private static StandardMessage messageWithNetwork() { return msg; } + @Vetoed + private static final class LegacyRateWorker extends TestSmppClientWorker { + private int permits; + private int legacySends; + private long legacySendTime; + private long coordinatedSendTime; + private Runnable afterPermit = () -> { }; + + private LegacyRateWorker() { + super(new TestConfigurationProvider(), new Queue<>(), new CapturingTracker()); + } + + private void useQueue(Queue queue) { + var resources = mock(WorkerResourceProvider.class); + var queues = mock(QueueProvider.class); + when(resources.geQueueProvider()).thenReturn(queues); + when(queues.subscribe(getFullName(), getQueueName(), false)).thenReturn(queue); + workerResources = resources; + checkSubscribeMessageQueue(); + } + + private void configureRate(double rate) { + configurationProvider.setProperty("tps", Double.toString(rate)); + configRateLimiter(); + } + + @Override + protected void applyRateLimit() { + permits++; + super.applyRateLimit(); + afterPermit.run(); + } + + @Override + public StandardMessage doMessage(int threadIndex, StandardMessage message) { + legacySends++; + legacySendTime = System.nanoTime(); + keepOnRunning = false; + return null; + } + + @Override + protected void sendCoordinatedRequest(SubmitSm request) { + coordinatedSendTime = System.nanoTime(); + } + } + @Vetoed private static final class LifecycleRouting extends StandardRoutingManager { private final java.util.List providers; From 59a213f8f9df82d10b22ee31055c582dcd389108 Mon Sep 17 00:00:00 2001 From: pavlos Date: Thu, 8 Oct 2026 16:38:49 +0300 Subject: [PATCH 16/20] fix(storage): skip router backlog scans during terminal cleanup Taken and routed selections have already left the ready deque. Unconditional removal searched the unrelated router backlog on every provider completion while holding shared storage and coordinator locks. Remove from the ready deque only for queued selections, retaining source-first cleanup, routing capacity accounting and idempotence. Verify queued, taken and routed cleanup preserves unrelated work ordering and capacity; all 80 focused storage and coordinator tests pass. Signed-off-by: pavlos --- .../memory/MemorySelectedRouterStore.java | 4 ++- .../storage/memory/MemoryStorageTest.java | 32 +++++++++++++++++++ 2 files changed, 35 insertions(+), 1 deletion(-) diff --git a/sendium-core/src/main/java/gr/cytech/sendium/core/storage/memory/MemorySelectedRouterStore.java b/sendium-core/src/main/java/gr/cytech/sendium/core/storage/memory/MemorySelectedRouterStore.java index a461a6f8..80a93acc 100644 --- a/sendium-core/src/main/java/gr/cytech/sendium/core/storage/memory/MemorySelectedRouterStore.java +++ b/sendium-core/src/main/java/gr/cytech/sendium/core/storage/memory/MemorySelectedRouterStore.java @@ -192,7 +192,9 @@ public void complete(SelectionId selection) { throw invalid("Complete pending sources before removing selected ownership"); } records.remove(selection); - ready.remove(entry); + if (entry.phase == Phase.QUEUED) { + ready.remove(entry); + } if (entry.phase != Phase.ROUTED) { routingSlots--; } diff --git a/sendium-core/src/test/java/gr/cytech/sendium/core/storage/memory/MemoryStorageTest.java b/sendium-core/src/test/java/gr/cytech/sendium/core/storage/memory/MemoryStorageTest.java index bd96bfe5..750a16b2 100644 --- a/sendium-core/src/test/java/gr/cytech/sendium/core/storage/memory/MemoryStorageTest.java +++ b/sendium-core/src/test/java/gr/cytech/sendium/core/storage/memory/MemoryStorageTest.java @@ -211,6 +211,38 @@ void routingDoesNotCompleteTheSourceAndCleanupIsRetrySafe() throws Exception { } } + @ParameterizedTest + @ValueSource(strings = {"queued", "taken", "routed"}) + void cleanupPreservesUnrelatedReadyWorkAndRoutingCapacity(String phase) throws Exception { + try (var stores = new Stores(3, 3)) { + SourceId source = source(); + stores.pending.admit(source, message("completed", 2)); + stores.pending.admit(source(), message("first", 2)); + stores.pending.admit(source(), message("second", 2)); + assertThat(stores.router.selectToRouter(3)).isEqualTo(3); + var selected = stores.router.take(Duration.ZERO).orElseThrow(); + if (phase.equals("queued")) { + stores.router.release(selected); + } else if (phase.equals("routed")) { + stores.router.markRouted(selected.id()); + } + + stores.pending.complete(Set.of(source)); + stores.router.complete(selected.id()); + stores.router.complete(selected.id()); + stores.pending.admit(source(), message("replacement", 2)); + assertThat(stores.router.selectToRouter(3)).isEqualTo(1); + + List bodies = new ArrayList<>(); + for (int i = 0; i < 3; i++) { + bodies.add(stores.router.take(Duration.ZERO).orElseThrow().message().body); + } + assertThat(bodies).containsExactly("first", "second", "replacement"); + assertThat(stores.router.take(Duration.ZERO)).isEmpty(); + assertThat(stores.pending.find(source)).isEmpty(); + } + } + @Test void failedAdmissionDoesNotReserveCapacityAndIdentitySnapshotsAreRejected() { AtomicBoolean fail = new AtomicBoolean(true); From e1b3e4db088b35bf7eaa30dbadc800172b0ab50f Mon Sep 17 00:00:00 2001 From: pavlos Date: Fri, 9 Oct 2026 15:50:09 +0300 Subject: [PATCH 17/20] fix(storage): isolate provider preparation in per-worker pools Blocking rate waits, worker filters and SMPP sends occupied the shared two-thread scheduler, allowing a slow destination to delay healthy destinations and limiting effective sending concurrency. Reserve worker slots before submitting preparation to bounded dispatcher-owned pools sized by threadCount. Preserve the shared per-worker TPS limiter and parent-owned provider retries and handoffs. Retire stopped-worker pools and shut down owned pools after drain without closing application-owned timer schedulers. Verify destination and timer isolation, three occupied slots with a fourth attempt waiting, pool shutdown and worker replacement. Full reactor verify passes 393 core unit tests, 39 integration tests and 43 app tests. Signed-off-by: pavlos --- docs/14-outbound-storage.md | 7 +- .../routing/StandardOutboundDispatch.java | 72 ++++++++++-- .../smpp/client/SmppClientWorkerTest.java | 111 +++++++++++++++++- 3 files changed, 178 insertions(+), 12 deletions(-) diff --git a/docs/14-outbound-storage.md b/docs/14-outbound-storage.md index 45ca5e63..c9881b6d 100644 --- a/docs/14-outbound-storage.md +++ b/docs/14-outbound-storage.md @@ -127,7 +127,12 @@ Partial terminal cleanup can retain routed records after source removal, so equa eliminate cleanup-related backpressure or provide cross-store transactions. The poller has a 100-millisecond fixed delay after each cycle. Batch size is not TPS or provider -concurrency: attempts use worker sending slots/rate limiting and a default shared two-thread scheduler. +concurrency: attempts use the worker's shared rate limiter and a dispatcher-owned execution pool +sized by its `threadCount`. A sending slot is reserved before executor submission, bounding queued +plus executing preparation tasks. The default shared two-thread scheduler handles destination +resolution and timed retries; rate waits, worker filters, and SMPP sends run on the worker's pool. +Stopped-worker pools are retired during transition maintenance, and dispatcher close shuts down +its worker pools after provider drain even when the timer scheduler is application-owned. Awaiting asynchronous outcomes does not hold a sending slot. Count limits do not bound message bytes. ## Retries and Failures diff --git a/sendium-core/src/main/java/gr/cytech/sendium/routing/StandardOutboundDispatch.java b/sendium-core/src/main/java/gr/cytech/sendium/routing/StandardOutboundDispatch.java index 33a7eaee..3ec568dc 100644 --- a/sendium-core/src/main/java/gr/cytech/sendium/routing/StandardOutboundDispatch.java +++ b/sendium-core/src/main/java/gr/cytech/sendium/routing/StandardOutboundDispatch.java @@ -19,6 +19,7 @@ import java.util.List; import java.util.Objects; import java.util.Optional; +import java.util.concurrent.ArrayBlockingQueue; import java.util.concurrent.CompletableFuture; import java.util.concurrent.CompletionStage; import java.util.concurrent.ConcurrentHashMap; @@ -27,6 +28,7 @@ import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.Semaphore; +import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; import java.util.function.Supplier; @@ -36,9 +38,10 @@ public final class StandardOutboundDispatch implements AutoCloseable { private final StandardRoutingManager routing; private final ScheduledExecutorService scheduler; private final boolean ownsScheduler; - private final ConcurrentMap, Semaphore> workerSlots = new ConcurrentHashMap<>(); + private final ConcurrentMap, WorkerExecution> workerExecutors = new ConcurrentHashMap<>(); private final ConcurrentMap> activeExecutions = new ConcurrentHashMap<>(); private boolean shuttingDown; + private boolean closed; private final ConcurrentMap routingTransitions = new ConcurrentHashMap<>(); public StandardOutboundDispatch(OutboundCoordinator coordinator, StandardRoutingManager routing) { @@ -115,6 +118,7 @@ public synchronized Optional> routeSelected(Selected execution.executor.shutdown()); + workerExecutors.clear(); if (ownsScheduler) { scheduler.shutdown(); } } + private synchronized WorkerExecution executionFor(SmppClientWorker worker) { + if (closed) { + throw new IllegalStateException("Outbound dispatch is closed"); + } + return workerExecutors.computeIfAbsent(worker, WorkerExecution::new); + } + + private void retireStoppedWorkers() { + workerExecutors.entrySet().removeIf(entry -> { + if (entry.getKey().isKeepOnRunning()) { + return false; + } + entry.getValue().executor.shutdown(); + return true; + }); + } + /** Completes explicitly supplied provider outcomes once all required part handoffs succeed. */ public synchronized CompletionStage finishProviderParts(Routed work, List> requiredHandoffs) { @@ -264,6 +288,21 @@ public synchronized CompletionStage finishProviderParts(Routed worker) { + int threads = Math.max(1, worker.getThreadCount()); + slots = new Semaphore(threads); + executor = new ThreadPoolExecutor(threads, threads, 30, TimeUnit.SECONDS, + new ArrayBlockingQueue<>(threads), + Thread.ofPlatform().name("outbound-provider-" + worker.getFullName() + "-", 1).daemon(true).factory()); + executor.allowCoreThreadTimeOut(true); + } + } + public enum AttemptState { QUEUED, PREPARING, AWAITING_COMPLETION, COMPLETED, FAILED } @@ -319,7 +358,8 @@ public void run() { } state = AttemptState.PREPARING; } - Semaphore slot = null; + WorkerExecution destination = null; + boolean reserved = false; try { if (worker == null) { worker = Objects.requireNonNull(resolveDestination.get(), "retry destination"); @@ -334,16 +374,32 @@ public void run() { } return; } - slot = workerSlots.computeIfAbsent(worker, destination -> - new Semaphore(Math.max(1, destination.getThreadCount()))); - if (!slot.tryAcquire()) { - slot = null; + destination = executionFor(worker); + if (!destination.slots.tryAcquire()) { synchronized (this) { state = AttemptState.QUEUED; schedule(100); } return; } + reserved = true; + WorkerExecution reservedDestination = destination; + destination.executor.execute(() -> prepare(reservedDestination)); + reserved = false; + } catch (Exception error) { + synchronized (this) { + state = AttemptState.FAILED; + failure = error; + } + } finally { + if (reserved) { + destination.slots.release(); + } + } + } + + private void prepare(WorkerExecution destination) { + try { var submission = worker.submitPreparedCoordinated(message, (payload, policy) -> scheduleAttempt(payload, policy == SmppClientWorker.NackHandlePolicy.RETRY_ROUTER ? () -> retryDestination(payload) : () -> worker, execution)); @@ -366,9 +422,7 @@ public void run() { failure = error; } } finally { - if (slot != null) { - slot.release(); - } + destination.slots.release(); } } } diff --git a/sendium-core/src/test/java/gr/cytech/sendium/core/smpp/client/SmppClientWorkerTest.java b/sendium-core/src/test/java/gr/cytech/sendium/core/smpp/client/SmppClientWorkerTest.java index b2143207..f66743dd 100644 --- a/sendium-core/src/test/java/gr/cytech/sendium/core/smpp/client/SmppClientWorkerTest.java +++ b/sendium-core/src/test/java/gr/cytech/sendium/core/smpp/client/SmppClientWorkerTest.java @@ -259,6 +259,109 @@ void workerSlotAndRateGatePreventConcurrentPreparationAndEarlySending() throws E } } + @Test + void blockedWorkerUsesItsConfiguredSlotsWithoutBlockingOtherDestinationsOrTimers() throws Exception { + var scheduler = new ScheduledThreadPoolExecutor(1); + var rateGate = new CompletableFuture(); + var blocked = new TestSmppClientWorker(new TestConfigurationProvider(), new Queue<>(), new CapturingTracker()); + blocked.threadCount = 3; + blocked.rateGate = rateGate; + blocked.rateEntered = new CountDownLatch(3); + var healthy = spy(new TestSmppClientWorker(new TestConfigurationProvider(), new Queue<>(), new CapturingTracker())); + when(healthy.getFullName()).thenReturn("smppclient.healthy"); + try (var fixture = new OutboundIngressFixture(5)) { + var dispatch = new StandardOutboundDispatch(fixture.coordinator, mock(StandardRoutingManager.class), scheduler); + var executions = new java.util.ArrayList(); + try { + for (int i = 0; i < 4; i++) { + executions.add(submitOwned(dispatch, fixture, blocked)); + } + assertThat(blocked.rateEntered.await(5, TimeUnit.SECONDS)).isTrue(); + scheduler.submit(() -> { }).get(5, TimeUnit.SECONDS); + assertThat(blocked.rateChecks).isEqualTo(3); + assertThat(blocked.rateThreads).hasSize(3); + assertThat(blocked.coordinatedRequests).isEmpty(); + + var healthyExecution = submitOwned(dispatch, fixture, healthy); + healthy.awaitRequests(1); + respond(handler(healthy), healthy.coordinatedRequests.getFirst(), SmppConstants.STATUS_OK); + healthyExecution.completion().toCompletableFuture().get(5, TimeUnit.SECONDS); + assertThat(blocked.rateChecks).isEqualTo(3); + executions.forEach(execution -> assertThat(execution.completion().toCompletableFuture()).isNotDone()); + + dispatch.beginShutdown(); + assertThat(dispatch.awaitProviderDrain(Duration.ZERO)).isFalse(); + assertThatThrownBy(dispatch::close).isInstanceOf(IllegalStateException.class); + rateGate.complete(null); + blocked.awaitRequests(4); + blocked.coordinatedRequests.forEach(request -> respond(handler(blocked), request, SmppConstants.STATUS_OK)); + for (var execution : executions) { + execution.completion().toCompletableFuture().get(5, TimeUnit.SECONDS); + } + assertThat(blocked.rateChecks).isEqualTo(4); + assertThat(dispatch.awaitProviderDrain(Duration.ofSeconds(1))).isTrue(); + } finally { + rateGate.complete(null); + dispatch.close(); + } + for (Thread thread : blocked.rateThreads) { + thread.join(Duration.ofSeconds(5)); + assertThat(thread.isAlive()).isFalse(); + } + for (Thread thread : healthy.rateThreads) { + thread.join(Duration.ofSeconds(5)); + assertThat(thread.isAlive()).isFalse(); + } + assertThat(scheduler.isShutdown()).isFalse(); + } finally { + rateGate.complete(null); + scheduler.shutdownNow(); + } + } + + @Test + void transitionMaintenanceRetiresStoppedWorkerPoolAndReplacementUsesItsOwnPool() throws Exception { + var stopped = new TestSmppClientWorker(new TestConfigurationProvider(), new Queue<>(), new CapturingTracker()); + var replacement = new TestSmppClientWorker(new TestConfigurationProvider(), new Queue<>(), new CapturingTracker()); + try (var fixture = new OutboundIngressFixture(1); + var dispatch = new StandardOutboundDispatch(fixture.coordinator, mock(StandardRoutingManager.class))) { + var first = submitOwned(dispatch, fixture, stopped); + stopped.awaitRequests(1); + respond(handler(stopped), stopped.coordinatedRequests.getFirst(), SmppConstants.STATUS_OK); + first.completion().toCompletableFuture().get(5, TimeUnit.SECONDS); + + stopped.keepOnRunning = false; + dispatch.retryRoutingTransitions(); + for (Thread thread : stopped.rateThreads) { + thread.join(Duration.ofSeconds(5)); + assertThat(thread.isAlive()).isFalse(); + } + + var second = submitOwned(dispatch, fixture, replacement); + replacement.awaitRequests(1); + respond(handler(replacement), replacement.coordinatedRequests.getFirst(), SmppConstants.STATUS_OK); + second.completion().toCompletableFuture().get(5, TimeUnit.SECONDS); + assertThat(stopped.coordinatedRequests).hasSize(1); + assertThat(replacement.rateThreads).doesNotContainAnyElementsOf(stopped.rateThreads); + } + for (Thread thread : replacement.rateThreads) { + thread.join(Duration.ofSeconds(5)); + assertThat(thread.isAlive()).isFalse(); + } + } + + private static StandardOutboundDispatch.ProviderExecution submitOwned(StandardOutboundDispatch dispatch, + OutboundIngressFixture fixture, TestSmppClientWorker worker) throws Exception { + var message = messageWithNetwork(); + message.body = "hello"; + fixture.coordinator.accept(new SourceId(UUID.randomUUID()), message); + fixture.coordinator.selectToRouter(1); + var selected = fixture.coordinator.takeForRouting(Duration.ZERO).orElseThrow(); + fixture.coordinator.recordToRouted(selected, new Destination<>(worker.getFullName(), selected.message())); + return dispatch.submitToProvider( + fixture.coordinator.takeFromRouted(worker.getFullName(), Duration.ZERO).orElseThrow(), worker); + } + @Test void failedRetryLookupCanResumeWithoutLosingParentOrRepeatingOriginalSubmission() throws Exception { var scheduler = new ScheduledThreadPoolExecutor(1); @@ -974,7 +1077,8 @@ private static class TestSmppClientWorker extends SmppClientWorker rateGate; - private final CountDownLatch rateEntered = new CountDownLatch(1); + private CountDownLatch rateEntered = new CountDownLatch(1); + private final Set rateThreads = java.util.concurrent.ConcurrentHashMap.newKeySet(); private boolean mapCharacters; private gr.cytech.sendium.external.filter.FilterStatusCodes preparationStatus; private final CountDownLatch pauseChecked = new CountDownLatch(1); @@ -987,7 +1091,10 @@ public boolean isPause() { @Override protected void applyRateLimit() { - rateChecks++; + synchronized (this) { + rateChecks++; + } + rateThreads.add(Thread.currentThread()); if (rateGate != null) { rateEntered.countDown(); rateGate.join(); From ab9d3bde42718f1ef833e7b9a06e9d86dda7ffc7 Mon Sep 17 00:00:00 2001 From: pavlos Date: Fri, 9 Oct 2026 16:03:41 +0300 Subject: [PATCH 18/20] fix(storage): continue productive outbound batches without delay A mandatory 100 ms pause after every bounded processing cycle imposed a global staging ceiling even when ready messages and provider capacity were available. Schedule the next bounded cycle immediately after successful routing or provider scheduling. Retain idle, pause and failure backoff so unroutable work cannot trigger a busy loop. Fence queued cycles and rescheduling during shutdown, preserving per-worker TPS and execution limits. Add seven deterministic scheduling cases and retain real HTTP pipeline and shutdown coverage. Full reactor verify passes 393 core unit tests, 39 integration tests and 50 app tests with zero Checkstyle violations. Signed-off-by: pavlos --- docs/14-outbound-storage.md | 5 +- .../storage/StandaloneOutboundPipeline.java | 38 +++- .../StandaloneOutboundSchedulingTest.java | 207 ++++++++++++++++++ 3 files changed, 244 insertions(+), 6 deletions(-) create mode 100644 sendium-app/src/test/java/gr/cytech/sendium/app/storage/StandaloneOutboundSchedulingTest.java diff --git a/docs/14-outbound-storage.md b/docs/14-outbound-storage.md index c9881b6d..fac2c1c4 100644 --- a/docs/14-outbound-storage.md +++ b/docs/14-outbound-storage.md @@ -126,7 +126,10 @@ A group of N accepted multipart sources uses N pending slots but one selected sl Partial terminal cleanup can retain routed records after source removal, so equal capacities do not eliminate cleanup-related backpressure or provide cross-store transactions. -The poller has a 100-millisecond fixed delay after each cycle. Batch size is not TPS or provider +The processor schedules the next bounded cycle without a mandatory delay after successful routing +or provider scheduling. Idle, paused, or unsuccessful cycles back off for 100 milliseconds; +selection or returning an unroutable item alone does not count as progress. Shutdown prevents +further cycle scheduling. Batch size is not TPS or provider concurrency: attempts use the worker's shared rate limiter and a dispatcher-owned execution pool sized by its `threadCount`. A sending slot is reserved before executor submission, bounding queued plus executing preparation tasks. The default shared two-thread scheduler handles destination diff --git a/sendium-app/src/main/java/gr/cytech/sendium/app/storage/StandaloneOutboundPipeline.java b/sendium-app/src/main/java/gr/cytech/sendium/app/storage/StandaloneOutboundPipeline.java index 99a61bd8..88fcda32 100644 --- a/sendium-app/src/main/java/gr/cytech/sendium/app/storage/StandaloneOutboundPipeline.java +++ b/sendium-app/src/main/java/gr/cytech/sendium/app/storage/StandaloneOutboundPipeline.java @@ -54,6 +54,8 @@ public class StandaloneOutboundPipeline { private StandardOutboundDispatch dispatch; private ScheduledExecutorService processors; private int batchSize; + private final Object schedulingLock = new Object(); + private boolean stopping; private final Map executions = new ConcurrentHashMap<>(); @Produces @@ -83,15 +85,34 @@ void start(@Observes @Priority(Interceptor.Priority.APPLICATION + 100) StartupEv thread.setDaemon(true); return thread; }); - processors.scheduleWithFixedDelay(this::process, 0, 100, TimeUnit.MILLISECONDS); + scheduleNextCycle(0); logger.info("Standalone outbound memory pipeline activated"); } - void process() { + private void scheduleNextCycle(long delay) { + synchronized (schedulingLock) { + if (!stopping) { + processors.schedule(this::runProcessingCycle, delay, TimeUnit.MILLISECONDS); + } + } + } + + private void runProcessingCycle() { + synchronized (schedulingLock) { + if (stopping) { + return; + } + } + boolean progressed = process(); + scheduleNextCycle(progressed ? 0 : 100); + } + + boolean process() { + boolean progressed = false; try { retryExecutions(); if (configuration.getBlnPrpt(AbstractRoutingManager._pause)) { - return; + return false; } coordinator.selectToRouter(batchSize); for (int i = 0; i < batchSize; i++) { @@ -100,7 +121,9 @@ void process() { break; } try { - dispatch.routeSelected(selected.orElseThrow()); + if (dispatch.routeSelected(selected.orElseThrow()).isPresent()) { + progressed = true; + } } catch (Exception failure) { logFailure("routing", failure); } @@ -128,6 +151,7 @@ void process() { try { var execution = dispatch.submitToProvider(work.orElseThrow(), (SmppClientWorker) client); executions.put(work.orElseThrow().id(), execution); + progressed = true; } catch (Exception failure) { coordinator.returnToRouted(work.orElseThrow()); throw failure; @@ -139,6 +163,7 @@ void process() { } catch (Exception failure) { logFailure("provider dispatch", failure); } + return progressed; } private void retryExecutions() { @@ -161,7 +186,10 @@ void stop(@Observes @Priority(Interceptor.Priority.PLATFORM_BEFORE) ShutdownEven return; } dispatch.beginShutdown(); - processors.shutdown(); + synchronized (schedulingLock) { + stopping = true; + processors.shutdown(); + } while (!processors.awaitTermination(1, TimeUnit.SECONDS)) { logger.warn("Waiting for standalone outbound processors to stop before ownership restoration"); } diff --git a/sendium-app/src/test/java/gr/cytech/sendium/app/storage/StandaloneOutboundSchedulingTest.java b/sendium-app/src/test/java/gr/cytech/sendium/app/storage/StandaloneOutboundSchedulingTest.java new file mode 100644 index 00000000..2fd51121 --- /dev/null +++ b/sendium-app/src/test/java/gr/cytech/sendium/app/storage/StandaloneOutboundSchedulingTest.java @@ -0,0 +1,207 @@ +package gr.cytech.sendium.app.storage; + +import gr.cytech.sendium.conf.SendiumConfigurationHandler; +import gr.cytech.sendium.core.message.StandardMessage; +import gr.cytech.sendium.core.outbound.OutboundWork.SourceId; +import gr.cytech.sendium.core.smpp.client.SmppClientWorker; +import gr.cytech.sendium.routing.AbstractRoutingManager; +import gr.cytech.sendium.routing.OutgoingWorkerManager; +import gr.cytech.sendium.routing.RoutingLookupResult; +import gr.cytech.sendium.routing.StandardRoutingManager; +import org.eclipse.microprofile.config.Config; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; +import java.util.UUID; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ThreadFactory; +import java.util.concurrent.TimeUnit; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.CALLS_REAL_METHODS; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.mockStatic; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +class StandaloneOutboundSchedulingTest { + @Test + void productiveBacklogContinuesImmediatelyInBoundedCyclesThenBacksOff() throws Exception { + try (var fixture = new SchedulingFixture()) { + fixture.accept(5); + fixture.runNext(); + verify(fixture.routing, times(2)).lookupForLifecycle(any()); + assertThat(fixture.nextDelay()).isZero(); + fixture.runNext(); + verify(fixture.routing, times(4)).lookupForLifecycle(any()); + assertThat(fixture.nextDelay()).isZero(); + fixture.runNext(); + verify(fixture.routing, times(5)).lookupForLifecycle(any()); + assertThat(fixture.nextDelay()).isZero(); + fixture.runNext(); + verify(fixture.routing, times(5)).lookupForLifecycle(any()); + assertThat(fixture.nextDelay()).isEqualTo(100); + } + } + + @ParameterizedTest + @ValueSource(booleans = {false, true}) + void routingMissesAndFailuresBackOffEvenAfterSelectingWork(boolean fails) throws Exception { + try (var fixture = new SchedulingFixture()) { + if (fails) { + when(fixture.routing.lookupForLifecycle(any())).thenThrow(new IOException("lookup unavailable")); + } else { + when(fixture.routing.lookupForLifecycle(any())).thenReturn(new RoutingLookupResult(List.of(), true)); + } + fixture.accept(1); + fixture.runNext(); + assertThat(fixture.nextDelay()).isEqualTo(100); + fixture.runNext(); + assertThat(fixture.nextDelay()).isEqualTo(100); + verify(fixture.routing, times(4)).lookupForLifecycle(any()); + assertThat(fixture.pipeline.coordinator.selectToRouter(2)).isZero(); + } + } + + @Test + void pausedPipelineBacksOffAndResumesItsExistingBacklog() throws Exception { + try (var fixture = new SchedulingFixture()) { + fixture.accept(3); + when(fixture.pipeline.configuration.getBlnPrpt(AbstractRoutingManager._pause)).thenReturn(true); + fixture.runNext(); + assertThat(fixture.nextDelay()).isEqualTo(100); + verify(fixture.routing, never()).lookupForLifecycle(any()); + when(fixture.pipeline.configuration.getBlnPrpt(AbstractRoutingManager._pause)).thenReturn(false); + fixture.runNext(); + assertThat(fixture.nextDelay()).isZero(); + verify(fixture.routing, times(2)).lookupForLifecycle(any()); + } + } + + @Test + void pausedDestinationBacksOffAndProviderSchedulingResumesImmediately() throws Exception { + try (var fixture = new SchedulingFixture()) { + fixture.accept(1); + fixture.runNext(); + var provider = new StandaloneOutboundPipelineTest.Providers(); + provider.handoff.complete(null); + when(fixture.pipeline.workers.getWorkersCopy()).thenReturn(List.of(provider.worker)); + when(provider.worker.isPause()).thenReturn(true); + fixture.runNext(); + assertThat(fixture.nextDelay()).isEqualTo(100); + assertThat(provider.calls).hasValue(0); + when(provider.worker.isPause()).thenReturn(false); + fixture.runNext(); + assertThat(fixture.nextDelay()).isZero(); + assertThat(provider.submitted.await(5, TimeUnit.SECONDS)).isTrue(); + assertThat(provider.calls).hasValue(1); + } + } + + @Test + void selectionFailureBacksOffWithoutStoppingTheProcessingChain() throws Exception { + try (var fixture = new SchedulingFixture()) { + fixture.accept(1); + doThrow(new IllegalStateException("selection unavailable")).doCallRealMethod() + .when(fixture.pipeline.coordinator).selectToRouter(2); + fixture.runNext(); + assertThat(fixture.nextDelay()).isEqualTo(100); + fixture.runNext(); + assertThat(fixture.nextDelay()).isZero(); + verify(fixture.routing, times(1)).lookupForLifecycle(any()); + } + } + + @Test + void shutdownFencesQueuedCyclesAndFurtherRescheduling() throws Exception { + try (var fixture = new SchedulingFixture()) { + fixture.runNext(); + assertThat(fixture.nextDelay()).isEqualTo(100); + int scheduled = fixture.cycles.size(); + fixture.close(); + fixture.runNext(); + assertThat(fixture.cycles).hasSize(scheduled); + verify(fixture.pipeline.coordinator, times(1)).selectToRouter(2); + verify(fixture.scheduler).shutdown(); + } + } + + /** Captures production scheduling decisions without real-time latency assertions. */ + private static final class SchedulingFixture implements AutoCloseable { + private final ScheduledExecutorService scheduler = mock(ScheduledExecutorService.class); + private final StandardRoutingManager routing = mock(StandardRoutingManager.class); + private final StandaloneOutboundPipeline pipeline = new StandaloneOutboundPipeline(); + private final List cycles = new ArrayList<>(); + private final List delays = new ArrayList<>(); + private int nextCycle; + private boolean closed; + + private SchedulingFixture() throws Exception { + var config = mock(Config.class); + when(config.getOptionalValue(anyString(), eq(Integer.class))).thenReturn(Optional.empty()); + when(config.getOptionalValue("sendium.message.selection-batch-size", Integer.class)).thenReturn(Optional.of(2)); + pipeline.coordinator = spy(StandaloneOutboundPipeline.coordinator( + new MessageStorageProfile("memory", "memory", "memory"), config)); + pipeline.config = config; + pipeline.routing = routing; + pipeline.workers = mock(OutgoingWorkerManager.class); + when(pipeline.workers.getWorkersCopy()).thenReturn(List.of()); + when(pipeline.workers.stop()).thenReturn(true); + pipeline.configuration = mock(SendiumConfigurationHandler.class); + var provider = mock(SmppClientWorker.class); + when(provider.getFullName()).thenReturn("smppclient.provider"); + when(routing.lookupForLifecycle(any())).thenReturn(new RoutingLookupResult(List.of(provider), true)); + when(scheduler.awaitTermination(anyLong(), any())).thenReturn(true); + doAnswer(invocation -> { + cycles.add(invocation.getArgument(0)); + delays.add(invocation.getArgument(1)); + return null; + }).when(scheduler).schedule(any(Runnable.class), anyLong(), eq(TimeUnit.MILLISECONDS)); + try (var executors = mockStatic(Executors.class, CALLS_REAL_METHODS)) { + executors.when(() -> Executors.newSingleThreadScheduledExecutor(any(ThreadFactory.class))) + .thenReturn(scheduler); + pipeline.start(null); + } + assertThat(nextDelay()).isZero(); + } + + private void accept(int count) { + for (int i = 0; i < count; i++) { + var message = new StandardMessage(); + message.body = "message-" + i; + pipeline.coordinator.accept(new SourceId(UUID.randomUUID()), message); + } + } + + private void runNext() { + cycles.get(nextCycle++).run(); + } + + private long nextDelay() { + return delays.getLast(); + } + + @Override + public void close() throws Exception { + if (!closed) { + pipeline.stop(null); + closed = true; + } + } + } +} From ae31149d81fb5a9fde836e02b3e1bed37b17d979 Mon Sep 17 00:00:00 2001 From: pavlos Date: Fri, 9 Oct 2026 16:42:42 +0300 Subject: [PATCH 19/20] test(storage): address code-quality review annotations Match the credential fixture's synchronized getter contract and annotate multipart listener overrides. Explain why CDI observer parameters and the validated-profile producer dependency remain necessary despite unused payload values. Preserve lifecycle ordering and the existing immutable source snapshots rather than removing dependencies or adding redundant copies to silence analyzer comments. All 42 focused ownership, HTTP and startup-profile tests pass with zero Checkstyle violations. Signed-off-by: pavlos --- .../cytech/sendium/app/storage/StandaloneMessageStorage.java | 1 + .../cytech/sendium/app/storage/StandaloneOutboundPipeline.java | 3 +++ .../gr/cytech/sendium/app/storage/MessageAdmissionProbe.java | 1 + .../cytech/sendium/app/storage/MessageLifecycleHttpTest.java | 2 +- .../sendium/core/smpp/server/MessagePartsOwnershipTest.java | 3 +++ 5 files changed, 9 insertions(+), 1 deletion(-) diff --git a/sendium-app/src/main/java/gr/cytech/sendium/app/storage/StandaloneMessageStorage.java b/sendium-app/src/main/java/gr/cytech/sendium/app/storage/StandaloneMessageStorage.java index 414f577d..828307ba 100644 --- a/sendium-app/src/main/java/gr/cytech/sendium/app/storage/StandaloneMessageStorage.java +++ b/sendium-app/src/main/java/gr/cytech/sendium/app/storage/StandaloneMessageStorage.java @@ -30,6 +30,7 @@ MessageStorageProfile profile(Config config) { } } + // The observed parameter registers early startup validation/logging; its payload is not needed. void start(@Observes @Priority(Interceptor.Priority.PLATFORM_BEFORE) StartupEvent event, MessageStorageProfile profile) { logger.info("Outbound message storage profile (pending/router-queue/routed-work): {}", profile); logger.warn("NON-DURABLE outbound message storage: accepted messages and in-flight work can be lost on process restart."); diff --git a/sendium-app/src/main/java/gr/cytech/sendium/app/storage/StandaloneOutboundPipeline.java b/sendium-app/src/main/java/gr/cytech/sendium/app/storage/StandaloneOutboundPipeline.java index 88fcda32..e3a82483 100644 --- a/sendium-app/src/main/java/gr/cytech/sendium/app/storage/StandaloneOutboundPipeline.java +++ b/sendium-app/src/main/java/gr/cytech/sendium/app/storage/StandaloneOutboundPipeline.java @@ -58,6 +58,7 @@ public class StandaloneOutboundPipeline { private boolean stopping; private final Map executions = new ConcurrentHashMap<>(); + // Resolving profile validates the configured backends before constructing or starting any stage. @Produces @Singleton @DefaultBean @@ -74,6 +75,7 @@ static OutboundCoordinator coordinator(MessageStorageProfile pr return result; } + // The observed parameter registers pipeline startup after worker startup; its payload is not needed. void start(@Observes @Priority(Interceptor.Priority.APPLICATION + 100) StartupEvent event) { if (coordinator == null) { coordinator = coordinators.get(); @@ -181,6 +183,7 @@ private void retryExecutions() { }); } + // The observed parameter registers draining before worker shutdown; its payload is not needed. void stop(@Observes @Priority(Interceptor.Priority.PLATFORM_BEFORE) ShutdownEvent event) throws InterruptedException { if (dispatch == null) { return; diff --git a/sendium-app/src/test/java/gr/cytech/sendium/app/storage/MessageAdmissionProbe.java b/sendium-app/src/test/java/gr/cytech/sendium/app/storage/MessageAdmissionProbe.java index e14ae728..4cfd6ba6 100644 --- a/sendium-app/src/test/java/gr/cytech/sendium/app/storage/MessageAdmissionProbe.java +++ b/sendium-app/src/test/java/gr/cytech/sendium/app/storage/MessageAdmissionProbe.java @@ -15,6 +15,7 @@ public class MessageAdmissionProbe { private MessageStorageProfile profileAtStartup; + // The observed parameter registers the startup callback; the event payload is not needed. void start(@Observes @Priority(Interceptor.Priority.APPLICATION) StartupEvent event) { profileAtStartup = profile; LoggerFactory.getLogger(MessageAdmissionProbe.class).info("test admission startup reached"); diff --git a/sendium-app/src/test/java/gr/cytech/sendium/app/storage/MessageLifecycleHttpTest.java b/sendium-app/src/test/java/gr/cytech/sendium/app/storage/MessageLifecycleHttpTest.java index f32e660d..6d37bc9a 100644 --- a/sendium-app/src/test/java/gr/cytech/sendium/app/storage/MessageLifecycleHttpTest.java +++ b/sendium-app/src/test/java/gr/cytech/sendium/app/storage/MessageLifecycleHttpTest.java @@ -125,7 +125,7 @@ private static StandardMessage snapshot(StandardMessage message) { @Vetoed public static class Credentials extends CredentialFileWatcher { @Override - public Map getValidCredentials() { + public synchronized Map getValidCredentials() { return Map.of("http-user", new Credential(CredentialType.HTTP, null, null, "http-user", "secret", null, Set.of())); } diff --git a/sendium-core/src/test/java/gr/cytech/sendium/core/smpp/server/MessagePartsOwnershipTest.java b/sendium-core/src/test/java/gr/cytech/sendium/core/smpp/server/MessagePartsOwnershipTest.java index adc75ee3..ab6d7793 100644 --- a/sendium-core/src/test/java/gr/cytech/sendium/core/smpp/server/MessagePartsOwnershipTest.java +++ b/sendium-core/src/test/java/gr/cytech/sendium/core/smpp/server/MessagePartsOwnershipTest.java @@ -67,16 +67,19 @@ private static final class Listener implements MessagePartsEventsListener parts) { events.add(type); groups.add(parts); } + @Override public void onDuplicateMessagePart(StandardMessage original, StandardMessage duplicate) { this.original = original; this.duplicate = duplicate; } + @Override public String getName() { return "ownership-test"; } From d9f669e833da3589b98205016a20d53c109d71de Mon Sep 17 00:00:00 2001 From: pavlos Date: Tue, 20 Oct 2026 16:29:09 +0300 Subject: [PATCH 20/20] fix(storage): restore outbound progress under sustained load Reuse private completion monitors and notification-based drain waits to avoid retaining future copies during backlog polling. Drain per-worker FIFO attempts directly instead of scheduling per-message contention timers, preserving preparation concurrency and asynchronous SMPP windows. Report fatal processing and rescheduling failures through standalone readiness. Cover monitor retention, FIFO and pause scheduling, shutdown drain, and processor failure handling. Full reactor verify passes 396 core unit tests, 39 integration tests and 53 app tests with zero Checkstyle violations. Matched 100-msg/s HTTP+DLR and SMPP+DLR rehearsals each complete all 60,000 messages and receipts without rejection or unfinished work. Signed-off-by: pavlos --- docs/14-outbound-storage.md | 21 +- .../storage/StandaloneOutboundPipeline.java | 46 +++- .../StandaloneOutboundSchedulingTest.java | 41 ++- .../routing/StandardOutboundDispatch.java | 234 ++++++++++++------ .../smpp/client/SmppClientWorkerTest.java | 95 ++++++- .../routing/StandardOutboundDispatchTest.java | 51 ++++ 6 files changed, 406 insertions(+), 82 deletions(-) diff --git a/docs/14-outbound-storage.md b/docs/14-outbound-storage.md index fac2c1c4..4ce94894 100644 --- a/docs/14-outbound-storage.md +++ b/docs/14-outbound-storage.md @@ -131,9 +131,9 @@ or provider scheduling. Idle, paused, or unsuccessful cycles back off for 100 mi selection or returning an unroutable item alone does not count as progress. Shutdown prevents further cycle scheduling. Batch size is not TPS or provider concurrency: attempts use the worker's shared rate limiter and a dispatcher-owned execution pool -sized by its `threadCount`. A sending slot is reserved before executor submission, bounding queued -plus executing preparation tasks. The default shared two-thread scheduler handles destination -resolution and timed retries; rate waits, worker filters, and SMPP sends run on the worker's pool. +sized by its `threadCount`. At most that many preparation loops execute concurrently, taking waiting +attempts from the worker FIFO. The default shared two-thread scheduler handles destination resolution +and timed retries; rate waits, worker filters, and SMPP sends run on the worker's pool. Stopped-worker pools are retired during transition maintenance, and dispatcher close shuts down its worker pools after provider drain even when the timer scheduler is application-owned. Awaiting asynchronous outcomes does not hold a sending slot. Count limits do not bound message bytes. @@ -175,6 +175,21 @@ Each provider response is claimed once; duplicate callbacks cannot create anothe complete unrelated sources. Admission deduplication applies while ownership is retained, not forever. Runtime worker removal's legacy queue return does not migrate coordinator-owned parent records. +Provider preparation uses a per-worker FIFO of already-owned runtime attempts. Up to the worker's +`threadCount` preparation threads drain it directly as capacity becomes available; busy work does not +create a separate contention timer per message. Paused backlogs share one worker wake-up. Preparation +capacity is released after preparation/submission, without waiting for asynchronous provider responses +or DLR handoffs. The retained source-capacity bound still governs admitted work. + +Each provider execution retains one private completion monitor per terminal-cleanup attempt. Status +polling reuses that monitor, and repeated drain timeouts wait on completion notifications instead of +attaching new future copies or aggregate completion trees to outstanding work. + +The standalone pipeline contributes the `standalone-outbound` readiness check. Recoverable processing +cycle failures back off and retry. A fatal cycle failure or failed rescheduling is logged with its +exception and makes readiness DOWN, with only the failure class exposed in health data. Readiness is +also DOWN before pipeline activation and during shutdown; a terminated chain is not silently restarted. + ## Graceful Shutdown ```mermaid diff --git a/sendium-app/src/main/java/gr/cytech/sendium/app/storage/StandaloneOutboundPipeline.java b/sendium-app/src/main/java/gr/cytech/sendium/app/storage/StandaloneOutboundPipeline.java index e3a82483..99c09daa 100644 --- a/sendium-app/src/main/java/gr/cytech/sendium/app/storage/StandaloneOutboundPipeline.java +++ b/sendium-app/src/main/java/gr/cytech/sendium/app/storage/StandaloneOutboundPipeline.java @@ -30,6 +30,9 @@ import jakarta.inject.Singleton; import jakarta.interceptor.Interceptor; import org.eclipse.microprofile.config.Config; +import org.eclipse.microprofile.health.HealthCheck; +import org.eclipse.microprofile.health.HealthCheckResponse; +import org.eclipse.microprofile.health.Readiness; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -43,7 +46,8 @@ /** Standalone-only assembly; importing sendium-core never starts these processors. */ @ApplicationScoped -public class StandaloneOutboundPipeline { +@Readiness +public class StandaloneOutboundPipeline implements HealthCheck { private static final Logger logger = LoggerFactory.getLogger(StandaloneOutboundPipeline.class); @Inject Instance> coordinators; OutboundCoordinator coordinator; @@ -56,6 +60,7 @@ public class StandaloneOutboundPipeline { private int batchSize; private final Object schedulingLock = new Object(); private boolean stopping; + private String processorFailure; private final Map executions = new ConcurrentHashMap<>(); // Resolving profile validates the configured backends before constructing or starting any stage. @@ -93,7 +98,7 @@ void start(@Observes @Priority(Interceptor.Priority.APPLICATION + 100) StartupEv private void scheduleNextCycle(long delay) { synchronized (schedulingLock) { - if (!stopping) { + if (!stopping && processorFailure == null) { processors.schedule(this::runProcessingCycle, delay, TimeUnit.MILLISECONDS); } } @@ -101,12 +106,38 @@ private void scheduleNextCycle(long delay) { private void runProcessingCycle() { synchronized (schedulingLock) { - if (stopping) { + if (stopping || processorFailure != null) { return; } } - boolean progressed = process(); - scheduleNextCycle(progressed ? 0 : 100); + try { + boolean progressed; + try { + progressed = process(); + } catch (RuntimeException failure) { + logFailure("processing cycle", failure); + progressed = false; + } + scheduleNextCycle(progressed ? 0 : 100); + } catch (RuntimeException | Error failure) { + synchronized (schedulingLock) { + processorFailure = failure.getClass().getName(); + } + logger.error("Standalone outbound processor terminated; readiness is DOWN", failure); + throw failure; + } + } + + @Override + public HealthCheckResponse call() { + synchronized (schedulingLock) { + var health = HealthCheckResponse.named("standalone-outbound") + .status(processors != null && !stopping && processorFailure == null); + if (processorFailure != null) { + health.withData("failureType", processorFailure); + } + return health.build(); + } } boolean process() { @@ -171,12 +202,11 @@ boolean process() { private void retryExecutions() { executions.entrySet().removeIf(entry -> { var execution = entry.getValue(); - var completion = execution.completion().toCompletableFuture(); - if (completion.isDone() && !completion.isCompletedExceptionally()) { + if (execution.isCompletedSuccessfully()) { return true; } execution.retryPendingHandoffs(); - if (completion.isCompletedExceptionally()) { + if (execution.hasCompletionFailure()) { execution.retryTerminalCleanup(); } return false; diff --git a/sendium-app/src/test/java/gr/cytech/sendium/app/storage/StandaloneOutboundSchedulingTest.java b/sendium-app/src/test/java/gr/cytech/sendium/app/storage/StandaloneOutboundSchedulingTest.java index 2fd51121..afce21a6 100644 --- a/sendium-app/src/test/java/gr/cytech/sendium/app/storage/StandaloneOutboundSchedulingTest.java +++ b/sendium-app/src/test/java/gr/cytech/sendium/app/storage/StandaloneOutboundSchedulingTest.java @@ -9,6 +9,7 @@ import gr.cytech.sendium.routing.RoutingLookupResult; import gr.cytech.sendium.routing.StandardRoutingManager; import org.eclipse.microprofile.config.Config; +import org.eclipse.microprofile.health.HealthCheckResponse; import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.ValueSource; @@ -24,6 +25,7 @@ import java.util.concurrent.TimeUnit; import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyLong; import static org.mockito.ArgumentMatchers.anyString; @@ -40,6 +42,43 @@ import static org.mockito.Mockito.when; class StandaloneOutboundSchedulingTest { + @Test + void recoverableCycleFailureBacksOffAndKeepsReadinessUp() throws Exception { + try (var fixture = new SchedulingFixture()) { + doThrow(new IllegalStateException("cycle unavailable")).doCallRealMethod().when(fixture.pipeline).process(); + fixture.runNext(); + assertThat(fixture.nextDelay()).isEqualTo(100); + assertThat(fixture.pipeline.call().getStatus()).isEqualTo(HealthCheckResponse.Status.UP); + fixture.accept(1); + fixture.runNext(); + assertThat(fixture.nextDelay()).isZero(); + verify(fixture.routing).lookupForLifecycle(any()); + } + } + + @Test + void fatalCycleFailureMarksReadinessDownAndDoesNotSilentlyReschedule() throws Exception { + try (var fixture = new SchedulingFixture()) { + var failure = new LinkageError("simulated fatal processor failure"); + doThrow(failure).when(fixture.pipeline.coordinator).selectToRouter(2); + assertThatThrownBy(fixture::runNext).isSameAs(failure); + assertThat(fixture.cycles).hasSize(1); + var health = fixture.pipeline.call(); + assertThat(health.getStatus()).isEqualTo(HealthCheckResponse.Status.DOWN); + assertThat(health.getData().orElseThrow()).containsEntry("failureType", LinkageError.class.getName()); + } + } + + @Test + void failedReschedulingMarksReadinessDown() throws Exception { + try (var fixture = new SchedulingFixture()) { + doThrow(new java.util.concurrent.RejectedExecutionException("scheduler unavailable")) + .when(fixture.scheduler).schedule(any(Runnable.class), anyLong(), eq(TimeUnit.MILLISECONDS)); + assertThatThrownBy(fixture::runNext).isInstanceOf(java.util.concurrent.RejectedExecutionException.class); + assertThat(fixture.pipeline.call().getStatus()).isEqualTo(HealthCheckResponse.Status.DOWN); + } + } + @Test void productiveBacklogContinuesImmediatelyInBoundedCyclesThenBacksOff() throws Exception { try (var fixture = new SchedulingFixture()) { @@ -145,7 +184,7 @@ void shutdownFencesQueuedCyclesAndFurtherRescheduling() throws Exception { private static final class SchedulingFixture implements AutoCloseable { private final ScheduledExecutorService scheduler = mock(ScheduledExecutorService.class); private final StandardRoutingManager routing = mock(StandardRoutingManager.class); - private final StandaloneOutboundPipeline pipeline = new StandaloneOutboundPipeline(); + private final StandaloneOutboundPipeline pipeline = spy(new StandaloneOutboundPipeline()); private final List cycles = new ArrayList<>(); private final List delays = new ArrayList<>(); private int nextCycle; diff --git a/sendium-core/src/main/java/gr/cytech/sendium/routing/StandardOutboundDispatch.java b/sendium-core/src/main/java/gr/cytech/sendium/routing/StandardOutboundDispatch.java index 3ec568dc..017553d3 100644 --- a/sendium-core/src/main/java/gr/cytech/sendium/routing/StandardOutboundDispatch.java +++ b/sendium-core/src/main/java/gr/cytech/sendium/routing/StandardOutboundDispatch.java @@ -16,6 +16,7 @@ import java.io.IOException; import java.time.Duration; +import java.util.ArrayDeque; import java.util.List; import java.util.Objects; import java.util.Optional; @@ -27,7 +28,7 @@ import java.util.concurrent.CopyOnWriteArrayList; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.Semaphore; +import java.util.concurrent.ScheduledFuture; import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; import java.util.function.Supplier; @@ -39,7 +40,8 @@ public final class StandardOutboundDispatch implements AutoCloseable { private final ScheduledExecutorService scheduler; private final boolean ownsScheduler; private final ConcurrentMap, WorkerExecution> workerExecutors = new ConcurrentHashMap<>(); - private final ConcurrentMap> activeExecutions = new ConcurrentHashMap<>(); + private final ConcurrentMap> activeExecutions = new ConcurrentHashMap<>(); + private final Object completionLock = new Object(); private boolean shuttingDown; private boolean closed; private final ConcurrentMap routingTransitions = new ConcurrentHashMap<>(); @@ -158,7 +160,7 @@ public synchronized ProviderExecution submitToProvider(Routed w execution.retryCompletion = () -> track(work.id(), coordinator.complete(work.id(), execution.attempts)); CompletionStage attempts = scheduleAttempt(work.message(), () -> worker, execution); execution.attempts = attempts; - execution.completion = track(work.id(), coordinator.complete(work.id(), attempts)); + execution.setCompletion(track(work.id(), coordinator.complete(work.id(), attempts))); return execution; } @@ -197,33 +199,53 @@ public boolean awaitProviderDrain(Duration timeout) throws InterruptedException if (timeout.isNegative()) { throw new IllegalArgumentException("Negative drain timeout"); } - CompletableFuture[] completions; + long remaining = timeout.toNanos(); synchronized (this) { if (!shuttingDown) { throw new IllegalStateException("Begin dispatch shutdown before awaiting provider drain"); } - completions = activeExecutions.values().stream().map(CompletionStage::toCompletableFuture) - .toArray(CompletableFuture[]::new); } - try { - CompletableFuture.allOf(completions).get(timeout.toNanos(), TimeUnit.NANOSECONDS); - removeSuccessfulExecutions(); - return true; - } catch (java.util.concurrent.TimeoutException timeoutFailure) { - return false; - } catch (java.util.concurrent.ExecutionException failure) { - throw new IllegalStateException("Resolve failed provider handoffs or terminal cleanup before shutdown", failure.getCause()); + synchronized (completionLock) { + while (true) { + removeSuccessfulExecutions(); + for (var completion : activeExecutions.values()) { + if (completion.isCompletedExceptionally()) { + try { + completion.join(); + } catch (java.util.concurrent.CompletionException | java.util.concurrent.CancellationException failure) { + throw new IllegalStateException("Resolve failed provider handoffs or terminal cleanup before shutdown", + failure.getCause() == null ? failure : failure.getCause()); + } + } + } + if (activeExecutions.isEmpty()) { + return true; + } + if (remaining <= 0) { + return false; + } + long started = System.nanoTime(); + TimeUnit.NANOSECONDS.timedWait(completionLock, remaining); + remaining -= System.nanoTime() - started; + } } } - private CompletionStage track(WorkId id, CompletionStage completion) { - activeExecutions.put(id, completion); - completion.whenComplete((ignored, failure) -> { - if (failure == null) { - activeExecutions.remove(id, completion); + private CompletableFuture track(WorkId id, CompletionStage completion) { + var monitor = completion.toCompletableFuture(); + synchronized (completionLock) { + activeExecutions.put(id, monitor); + completionLock.notifyAll(); + } + monitor.whenComplete((ignored, failure) -> { + synchronized (completionLock) { + if (failure == null) { + activeExecutions.remove(id, monitor); + } + completionLock.notifyAll(); } }); - return completion; + return monitor; } private void requireDispatching() { @@ -233,10 +255,9 @@ private void requireDispatching() { } private void removeSuccessfulExecutions() { - activeExecutions.entrySet().removeIf(entry -> { - var completion = entry.getValue().toCompletableFuture(); - return completion.isDone() && !completion.isCompletedExceptionally(); - }); + synchronized (completionLock) { + activeExecutions.values().removeIf(completion -> completion.isDone() && !completion.isCompletedExceptionally()); + } } /** Call after drain and execution-loop stop/join; parent stores remain application-owned. */ @@ -251,7 +272,7 @@ public synchronized void close() { throw new IllegalStateException("Resolve unfinished routing transitions before closing outbound dispatch"); } closed = true; - workerExecutors.values().forEach(execution -> execution.executor.shutdown()); + workerExecutors.values().forEach(WorkerExecution::stop); workerExecutors.clear(); if (ownsScheduler) { scheduler.shutdown(); @@ -270,7 +291,7 @@ private void retireStoppedWorkers() { if (entry.getKey().isKeepOnRunning()) { return false; } - entry.getValue().executor.shutdown(); + entry.getValue().stop(); return true; }); } @@ -285,22 +306,100 @@ public synchronized CompletionStage finishProviderParts(Routed[] parts = handoffs.stream().map(CompletionStage::toCompletableFuture) .toArray(CompletableFuture[]::new); - return track(work.id(), coordinator.complete(work.id(), CompletableFuture.allOf(parts))); + return track(work.id(), coordinator.complete(work.id(), CompletableFuture.allOf(parts))).minimalCompletionStage(); } - /** Slots bound submitted preparation tasks, including any tasks waiting in the executor. */ - private static final class WorkerExecution { - private final Semaphore slots; + /** Owned attempts wait once in FIFO order; preparation threads continue directly when capacity frees. */ + private final class WorkerExecution { + private final SmppClientWorker worker; + private final int threads; + private final ArrayDeque waiting = new ArrayDeque<>(); private final ThreadPoolExecutor executor; + private int running; + private ScheduledFuture wakeup; private WorkerExecution(SmppClientWorker worker) { - int threads = Math.max(1, worker.getThreadCount()); - slots = new Semaphore(threads); + this.worker = worker; + threads = Math.max(1, worker.getThreadCount()); executor = new ThreadPoolExecutor(threads, threads, 30, TimeUnit.SECONDS, new ArrayBlockingQueue<>(threads), Thread.ofPlatform().name("outbound-provider-" + worker.getFullName() + "-", 1).daemon(true).factory()); executor.allowCoreThreadTimeOut(true); } + + private synchronized void enqueue(ProviderAttempt attempt) { + waiting.addLast(attempt); + dispatchWaiting(); + } + + private void dispatchWaiting() { + if (waiting.isEmpty()) { + return; + } + try { + if (!worker.isKeepOnRunning() || executor.isShutdown()) { + throw new IllegalStateException("Destination worker stopped: " + worker.getFullName()); + } + if (worker.isPause()) { + if (wakeup == null) { + wakeup = scheduler.schedule(this::resume, 100, TimeUnit.MILLISECONDS); + } + return; + } + int available = Math.min(threads - running, waiting.size()); + for (int i = 0; i < available; i++) { + running++; + try { + executor.execute(this::drain); + } catch (RuntimeException failure) { + running--; + throw failure; + } + } + } catch (RuntimeException failure) { + failWaiting(failure); + } + } + + private synchronized void resume() { + wakeup = null; + dispatchWaiting(); + } + + private void drain() { + try { + while (true) { + ProviderAttempt next; + synchronized (this) { + if (waiting.isEmpty() || !worker.isKeepOnRunning() || worker.isPause()) { + return; + } + next = waiting.removeFirst(); + } + next.prepare(); + } + } finally { + synchronized (this) { + running--; + dispatchWaiting(); + } + } + } + + private void failWaiting(RuntimeException failure) { + while (!waiting.isEmpty()) { + waiting.removeFirst().fail(failure); + } + } + + private synchronized void stop() { + if (wakeup != null) { + wakeup.cancel(false); + wakeup = null; + } + failWaiting(new IllegalStateException("Destination worker stopped: " + worker.getFullName())); + executor.shutdown(); + } } public enum AttemptState { @@ -358,8 +457,6 @@ public void run() { } state = AttemptState.PREPARING; } - WorkerExecution destination = null; - boolean reserved = false; try { if (worker == null) { worker = Objects.requireNonNull(resolveDestination.get(), "retry destination"); @@ -367,38 +464,24 @@ public void run() { if (!worker.isKeepOnRunning()) { throw new IllegalStateException("Destination worker stopped: " + worker.getFullName()); } - if (worker.isPause()) { - synchronized (this) { - state = AttemptState.QUEUED; - schedule(100); - } - return; - } - destination = executionFor(worker); - if (!destination.slots.tryAcquire()) { - synchronized (this) { - state = AttemptState.QUEUED; - schedule(100); - } - return; - } - reserved = true; - WorkerExecution reservedDestination = destination; - destination.executor.execute(() -> prepare(reservedDestination)); - reserved = false; - } catch (Exception error) { synchronized (this) { - state = AttemptState.FAILED; - failure = error; - } - } finally { - if (reserved) { - destination.slots.release(); + state = AttemptState.QUEUED; } + executionFor(worker).enqueue(this); + } catch (Exception error) { + fail(error); } } - private void prepare(WorkerExecution destination) { + private synchronized void fail(Throwable error) { + state = AttemptState.FAILED; + failure = error; + } + + private void prepare() { + synchronized (this) { + state = AttemptState.PREPARING; + } try { var submission = worker.submitPreparedCoordinated(message, (payload, policy) -> scheduleAttempt(payload, policy == SmppClientWorker.NackHandlePolicy.RETRY_ROUTER ? @@ -417,12 +500,10 @@ private void prepare(WorkerExecution destination) { } }); } catch (Exception error) { - synchronized (this) { - state = AttemptState.FAILED; - failure = error; - } - } finally { - destination.slots.release(); + fail(error); + } catch (Error error) { + fail(error); + throw error; } } } @@ -432,13 +513,28 @@ public static final class ProviderExecution { private final List> submissions = new CopyOnWriteArrayList<>(); private final List providerAttempts = new CopyOnWriteArrayList<>(); private CompletionStage completion; + private CompletableFuture completionMonitor; private CompletionStage attempts; - private java.util.function.Supplier> retryCompletion; + private Supplier> retryCompletion; + + private void setCompletion(CompletableFuture monitor) { + completionMonitor = monitor; + completion = monitor.minimalCompletionStage(); + } public CompletionStage completion() { return completion; } + /** Status checks reuse the monitor instead of copying an incomplete minimal stage on every poll. */ + public boolean isCompletedSuccessfully() { + return completionMonitor.isDone() && !completionMonitor.isCompletedExceptionally(); + } + + public boolean hasCompletionFailure() { + return completionMonitor.isCompletedExceptionally(); + } + public void retryPendingHandoffs() { List.copyOf(submissions).forEach(CoordinatedSmppSubmission::retryPendingHandoffs); List.copyOf(providerAttempts).forEach(ProviderAttempt::retry); @@ -451,7 +547,7 @@ public List attempts() { /** Retries store cleanup after successful handoffs without sending the message again. */ public CompletionStage retryTerminalCleanup() { - completion = retryCompletion.get(); + setCompletion(retryCompletion.get()); return completion; } } diff --git a/sendium-core/src/test/java/gr/cytech/sendium/core/smpp/client/SmppClientWorkerTest.java b/sendium-core/src/test/java/gr/cytech/sendium/core/smpp/client/SmppClientWorkerTest.java index f66743dd..16665422 100644 --- a/sendium-core/src/test/java/gr/cytech/sendium/core/smpp/client/SmppClientWorkerTest.java +++ b/sendium-core/src/test/java/gr/cytech/sendium/core/smpp/client/SmppClientWorkerTest.java @@ -41,7 +41,9 @@ import java.util.concurrent.CompletableFuture; import java.util.concurrent.CompletionStage; import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ScheduledFuture; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; @@ -117,7 +119,6 @@ public StandardMessage dequeue(long timeout) throws InterruptedException { @Test void legacyAndCoordinatedMessagesShareTheConfiguredWorkerRateLimit() throws Exception { var worker = new LegacyRateWorker(); - worker.configureRate(2); var queue = new Queue(); queue.enqueue(messageWithNetwork()); worker.useQueue(queue); @@ -125,6 +126,8 @@ void legacyAndCoordinatedMessagesShareTheConfiguredWorkerRateLimit() throws Exce when(container.requestContext()).thenReturn(mock(ManagedContext.class)); try (var arc = mockStatic(Arc.class)) { arc.when(Arc::container).thenReturn(container); + // Start the limiter after mock instrumentation, which otherwise accrues stored permits. + worker.configureRate(2); worker.new Worker(0).run(); } assertThat(worker.legacySends).isEqualTo(1); @@ -259,6 +262,96 @@ void workerSlotAndRateGatePreventConcurrentPreparationAndEarlySending() throws E } } + @Test + void busyWorkerDrainsItsFifoWithoutContentionTimersOrWaitingForProviderResponses() throws Exception { + var delayedTasks = new AtomicInteger(); + var scheduler = new ScheduledThreadPoolExecutor(1) { + @Override + public ScheduledFuture schedule(Runnable task, long delay, TimeUnit unit) { + if (delay > 0) { + delayedTasks.incrementAndGet(); + return super.schedule(task, 1, TimeUnit.DAYS); + } + return super.schedule(task, delay, unit); + } + }; + var rateGate = new CompletableFuture(); + var worker = new TestSmppClientWorker(new TestConfigurationProvider(), new Queue<>(), new CapturingTracker()); + worker.rateGate = rateGate; + int count = 256; + try (var fixture = new OutboundIngressFixture(count); + var dispatch = new StandardOutboundDispatch(fixture.coordinator, mock(StandardRoutingManager.class), scheduler)) { + var executions = new java.util.ArrayList(); + var messages = new java.util.ArrayList(); + for (int i = 0; i < count; i++) { + var message = messageWithNetwork(); + message.body = "message-" + i; + messages.add(message.body); + fixture.coordinator.accept(new SourceId(UUID.randomUUID()), message); + } + fixture.coordinator.selectToRouter(count); + for (int i = 0; i < count; i++) { + var selected = fixture.coordinator.takeForRouting(Duration.ZERO).orElseThrow(); + fixture.coordinator.recordToRouted(selected, new Destination<>(worker.getFullName(), selected.message())); + executions.add(dispatch.submitToProvider( + fixture.coordinator.takeFromRouted(worker.getFullName(), Duration.ZERO).orElseThrow(), worker)); + } + assertThat(worker.rateEntered.await(5, TimeUnit.SECONDS)).isTrue(); + scheduler.submit(() -> { }).get(5, TimeUnit.SECONDS); + assertThat(worker.rateChecks).isEqualTo(1); + assertThat(delayedTasks).hasValue(0); + dispatch.beginShutdown(); + rateGate.complete(null); + worker.awaitRequests(count); + assertThat(worker.coordinatedRequests).extracting(request -> + CharsetUtil.decode(request.getShortMessage(), CharsetUtil.NAME_GSM)).containsExactlyElementsOf(messages); + assertThat(delayedTasks).hasValue(0); + // Preparation slots are released before provider responses/handoffs, preserving the SMPP window. + assertThat(executions).allSatisfy(execution -> assertThat(execution.isCompletedSuccessfully()).isFalse()); + worker.coordinatedRequests.forEach(request -> respond(handler(worker), request, SmppConstants.STATUS_OK)); + assertThat(dispatch.awaitProviderDrain(Duration.ofSeconds(5))).isTrue(); + assertThat(executions).allSatisfy(execution -> assertThat(execution.isCompletedSuccessfully()).isTrue()); + } finally { + rateGate.complete(null); + scheduler.shutdownNow(); + } + } + + @Test + void pausedWorkerSharesOneWakeupAcrossItsBacklogAndResumesDuringDrain() throws Exception { + var wakeups = new java.util.concurrent.CopyOnWriteArrayList(); + var scheduler = new ScheduledThreadPoolExecutor(1) { + @Override + public ScheduledFuture schedule(Runnable task, long delay, TimeUnit unit) { + if (delay > 0) { + wakeups.add(task); + return super.schedule(task, 1, TimeUnit.DAYS); + } + return super.schedule(task, delay, unit); + } + }; + var worker = new TestSmppClientWorker(new TestConfigurationProvider(), new Queue<>(), new CapturingTracker()); + worker.testPaused = true; + try (var fixture = new OutboundIngressFixture(16); + var dispatch = new StandardOutboundDispatch(fixture.coordinator, mock(StandardRoutingManager.class), scheduler)) { + for (int i = 0; i < 16; i++) { + submitOwned(dispatch, fixture, worker); + } + scheduler.submit(() -> { }).get(5, TimeUnit.SECONDS); + assertThat(wakeups).hasSize(1); + assertThat(worker.preparations).isZero(); + dispatch.beginShutdown(); + worker.testPaused = false; + wakeups.getFirst().run(); + worker.awaitRequests(16); + worker.coordinatedRequests.forEach(request -> respond(handler(worker), request, SmppConstants.STATUS_OK)); + assertThat(dispatch.awaitProviderDrain(Duration.ofSeconds(5))).isTrue(); + assertThat(wakeups).hasSize(1); + } finally { + scheduler.shutdownNow(); + } + } + @Test void blockedWorkerUsesItsConfiguredSlotsWithoutBlockingOtherDestinationsOrTimers() throws Exception { var scheduler = new ScheduledThreadPoolExecutor(1); diff --git a/sendium-core/src/test/java/gr/cytech/sendium/routing/StandardOutboundDispatchTest.java b/sendium-core/src/test/java/gr/cytech/sendium/routing/StandardOutboundDispatchTest.java index 263ec0ef..4a3449b4 100644 --- a/sendium-core/src/test/java/gr/cytech/sendium/routing/StandardOutboundDispatchTest.java +++ b/sendium-core/src/test/java/gr/cytech/sendium/routing/StandardOutboundDispatchTest.java @@ -2,8 +2,12 @@ import gr.cytech.sendium.core.AbstractOutWorker; import gr.cytech.sendium.core.message.StandardMessage; +import gr.cytech.sendium.core.outbound.OutboundCoordinator; import gr.cytech.sendium.core.outbound.OutboundWork.Routed; +import gr.cytech.sendium.core.outbound.OutboundWork.SelectionId; import gr.cytech.sendium.core.outbound.OutboundWork.SourceId; +import gr.cytech.sendium.core.outbound.OutboundWork.WorkId; +import gr.cytech.sendium.core.smpp.client.SmppClientWorker; import gr.cytech.sendium.core.storage.OutboundStorageException; import gr.cytech.sendium.external.filter.FilterException; import gr.cytech.sendium.external.filter.FilterStatusCodes; @@ -16,6 +20,7 @@ import java.util.Optional; import java.util.UUID; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.atomic.AtomicInteger; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; @@ -27,6 +32,52 @@ import static org.mockito.Mockito.when; class StandardOutboundDispatchTest { + @Test + @SuppressWarnings("unchecked") + void repeatedStatusChecksAndDrainTimeoutsKeepOneMonitorPerCleanupAttempt() throws Exception { + var coordinator = mock(OutboundCoordinator.class); + var first = new CopyingCompletion(); + var replacement = new CopyingCompletion(); + var work = new Routed<>(new WorkId(UUID.randomUUID()), new SelectionId(UUID.randomUUID()), + "smpp.provider", message("hello")); + when(coordinator.complete(any(), any())).thenReturn(first, replacement); + var provider = mock(SmppClientWorker.class); + when(provider.getFullName()).thenReturn(work.destination()); + try (var dispatch = new StandardOutboundDispatch(coordinator, mock(StandardRoutingManager.class))) { + var execution = dispatch.submitToProvider(work, provider); + dispatch.beginShutdown(); + for (int i = 0; i < 10_000; i++) { + assertThat(execution.isCompletedSuccessfully()).isFalse(); + assertThat(execution.hasCompletionFailure()).isFalse(); + assertThat(dispatch.awaitProviderDrain(Duration.ZERO)).isFalse(); + } + assertThat(first.copies).hasValue(1); + // Neither repeated waits nor their timeouts add dependents to the retained handoff. + assertThat(first.getNumberOfDependents()).isEqualTo(1); + + first.completeExceptionally(new IllegalStateException("cleanup unavailable")); + assertThat(execution.hasCompletionFailure()).isTrue(); + assertThatThrownBy(() -> dispatch.awaitProviderDrain(Duration.ZERO)) + .isInstanceOf(IllegalStateException.class).hasRootCauseMessage("cleanup unavailable"); + execution.retryTerminalCleanup(); + assertThat(execution.hasCompletionFailure()).isFalse(); + assertThat(replacement.copies).hasValue(1); + replacement.complete(null); + assertThat(execution.isCompletedSuccessfully()).isTrue(); + assertThat(dispatch.awaitProviderDrain(Duration.ZERO)).isTrue(); + } + } + + private static final class CopyingCompletion extends CompletableFuture { + private final AtomicInteger copies = new AtomicInteger(); + + @Override + public CompletableFuture toCompletableFuture() { + copies.incrementAndGet(); + return minimalCompletionStage().toCompletableFuture(); + } + } + @Test void interruptedDestinationRecordingRetriesTheOriginalIntentWithoutRepeatingLookup() throws Exception { try (var fixture = new OutboundIngressFixture(1)) {