From 2b22b2e082c514e1c8b1b82bb44b0e20a12ad522 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dominik=20Kr=C3=BCmpelmann?= Date: Thu, 22 Oct 2026 19:52:59 +0200 Subject: [PATCH 1/4] feat(shm): add versioned object headers Every named object now carries a stable ShmObjectHeader with separate header and payload versions. The payload lives in a separate allocation behind a relative offset, so changes to the library's bookkeeping no longer move it. Add ShmPersistence::{Volatile, Derived, Persistent} and per-type v(n)->v(n+1) payload migrations. Segments whose objects are all volatile or derived can be recreated without operator input; persistent objects still require a migration or a registered migrator. BREAKING CHANGE: the segment layout is version 3. Segments written by older builds have no readable per-object header and still need the existing migrator or LINTEL_SHM_RECREATE path. --- CHANGELOG.md | 15 + docs/shm-migration-strategy.md | 94 +++- docs/shm-payload-abi-plan.md | 130 ++++++ src/CMakeLists.txt | 4 +- .../plugins/SharedMemoryBootstrapPlugin.cpp | 25 +- src/core/shm/ShmView.cpp | 24 +- src/features/base/msg/MessageQueueShmCore.cpp | 52 ++- .../base/services/EventBusService.cpp | 14 +- .../services/SharedMemoryCleanupService.cpp | 41 +- .../base/services/SharedMemoryRepository.cpp | 211 --------- .../base/services/SharedMemoryService.cpp | 100 +++- src/features/base/services/ShmObject.cpp | 434 ++++++++++++++++++ .../SharedMemoryPropertyRepository.cpp | 22 +- .../lintel/core/shm/SharedMemoryRepository.h | 185 +++----- src/include/lintel/core/shm/ShmObject.h | 120 +++++ src/include/lintel/core/shm/ShmObjectHeader.h | 237 ++++++++++ .../lintel/core/shm/ShmSegmentAccessor.h | 252 ++++++++++ src/include/lintel/core/utils/TypeName.h | 28 +- .../repositories/SharedMemoryRepository.h | 46 +- .../features/base/services/ShmConstructors.h | 267 +++++++++-- tests/CMakeLists.txt | 1 + tests/features/SharedMemoryServiceTest.cpp | 4 +- tests/features/ShmObjectMigrationTest.cpp | 259 +++++++++++ .../SharedMemoryBootstrapPluginTest.cpp | 12 +- 24 files changed, 2129 insertions(+), 448 deletions(-) create mode 100644 docs/shm-payload-abi-plan.md delete mode 100644 src/features/base/services/SharedMemoryRepository.cpp create mode 100644 src/features/base/services/ShmObject.cpp create mode 100644 src/include/lintel/core/shm/ShmObject.h create mode 100644 src/include/lintel/core/shm/ShmObjectHeader.h create mode 100644 tests/features/ShmObjectMigrationTest.cpp diff --git a/CHANGELOG.md b/CHANGELOG.md index 69fb516c..cf6471f1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,16 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] ### Added +- **Stable shared-memory object headers, payload migrations and persistence classes.** Every + named object in a segment now starts with a 64-byte `lintel::shm::ShmObjectHeader` + (`headerVersion`, `payloadVersion`, type id, persistence class); the payload is a separate + allocation addressed by a relative offset, so a change to the library's bookkeeping no longer + moves the payload. `shm::registerPayloadMigration(n, n+1, …)` chains per-type migrations, + and `shm::ShmObjectOptions { payloadVersion, persistence, payloadTypeId }` is accepted by + `shm::SharedMemoryRepository`, the `shm::construct*` helpers and the JSON mirror repositories. + `Volatile`/`Derived` objects may be recreated without operator input, `Persistent` (the default) + aborts instead. The segment layout marker is now version 3. See + `docs/shm-payload-abi-plan.md` and `docs/shm-migration-strategy.md`. - **Group management over HTTP and in the CLI.** `POST /user/groups`, `DELETE /user/groups/{name}` and `POST`/`DELETE` `/user/groups/{name}/subgroups/{sub}` (all `Admin-User`) expose the group operations `GroupRepository` already had; the CLI gains `group_add`, `group_remove`, @@ -212,6 +222,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 `shm::SharedMemoryRepository` and the containers it stores - is exactly what such a migrator is for; properties (database or bootstrap XML, the shared memory repository is a mirror that is rebuilt), users, groups and history live outside the segment and stay untouched. +- **In-segment layout is version 3 (breaking).** Named shared-memory objects are now + `ShmObjectHeader` + separately allocated payload instead of a raw `find_or_construct()` + object. Segments written by an older build have no readable per-object header and are treated + as `Persistent`; they still need the existing migrator/`LINTEL_SHM_RECREATE=1` path. See + `docs/shm-payload-abi-plan.md`. - **`SharedMemoryRepositoryRegistry.h` moved** from `features/base/services/` to `core/shm/`; the typed repository header may not depend on a feature header. - **Logging and property registration use `std::source_location` (breaking).** diff --git a/docs/shm-migration-strategy.md b/docs/shm-migration-strategy.md index 796840b0..56e23c73 100644 --- a/docs/shm-migration-strategy.md +++ b/docs/shm-migration-strategy.md @@ -1,6 +1,6 @@ # Migrationsstrategie für Shared-Memory-Segmente -Stand: 22. Oktober 2026 · gilt ab Layout-Version 2 (`lintel_segment_layout`) +Stand: 22. Oktober 2026 · gilt ab Layout-Version 3 (`lintel_segment_layout`) Ein Segment ist eine Datei mit einem Layout, das die Bibliothek definiert. Ändert sich dieses Layout (in 0.2 zum Beispiel: `ShmSegmentAccessor` von einem Zeiger auf 40 Byte, @@ -29,10 +29,12 @@ folgende Reihenfolge (`SharedMemoryService.cpp`, `create()`): 1. **Migrator registriert** (`lintel::shm::SegmentLayoutPolicy::setMigrator`) → er entscheidet. `true` = Inhalte dürfen weg, `false` = Start abbrechen. -2. **Policy `Recreate`** (`set(LayoutMismatchPolicy::Recreate)`) oder +2. **Alle Objekte `Volatile`/`Derived`** (und ihre Header lesbar) → das Segment wird + ohne Operator-Eingriff neu angelegt. +3. **Policy `Recreate`** (`set(LayoutMismatchPolicy::Recreate)`) oder `LINTEL_SHM_RECREATE=1` → Inhalt wird verworfen. -3. Sonst **`Fail`** (Default): `std::runtime_error` mit Segmentname und Dateipfad, die - Datei bleibt unangetastet. +4. Sonst **`Fail`** (Default): `std::runtime_error` mit Segmentname, Dateipfad und den + blockierenden Objekten; die Datei bleibt unangetastet. Der Migrator läuft, **bevor** irgendetwas gelöscht wird, und bekommt den Pfad der aktuellen Datei sowie die gefundene und die erwartete Layout-Version. @@ -69,21 +71,85 @@ Regeln für den Migrator: ## 4. Grenze der aktuellen Umsetzung -Die Entscheidung gilt **pro Segment, nicht pro Objekt**: die Objekte im Segment tragen -keine eigene Version, also kann der Migrator nur „alles retten" oder „alles aufgeben". -Für die typisierten Repositories ist das ausreichend, weil ihr Payload ohnehin über -`serialize()`/`deserialize()` der DTOs geht. +Die segmentweite Entscheidung bleibt der letzte Fallback. Ab Layout-Version 3 +trägt jedes benannte Objekt einen eigenen `ShmObjectHeader`; die Bibliothek kann +damit **pro Objekt** entscheiden statt nur pro Segment: + +- `Volatile` und `Derived` dürfen ohne Migrator neu angelegt werden. +- `Persistent` verlangt eine registrierte Migration oder einen Migrator. +- Ein Objekt ohne lesbaren Header (Segment aus einem älteren Build) gilt + weiterhin als `Persistent`, weil seine Persistenzklasse unbekannt ist. + +Der Header liegt am Anfang des benannten Objekts; die Nutzdaten selbst sind eine +separate Allokation, die über `payloadOffset` adressiert wird. Änderungen an der +internen Buchhaltung (Accessor, Header, Record) verschieben die Nutzdaten daher +nicht mehr. ## 5. Nächste Stufen (Backlog) -| Stufe | Inhalt | Nutzen | +| Stufe | Inhalt | Status | |-------|--------|--------| -| **2** | `ShmObjectHeader { magic, version, payloadBytes }` vor jedem benannten Objekt; pro Typ eine Migrationsfunktion `v(n) → v(n+1)` | echte Teil-Migration, ohne Export/Import | -| **3** | Persistenzklasse pro Objekt beim Registrieren: `Volatile` (darf weg), `Derived` (aus DB/XML rekonstruierbar), `Persistent` (muss migriert werden, sonst Startabbruch) | die Bibliothek weiß selbst, was sie verwerfen darf | -| **4** | stabile Payload-ABI: Nutzdaten als versionierte Blobs, interne Buchhaltung (Accessor, Header) versioniert getrennt | Layout-Änderungen der Bibliothek invalidieren Nutzdaten nie wieder | +| **2** | `ShmObjectHeader { magic, headerVersion, payloadVersion, payloadBytes, … }` vor jedem benannten Objekt; pro Typ eine Migrationsfunktion `v(n) → v(n+1)` | **umgesetzt** (Layout-Version 3) | +| **3** | Persistenzklasse pro Objekt beim Registrieren: `Volatile`, `Derived`, `Persistent` | **umgesetzt** | +| **4** | stabile Payload-ABI: Nutzdaten als versionierte, separat allokierte Blobs; interne Buchhaltung (Accessor, Header) versioniert getrennt | **umgesetzt** (Header v1, Payload v1) | + +Die Umsetzung ist in `docs/shm-payload-abi-plan.md` in Phasen und +Akzeptanzkriterien zerlegt. + +### 5.1 Objekt registrieren + +```cpp +#include + +lintel::shm::ShmObjectOptions options{ + .payloadVersion = 2, + .persistence = lintel::shm::ShmPersistence::Persistent, +}; + +lintel::shm::SharedMemoryRepository repository( + service, segment, "my-dto", options, MyDto{}); +``` + +Ohne explizite Angabe bleibt `payloadVersion = 1` und `Persistent` der Default. +Das ist die sichere Variante: ein bestehender Aufrufer verliert nie +stillschweigend Daten. Der Property-Spiegel registriert sich als `Derived`, +EventBus und MessageQueue als `Volatile`. + +### 5.2 Payload migrieren + +```cpp +lintel::shm::registerPayloadMigration(1, 2, + [](const void *oldBytes, std::size_t oldSize, void *newPayload) { + // alten Stand lesen und den neuen Typ an newPayload konstruieren + }); +``` + +Die Registry verkettet `1 → 2 → 3` automatisch. Der alte Payload bleibt +unangetastet, bis die Migration vollständig durchgelaufen ist; schlägt sie fehl, +öffnet sich das Objekt beim nächsten Versuch erneut mit dem alten Stand. + +### 5.3 Segmentweite Entscheidung + +`SharedMemoryService` prüft bei einem Layout-Konflikt zuerst den Migrator, dann +die Objekt-Header: + +1. Migrator registriert → er entscheidet. +2. Alle Objekte sind `Volatile`/`Derived` und lesbar → Segment wird neu angelegt. +3. Policy `Recreate` bzw. `LINTEL_SHM_RECREATE=1` → Inhalt wird verworfen. +4. Sonst → `std::runtime_error` mit Segmentname, Pfad und den blockierenden + Objekten; die Datei bleibt unangetastet. + +### 5.4 Grenzen -Stufe 2 ist die Voraussetzung für 3 und 4; ohne sie bleibt es beim Segment als -Ganzes. +- Eine Payload-Migration kann den Destruktor des alten Typs nicht aufrufen, weil + dessen Layout im neuen Build nicht mehr existiert. Verschachtelte Allokationen + des alten Stands werden erst beim nächsten Neuanlegen des Segments frei. +- Die Typ-ID ist ein FNV-1a-Hash des vollqualifizierten Typnamens. Bei einem + Compiler-/ABI-Wechsel kann sie sich ändern; dann ist `payloadTypeId` explizit + zu setzen. +- `Derived` heißt „die Anwendung kann den Inhalt rekonstruieren“. Die Bibliothek + ruft keinen Rebuild-Hook auf; das bleibt Aufgabe der besitzenden + Service-Schicht (z. B. `PropertyService`). ## 6. Betrieb diff --git a/docs/shm-payload-abi-plan.md b/docs/shm-payload-abi-plan.md new file mode 100644 index 00000000..b5cf20ca --- /dev/null +++ b/docs/shm-payload-abi-plan.md @@ -0,0 +1,130 @@ +# Shared-Memory Payload-ABI — Umsetzungsplan für Stufe 2–4 + +Stand: 2026-10-22 +Basis: `docs/shm-migration-strategy.md` §5 (Backlog Stufe 2, 3, 4) +Branch: aktueller Arbeitsstand + +Ziel: Der Inhalt eines Segments wird nicht mehr als Ganzes verworfen, nur weil +sich die Buchhaltung der Bibliothek ändert. Jedes benannte Objekt bekommt eine +eigene, versionierte Hülle; Nutzdaten und interne Struktur (Accessor, Header, +Record) werden getrennt versioniert. Die Bibliothek kann dadurch selbst +entscheiden, welche Objekte sie verwerfen darf. + +## Leitentscheidungen + +1. **Ein stabiler Header pro benanntem Objekt.** `ShmObjectHeader` ist ein + `trivially copyable` POD mit fester Größe (64 Byte) und steht am Anfang des + benannten Objekts. Seine Felder `headerVersion` und `payloadVersion` sind + getrennt: Änderungen an der Hülle invalidieren die Nutzdaten nicht. +2. **Nutzdaten liegen in einer separaten Allokation.** Der Header speichert + `payloadOffset`, `payloadBytes` und `payloadAlignment` relativ zu sich + selbst. Ein Layout-Wechsel der Bibliothek verschiebt damit nie die Nutzdaten. +3. **Persistenzklasse pro Objekt.** `Volatile`, `Derived`, `Persistent` werden + beim Registrieren mitgegeben und im Header gespeichert. `Persistent` ist der + sichere Default; nur `Volatile`/`Derived` dürfen ohne Migrator verworfen + werden. +4. **Migration pro Typ und Version.** `ShmPayloadMigrations` registriert + `v(n) → v(n+1)`-Schritte. Die Bibliothek verkettet sie beim Öffnen eines + Objekts, bevor der erste Zugriff auf die Nutzdaten erfolgt. +5. **Segmentweite Entscheidung bleibt der letzte Fallback.** Der Migrator aus + `SegmentLayoutPolicy` hat weiterhin Vorrang. Ohne Migrator darf ein Segment + automatisch neu angelegt werden, wenn **alle** enthaltenen Objekte + `Volatile` oder `Derived` sind und ihre Header lesbar sind. + +## Phasen und Tasks + +### Phase 1 — Hülle und Migrationskern (Stufe 2 + 4) + +| ID | Task | Dateien | Tests | Akzeptanzkriterien | +|----|------|---------|-------|---------------------| +| **ABI-1** | `ShmObjectHeader` + `ShmPersistence` + `ShmObjectOptions` | `src/include/lintel/core/shm/ShmObjectHeader.h` (neu) | `tests/features/ShmObjectMigrationTest.cpp` | 64 Byte, `trivially copyable`, `headerVersion`/`payloadVersion` getrennt, Magic- und State-Prüfung. | +| **ABI-2** | Typ-ID und Migrationsregistry `v(n) → v(n+1)` | `ShmObjectHeader.h`, `src/features/base/services/ShmObject.cpp` (neu) | `ShmObjectMigrationTest.cpp` | Typ-ID stabil innerhalb eines Builds, Registrierung pro Typ/Version, Verkettung mehrerer Schritte, `reset()` für Tests. | +| **ABI-3** | `ShmObject` als typ-erased Hülle (bisher `ShmCellRecord`) | `src/include/lintel/core/shm/ShmObject.h` (neu), `src/features/base/services/ShmObject.cpp` | `ShmObjectMigrationTest.cpp`, `SharedMemoryRepositoryTest.cpp` | Header + separate Payload-Allokation, `resolve()` migriert oder legt neu an, `isStale()`/`generation()` bleiben. | +| **ABI-4** | `ShmCell`/`shm::SharedMemoryRepository` auf `ShmObject` umstellen | `src/include/lintel/core/shm/SharedMemoryRepository.h` | `SharedMemoryRepositoryTest.cpp` | Konstruktor und `of()` akzeptieren `ShmObjectOptions`; `header()`, `persistence()`, `payloadVersion()` verfügbar. Bestehende Aufrufe bleiben quellkompatibel. | +| **ABI-5** | Segment-Layout auf 3 anheben | `src/features/base/services/SharedMemoryService.cpp` | `SharedMemoryServiceTest.cpp` | Marker wird auf 3 gestempelt; Tests prüfen 3 statt 2. | + +**Phase-1-DoD:** Ein Objekt mit alter `payloadVersion` wird über eine +registrierte Kette migriert; ohne Migration und mit `Persistent` bricht der +Start ab. Nutzdaten liegen physisch getrennt vom Header. + +### Phase 2 — Persistenzklassen und segmentweite Entscheidung (Stufe 3) + +| ID | Task | Dateien | Tests | Akzeptanzkriterien | +|----|------|---------|-------|---------------------| +| **ABI-6** | Persistenzklasse in alle Registrierungspfade | `ShmObjectOptions`, `ShmConstructors.h`, `features/base/repositories/SharedMemoryRepository.h`, `SharedMemoryPropertyRepository.h` | `SharedMemoryRepositoryTest.cpp`, `SharedMemoryServiceTest.cpp` | Stack A und B können `Volatile`/`Derived`/`Persistent` setzen; Property-Spiegel = `Derived`; Default = `Persistent`. | +| **ABI-7** | Segment-Inspektion vor dem Verwerfen | `SharedMemoryService.cpp`, `ShmObjectHeader.h` | `SharedMemoryServiceTest.cpp` | Iteration über benannte Objekte, Header lesen, `allDiscardable` nur wenn alle Header gültig und nicht `Persistent`. | +| **ABI-8** | Auto-Recreate für rein flüchtige/abgeleitete Segmente | `SharedMemoryService.cpp` | `SharedMemoryServiceTest.cpp` | Alt-Segment mit nur `Volatile`/`Derived` wird ohne Env/Migrator neu angelegt; ein `Persistent`-Objekt verhindert das und liefert Pfad + Anleitung. | +| **ABI-9** | Objekt-Migration bei Versionskonflikt | `ShmObject.cpp` | `ShmObjectMigrationTest.cpp` | `Persistent` ohne Migration → `runtime_error`; `Volatile`/`Derived` ohne Migration → Objekt neu; mit Migration → Daten übernommen. | + +**Phase-2-DoD:** Die Bibliothek unterscheidet selbst zwischen „darf weg" und +„muss migriert werden"; der Operator-Eingriff ist nur noch bei `Persistent` +nötig. + +### Phase 3 — Bestehende Objekttypen integrieren (Stufe 2/3 flächendeckend) + +| ID | Task | Dateien | Tests | Akzeptanzkriterien | +|----|------|---------|-------|---------------------| +| **ABI-10** | `ShmConstructors`-Helfer auf Header umstellen | `ShmConstructors.h` | `SharedMemoryRepositoryTest.cpp`, `ShmViewTest.cpp`, `SharedMemoryCleanupServiceTest.cpp` | `constructArray/Map/Vector/Object/ObjectWith` liefern weiter `T&`, legen aber Header + Payload an; `constructString` bleibt Wert-Allokation. | +| **ABI-11** | EventBus als `Volatile` registrieren | `EventBusService.cpp`, `SharedMemoryBootstrapPlugin.cpp` | `EventBusServiceTest.cpp`, `SharedMemoryBootstrapPluginTest.cpp` | Bus-Objekt hat Header; Bootstrap liest die Payload über den Header; alte Segmente werden nicht als gültiger Bus interpretiert. | +| **ABI-12** | MessageQueueShmCore als `Volatile` registrieren | `ShmSegmentAccessor.h`, `MessageQueueShmCore.cpp` | `MessageQueueShmCoreTest.cpp` | Queue-Header liegt in einer versionierten Hülle; ein Versionskonflikt verwirft die flüchtige Queue, ohne das Segment zu löschen. | +| **ABI-13** | `ShmView` und Cleanup header-aware machen | `ShmView.cpp`, `SharedMemoryCleanupService.cpp` | `ShmViewTest.cpp`, `SharedMemoryCleanupServiceTest.cpp` | Views zeigen auf die Payload, nicht auf den Header; Cleanup gibt Header und Payload frei, ohne fremde Objekte zu treffen. | + +**Phase-3-DoD:** Jedes benannte Objekt im Segment trägt einen Header; die +bestehenden APIs bleiben kompatibel. + +### Phase 4 — Dokumentation und Abnahme (Stufe 4) + +| ID | Task | Dateien | Tests | Akzeptanzkriterien | +|----|------|---------|-------|---------------------| +| **ABI-14** | Migrationsstrategie fortschreiben | `docs/shm-migration-strategy.md` | — | Stufe 2–4 als umgesetzt beschrieben, neue API und Grenzen dokumentiert. | +| **ABI-15** | Changelog | `CHANGELOG.md` | — | `Added`/`Changed`-Einträge mit Verweis auf Plan und Strategie. | +| **ABI-16** | Abnahme | Build + CTest | alle SHM-Tests | Release-Build, `ctest -L unit`, Format- und Namespace-Check grün. | + +## API-Skizze + +```cpp +#include + +lintel::shm::ShmObjectOptions options{ + .payloadVersion = 2, + .persistence = lintel::shm::ShmPersistence::Persistent, +}; + +lintel::shm::registerPayloadMigration(1, 2, + [](const void *oldBytes, std::size_t oldSize, void *newPayload) { + // alten Stand lesen, neuen Stand an newPayload konstruieren + }); + +lintel::shm::SharedMemoryRepository repository( + service, segment, "my-dto", options, MyDto{}); +``` + +## Risiken und Grenzen + +- **Kein Destruktor für alte Nutzdaten.** Eine Migration liest die alten Bytes + und konstruiert den neuen Stand; verschachtelte Allokationen des alten Typs + werden nicht freigegeben. Das ist bei einem Layout-Wechsel unvermeidbar, weil + der alte Typ im neuen Build nicht mehr existiert. Der Speicher wird erst beim + nächsten Neuanlegen des Segments frei. +- **Header-Änderungen.** Der Header ist ab Version 1 eingefroren. Neue Felder + dürfen nur in `reserved` wandern; eine echte Header-Migration ist ein eigener + Schritt und muss `headerVersion` erhöhen. +- **Typ-ID.** Die ID wird aus dem vollqualifizierten Typnamen gehasht. Bei einem + Compiler-/ABI-Wechsel kann sie sich ändern; dann ist eine explizite ID in + `ShmObjectOptions` zu setzen. +- **`Derived`** bedeutet „die Anwendung kann den Inhalt rekonstruieren". Die + Bibliothek ruft keinen Rebuild-Hook auf; das bleibt Aufgabe der besitzenden + Service-Schicht (z. B. `PropertyService`). +- **`Persistent`** ist der Default. Bestehende Aufrufer ohne explizite Angabe + verlieren also nie stillschweigend Daten. + +## Fortschritt + +| Phase | Status | +|-------|--------| +| 1 — Hülle und Migrationskern | ☑ | +| 2 — Persistenzklassen | ☑ | +| 3 — Integration bestehender Typen | ☑ | +| 4 — Doku und Abnahme | ☑ | + +Letztes Update: 2026-10-22 diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 4e5f6403..09771d73 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -221,6 +221,8 @@ set(headers include/lintel/core/shm/ShmSegmentAccessor.h include/lintel/core/shm/SegmentLayoutPolicy.h include/lintel/core/shm/ShmView.h + include/lintel/core/shm/ShmObject.h + include/lintel/core/shm/ShmObjectHeader.h include/lintel/core/shm/SharedMemoryRepository.h include/lintel/core/shm/SharedMemoryRepositoryRegistry.h include/lintel/core/shm/containers/ArrayShm.h @@ -397,7 +399,7 @@ set(sources core/services/ReadinessRegistry.cpp features/base/services/ProcessService.cpp features/base/services/SharedMemoryService.cpp - features/base/services/SharedMemoryRepository.cpp + features/base/services/ShmObject.cpp features/base/services/SharedMemoryRepositoryRegistry.cpp core/services/StopWatchService.cpp # core - utils diff --git a/src/app/plugins/SharedMemoryBootstrapPlugin.cpp b/src/app/plugins/SharedMemoryBootstrapPlugin.cpp index ee61fb8b..1d96a9aa 100644 --- a/src/app/plugins/SharedMemoryBootstrapPlugin.cpp +++ b/src/app/plugins/SharedMemoryBootstrapPlugin.cpp @@ -1,5 +1,8 @@ #include "lintel/app/plugins/SharedMemoryBootstrapPlugin.h" +#include +#include +#include #include #include #include @@ -17,6 +20,7 @@ #include "lintel/core/persistence/Statement.h" #include "lintel/core/persistence/Transaction.h" #include "lintel/core/services/LoggerService.h" +#include "lintel/core/shm/ShmObjectHeader.h" #include "lintel/features/base/configuration/EventBusComponent.h" namespace lintel { @@ -110,9 +114,24 @@ void SharedMemoryBootstrapPlugin::validateEventBuses() // boost::interprocess takes a narrow file name; path::c_str() is wchar_t on Windows. const std::string filePath = path.string(); boost::interprocess::managed_mapped_file mapping(boost::interprocess::open_only, filePath.c_str()); - const auto bus = mapping.find(eventBusEntry->getName().c_str()); - if (bus.first == nullptr) { continue; } - const EventBusConfig persisted = bus.first->getConfig(); + // The named object is a shm::ShmObjectHeader; the bus itself is a separate + // allocation the header points at. An object without such a header was + // written by a build that predates the header, so it is skipped here and its + // bytes must never be interpreted as an EventBus - SharedMemoryService + // decides whether the segment may be recreated. + const auto named = mapping.find(eventBusEntry->getName().c_str()); + if (named.first == nullptr || named.second < sizeof(shm::ShmObjectHeader)) { continue; } + // Read the header as bytes first: a pre-header object is not guaranteed to + // be aligned like a ShmObjectHeader, so it is only reinterpreted after the + // magic proved that boost created it as one. + shm::ShmObjectHeader header{}; + std::memcpy(&header, named.first, sizeof(header)); + if (!header.isCompatible() || !header.isReady()) { continue; } + if (header.payloadOffset == 0 || header.payloadBytes < sizeof(EventBus)) { continue; } + const std::uintptr_t payloadAddress = + reinterpret_cast(named.first) + static_cast(header.payloadOffset); + const auto *bus = std::bit_cast(payloadAddress); + const EventBusConfig persisted = bus->getConfig(); if (persisted.busCapacity != config.busCapacity || persisted.subscriberCapacity != config.subscriberCapacity || persisted.maxTopics != config.maxTopics || persisted.maxSubscribers != config.maxSubscribers) { const std::string message = diff --git a/src/core/shm/ShmView.cpp b/src/core/shm/ShmView.cpp index 9936744c..0b6db697 100644 --- a/src/core/shm/ShmView.cpp +++ b/src/core/shm/ShmView.cpp @@ -1,5 +1,10 @@ #include "lintel/core/shm/ShmView.h" +#include +#include +#include + +#include "lintel/core/shm/ShmObjectHeader.h" #include "lintel/features/base/services/ShmConstructors.h" namespace lintel { @@ -14,7 +19,24 @@ namespace shm { SegmentSemaphoreGuard gate(handle); // Look the object up as a byte range: the view factory does not need the // element type to find the address, only to interpret the bytes afterwards. - return handle.m_managedMappedFile->find(name.c_str()).first; + const auto named = handle.m_managedMappedFile->find(name.c_str()); + if (named.first == nullptr || named.second < sizeof(ShmObjectHeader)) { + // Missing, or too small to carry a header: a pre-header object. Its bytes + // are not reinterpreted - viewOf() reports the object as absent. + return nullptr; + } + // Read the header as bytes first: a pre-header object is not guaranteed to + // be aligned like a ShmObjectHeader. Only after the magic proved that boost + // created the block as one does the address become a header. + ShmObjectHeader header{}; + std::memcpy(&header, named.first, sizeof(header)); + if (!header.isCompatible() || !header.isReady()) { return nullptr; } + if (header.payloadOffset == 0) { return nullptr; } + // The payload is a separate allocation, addressed relative to the header so + // that the offset survives every mapping of the segment. + const std::uintptr_t payloadAddress = + reinterpret_cast(named.first) + static_cast(header.payloadOffset); + return std::bit_cast(payloadAddress); } std::size_t generationOf(const std::weak_ptr &service) noexcept diff --git a/src/features/base/msg/MessageQueueShmCore.cpp b/src/features/base/msg/MessageQueueShmCore.cpp index 8f9ac544..68a05407 100644 --- a/src/features/base/msg/MessageQueueShmCore.cpp +++ b/src/features/base/msg/MessageQueueShmCore.cpp @@ -16,6 +16,8 @@ #include #include +#include "lintel/core/shm/ShmObjectHeader.h" + namespace lintel { namespace mqshm_detail { @@ -42,6 +44,15 @@ namespace mqshm_detail { constexpr std::uint64_t kQueueMagicInitialising = 1; /** Upper bound for waiting until another process finished that construction. */ constexpr std::chrono::milliseconds kInitWaitTimeout{ 5000 }; + /** Payload version of the queue header wrapper; bumped when QueueHeader or the + * ring layout stored in the payload changes. */ + constexpr std::uint16_t kQueuePayloadVersion = 1; + /** Type id of the queue payload: a fixed name, so the id does not depend on + * the (possibly local) full type name of QueueHeader. */ + constexpr std::uint32_t kQueuePayloadTypeId = shm::fnv1a32("lintel::MessageQueueShmCore::QueueHeader"); + /** The payload must be aligned for the ring; the segment allocator aligns to + * max_align_t, which the static_assert below checks. */ + constexpr std::size_t kQueuePayloadAlignment = alignof(std::max_align_t); [[nodiscard]] std::size_t alignUp(std::size_t value, std::size_t alignment) { @@ -274,21 +285,26 @@ namespace mqshm_detail { [[nodiscard]] static std::byte *payloadOf(SlotHeader *slot) { return reinterpret_cast(slot + 1); } }; - // The block comes from find_or_construct, whose allocation is aligned - // to the segment's memory alignment (max_align_t), and the ring is placed at - // a multiple of its own alignment behind the header - which only works while - // that alignment does not exceed the block alignment. + // The payload block is allocated by ShmSegmentAccessor::findOrConstructPayload(), + // which aligns it to max_align_t, and the ring is placed at a multiple of its + // own alignment behind the header - which only works while that alignment does + // not exceed the block alignment. static_assert(alignof(ShmRing) <= alignof(std::max_align_t), - "the named queue block must be aligned enough for the ring"); + "the named queue payload must be aligned enough for the ring"); /** * Cross-process anchor of one queue inside the segment. * - * The header is a named object of the segment ("lintel_mq_" + queue name) and - * is what a second process finds first: it describes where the ring sits and - * which layout it has. Everything in here is either a scalar or a byte - * offset, never a pointer - the segment is mapped at a different virtual - * address in every process. + * The queue lives in the versioned payload of the named object + * ("lintel_mq_" + queue name): the named object itself is a ShmObjectHeader, + * the payload behind it is this header followed by the ring. The wrapper + * records the payload version and the persistence class (Volatile), so a + * build with a different queue layout discards the queue instead of failing + * the whole segment. + * + * This header describes where the ring sits and which layout it has. + * Everything in here is either a scalar or a byte offset, never a pointer - + * the segment is mapped at a different virtual address in every process. * * The header is never constructed as a C++ object: the allocator hands out a * zero initialised block (magic == 0), which is exactly the "not @@ -455,13 +471,21 @@ namespace mqshm_detail { // The header is the cross-process anchor of the queue: it lives in the // segment under a name derived from the queue name, so a second process - // finds the very same ring instead of creating one of its own. The ring - // and its slot array follow the header inside the same segment, which is - // why only byte offsets are stored - a pointer would be process local. + // finds the very same ring instead of creating one of its own. The named + // object is a stable ShmObjectHeader, the queue header and the ring follow + // it in the payload allocation - which is why only byte offsets are stored, + // a pointer would be process local. const std::size_t ringOffset = alignUp(sizeof(QueueHeader), alignof(ShmRing)); const std::size_t blockSize = ringOffset + sizeof(ShmRing); const std::string headerName = headerNameOf(name); - void *block = segment.findOrConstructArray(headerName.c_str(), blockSize); + // A queue is ephemeral state: when this build cannot use the payload of the + // previous one, the queue is recreated zeroed instead of failing the + // segment (ShmPersistence::Volatile). + void *block = segment.findOrConstructPayload(headerName.c_str(), + blockSize, + kQueuePayloadAlignment, + kQueuePayloadTypeId, + shm::ShmObjectOptions{ .payloadVersion = kQueuePayloadVersion, .persistence = shm::ShmPersistence::Volatile }); if (block == nullptr) { throw std::runtime_error("cannot create the queue header '" + headerName + "' in the shared memory segment: the allocator does not support named objects"); diff --git a/src/features/base/services/EventBusService.cpp b/src/features/base/services/EventBusService.cpp index 0d6d8977..55c148a8 100644 --- a/src/features/base/services/EventBusService.cpp +++ b/src/features/base/services/EventBusService.cpp @@ -3,6 +3,7 @@ #include #include "lintel/core/services/LoggerService.h" +#include "lintel/core/shm/ShmObjectHeader.h" #include "lintel/core/shm/ShmSegmentAccessor.h" #include "lintel/features/base/configuration/EventBusComponent.h" #include "lintel/features/base/services/ShmConstructors.h" @@ -66,8 +67,17 @@ EventBus &EventBusService::resolve(const std::shared_ptr &entry) required); } const ShmSegmentAccessor accessor = m_sharedMemoryService->getSegmentAccessor(segment); - EventBus &bus = shm::constructObjectWith( - *m_sharedMemoryService, segment, entry->getName(), entry->getName(), config, accessor); + // The bus is transient state, and its address inside the segment is not + // stable across builds: the object gets a header that carries the payload + // version, and an incompatible payload may be recreated without asking the + // operator (ShmPersistence::Volatile). + EventBus &bus = shm::constructObjectWith(*m_sharedMemoryService, + segment, + entry->getName(), + shm::ShmObjectOptions{ .payloadVersion = 1, .persistence = shm::ShmPersistence::Volatile }, + entry->getName(), + config, + accessor); m_buses[entry->getName()] = &bus; m_generations[entry->getName()] = generation; return bus; diff --git a/src/features/base/services/SharedMemoryCleanupService.cpp b/src/features/base/services/SharedMemoryCleanupService.cpp index 06274824..4e353727 100644 --- a/src/features/base/services/SharedMemoryCleanupService.cpp +++ b/src/features/base/services/SharedMemoryCleanupService.cpp @@ -1,11 +1,15 @@ #include "lintel/features/base/services/SharedMemoryCleanupService.h" +#include +#include +#include #include #include #include "lintel/core/persistence/Connection.h" #include "lintel/core/persistence/ParameterBuilder.h" #include "lintel/core/persistence/Statement.h" +#include "lintel/core/shm/ShmObjectHeader.h" #include "lintel/features/base/services/SchedulerService.h" #include "lintel/features/base/services/SharedMemoryService.h" #include "lintel/features/base/services/ShmConstructors.h" @@ -21,13 +25,36 @@ namespace shm { std::lock_guard lock(service.segmentsMutexOf()); auto &handle = service.segmentOf(segment->getName()); SegmentSemaphoreGuard gate(handle); - // The byte-wise lookup works for every object size. destroy_ptr() on the - // resulting char pointer is safe because trackObject() restricts the - // type-erased removal to trivially destructible types, for which running the - // destructor is a no-op; the block itself is released by the segment manager. - auto found = handle.m_managedMappedFile->find(name.c_str()); - if (found.first == nullptr) { return true; }// already gone: idempotent - handle.m_managedMappedFile->destroy_ptr(found.first); + // The byte-wise lookup works for every object size. An object with a + // compatible header keeps its payload in a separate allocation: release it, + // then destroy the named header. trackObject() restricts the tracking to + // trivially destructible types, for which running the payload destructor is + // a no-op, so none is run here. + const auto named = handle.m_managedMappedFile->find(name.c_str()); + if (named.first == nullptr) { return true; }// already gone: idempotent + if (named.second >= sizeof(shm::ShmObjectHeader)) { + // Read the header as bytes first: a pre-header object is not guaranteed to + // be aligned like a ShmObjectHeader. Only after the magic proved that the + // block is a header does the address become one. + shm::ShmObjectHeader probe{}; + std::memcpy(&probe, named.first, sizeof(probe)); + if (probe.isCompatible()) { + auto *header = std::bit_cast(named.first); + if (header->isReady() && header->payloadOffset != 0 && header->payloadBytes != 0) { + void *payload = shm::payloadAddressOf(header); + // Stop readers before the payload disappears. + header->state = shm::ShmObjectHeader::kStateConstructing; + handle.m_managedMappedFile->get_segment_manager()->deallocate(payload); + } + // The header is trivially destructible, so destroying it runs no payload + // destructor; the segment manager releases the block and the name. + handle.m_managedMappedFile->destroy(name.c_str()); + return true; + } + } + // Pre-header object: destroy_ptr() on the char pointer keeps working for any + // size, and the segment manager releases the block itself. + handle.m_managedMappedFile->destroy_ptr(named.first); return true; } diff --git a/src/features/base/services/SharedMemoryRepository.cpp b/src/features/base/services/SharedMemoryRepository.cpp deleted file mode 100644 index 8dc72370..00000000 --- a/src/features/base/services/SharedMemoryRepository.cpp +++ /dev/null @@ -1,211 +0,0 @@ -#include "lintel/core/shm/SharedMemoryRepository.h" - -#include "lintel/core/models/SharedMemorySegment.h" -#include "lintel/core/shm/BoostSegmentAllocator.h" -#include "lintel/core/shm/ShmSegmentAccessor.h" -#include "lintel/features/base/services/SharedMemoryService.h" -#include "lintel/features/base/services/ShmConstructors.h" - -#include -#include -#include -#include -#include -#include -#include -#include -#include - -namespace lintel { -namespace { - - /** - * Fixed size bookkeeping record of one cell inside a shared memory segment. - * - * The record is what find_or_construct locates by name; the payload itself - * is a separate allocation of the segment and is referred to by a byte - * offset relative to the record. Both live in the same segment file, so - * both shift by the same amount in every mapping and the relative layout is - * identical in every process (see ShmCell's class documentation). - * - * The record deliberately does not store the ShmSegmentAccessor it was - * created with: the accessor points into the creating process' heap - * (BoostSegmentAllocator owned by that process' SharedMemoryService) and - * would be meaningless everywhere else. It is only needed while the payload - * is allocated, which happens inside this constructor. - * - * The magic guards against a name collision with an unrelated object that - * some other code placed in the segment under the same name: without it, - * the payload offset of that object would be reinterpreted as cell data. - */ - class ShmCellRecord - { - public: - /** Marks a cell record; "LINTELSH" in ASCII. */ - static constexpr std::uint64_t kMagic = 0x4c494e54454c5348ull; - - /** Allocate the payload from the segment and run the payload constructor. - * @param segment accessor of the segment the payload is allocated from - * @param layout payload size (> 0) and alignment (power of two) - * @param init payload constructor, run on the allocated memory - * @param initArgument constructor arguments for init - * @throws std::bad_alloc when the segment cannot hold the payload - * @throws anything init throws (the payload allocation is released first) */ - ShmCellRecord(ShmSegmentAccessor segment, - ShmCell::Layout layout, - ShmCell::InitFunction init, - ShmCell::InitArgument initArgument) - { - const std::size_t bytes = layout.size + layout.alignment - 1u; - void *raw = segment.allocate(bytes); - if (raw == nullptr) { throw std::bad_alloc(); } - void *aligned = raw; - std::size_t space = bytes; - if (std::align(layout.alignment, layout.size, aligned, space) == nullptr) { - segment.deallocate(raw); - throw std::bad_alloc(); - } - // The record is already placed at its final address when this - // constructor runs, so the offset is stable for the lifetime of the - // cell - and identical in every other mapping of the segment. - m_payloadOffset = reinterpret_cast(aligned) - reinterpret_cast(this); - m_payloadSize = layout.size; - m_alignment = layout.alignment; - try { - init(aligned, initArgument); - } catch (...) { - segment.deallocate(raw); - m_payloadOffset = 0; - throw; - } - } - - ShmCellRecord(const ShmCellRecord &) = delete; - ShmCellRecord &operator=(const ShmCellRecord &) = delete; - - /** @return pointer to the payload in this process' mapping */ - [[nodiscard]] void *payload() const - { - // The payload offset is stored in the record inside the segment (its - // address differs per mapping), so it has to be materialised here. The - // payload alignment is not a multiple of sizeof(ShmCellRecord), so - // element arithmetic is not an option; the address is therefore - // materialised in the integer domain. std::bit_cast restores it: a plain - // reinterpret_cast from the integer would be reported as - // performance-no-int-to-ptr, byte pointer arithmetic as - // cpp/suspicious-pointer-scaling. - const std::uintptr_t address = - reinterpret_cast(this) + static_cast(m_payloadOffset); - return std::bit_cast(address); - } - - [[nodiscard]] std::uint64_t magic() const { return m_magic; } - [[nodiscard]] std::size_t payloadSize() const { return m_payloadSize; } - [[nodiscard]] std::size_t alignment() const { return m_alignment; } - - private: - std::uint64_t m_magic = kMagic; - std::intptr_t m_payloadOffset = 0; - std::size_t m_payloadSize = 0; - std::size_t m_alignment = 0; - }; - - /** - * Locate the named cell record, constructing it (and its payload) on first use. - * - * Everything runs under the segment's construct lock: the process wide mutex - * serializes the lookup against a concurrent remap of this process, the - * cross-process semaphore serializes it against the remap / construct - * critical sections of every other process. That is what makes the single - * construction of the cell - record and payload - safe. - * - * @param service service owning the segment - * @param segment segment the cell lives in - * @param name name of the cell inside the segment - * @param layout payload size and alignment - * @param init payload constructor, run when the cell is created - * @param initArgument constructor arguments for init - * @return the record, valid until the segment is remapped - * @throws ShmSegmentNotFound when the service does not know the segment - * @throws std::invalid_argument when the name already holds an incompatible object - */ - ShmCellRecord *findRecordOf(SharedMemoryService &service, - const std::shared_ptr &segment, - const std::string &name, - ShmCell::Layout layout, - ShmCell::InitFunction init, - ShmCell::InitArgument initArgument) - { - std::lock_guard lock(service.segmentsMutexOf()); - auto &handle = service.segmentOf(segment->getName()); - shm::SegmentSemaphoreGuard gate(handle); - // Same lazy allocator creation as SharedMemoryService::getSegmentAccessor, - // but without calling it: that would deadlock on the mutex held here. - if (handle.m_segmentAllocator == nullptr) { - handle.m_segmentAllocator = - std::make_shared(*handle.m_managedMappedFile->get_segment_manager()); - } - const ShmSegmentAccessor accessor(*handle.m_segmentAllocator); - auto *record = - handle.m_managedMappedFile->find_or_construct(name.c_str())(accessor, layout, init, initArgument); - // Only reached with a freshly built record trivially, but also - after a - // restart or from a second process - with one that was found again: the - // constructor arguments of the finder are ignored there, so the layout - // has to be compared instead of trusted. - if (record->magic() != ShmCellRecord::kMagic || record->payloadSize() != layout.size - || record->alignment() != layout.alignment) { - throw std::invalid_argument("shared memory cell '" + name + "' of segment '" + segment->getName() - + "' already exists with an incompatible payload layout"); - } - return record; - } - -}// namespace - -ShmCell::ShmCell(std::shared_ptr service, - std::shared_ptr segment, - std::string name, - Layout layout, - InitFunction init, - std::shared_ptr initArgument) - : m_service(std::move(service)), m_segment(std::move(segment)), m_name(std::move(name)), m_payloadSize(layout.size), - m_alignment(layout.alignment), m_init(init), m_initArgument(std::move(initArgument)) -{ - if (m_service == nullptr) { throw std::invalid_argument("ShmCell needs a shared memory service"); } - if (m_segment == nullptr) { throw std::invalid_argument("ShmCell '" + m_name + "' needs a shared memory segment"); } - if (m_name.empty()) { throw std::invalid_argument("ShmCell name must not be empty"); } - if (m_payloadSize == 0) { throw std::invalid_argument("ShmCell '" + m_name + "' payload must not be empty"); } - if (m_init == nullptr) { throw std::invalid_argument("ShmCell '" + m_name + "' needs a payload constructor"); } - if (m_initArgument == nullptr) { - throw std::invalid_argument("ShmCell '" + m_name + "' needs constructor arguments"); - } - if (m_alignment == 0 || m_alignment > kMaxAlignment || (m_alignment & (m_alignment - 1u)) != 0) { - throw std::invalid_argument("ShmCell '" + m_name + "' alignment must be a power of two of at most " - + std::to_string(kMaxAlignment) + " bytes"); - } -} - -void *ShmCell::resolve() const -{ - // Generation read *before* the lookup and stored afterwards: a remap that - // slips in between then shows up as a stale binding on the next access, - // which costs one extra lookup - the other order would cache a pointer of - // the old mapping as up to date, which is the dangerous direction. - const std::size_t observed = m_service->getGeneration(); - if (m_record != nullptr && observed == m_generation) { return static_cast(m_record)->payload(); } - auto *record = findRecordOf(*m_service, - m_segment, - m_name, - Layout{ m_payloadSize, m_alignment }, - m_init, - ShmCell::InitArgument{ m_initArgument.get() }); - m_record = record; - m_generation = observed; - return record->payload(); -} - -bool ShmCell::isStale() const { return m_record == nullptr || m_service->getGeneration() != m_generation; } - -std::size_t ShmCell::generation() const { return m_generation; } - -}// namespace lintel diff --git a/src/features/base/services/SharedMemoryService.cpp b/src/features/base/services/SharedMemoryService.cpp index be3f92d8..6d5de656 100644 --- a/src/features/base/services/SharedMemoryService.cpp +++ b/src/features/base/services/SharedMemoryService.cpp @@ -6,8 +6,12 @@ #include #include #include +#include #include +#include +#include + namespace lintel { namespace { /** Layout marker of the objects stored inside a segment. @@ -16,10 +20,70 @@ namespace { * ShmSegmentAccessor grew from one pointer to a named 40 byte value in 0.2, * for example). A segment written by an older build is recreated instead of * being reinterpreted with the new layout, which would corrupt it silently. + * + * Version 3 introduced the per-object ShmObjectHeader. Objects written by + * earlier builds have no readable header and are therefore still treated as + * authoritative unless the operator opts in. */ - constexpr std::uint64_t kSegmentLayoutVersion = 2; + constexpr std::uint64_t kSegmentLayoutVersion = 3; constexpr char kSegmentLayoutName[] = "lintel_segment_layout"; + /** What a stale segment contains, as far as the stable object headers say. */ + struct SegmentContent + { + /** number of named objects besides the layout marker */ + std::size_t objects = 0; + /** true when every object has a readable header and is Volatile/Derived */ + bool allDiscardable = true; + /** names that prevent an automatic discard, for the error message */ + std::vector blocking; + }; + + /** Inspect the named objects of a mapped segment. + * + * Only the stable ShmObjectHeader is read; the payload types are irrelevant + * for the decision. An object without a compatible header (a segment written + * before version 3) is unknown and therefore blocks an automatic discard. + * + * @param file mapped segment + * @return summary used for the persistence-class decision */ + SegmentContent inspectContent(boost::interprocess::managed_mapped_file &file) + { + SegmentContent content; + auto *manager = file.get_segment_manager(); + for (auto iterator = manager->named_begin(); iterator != manager->named_end(); ++iterator) { + const std::string name(iterator->name()); + if (name == kSegmentLayoutName) { continue; } + ++content.objects; + // The named index does not hand out the allocation size, so ask the + // segment manager: a pre-header object can be smaller than the header and + // must not be read past its end. + const auto *bytes = static_cast(iterator->value()); + const auto length = manager->get_instance_length(bytes); + if (bytes == nullptr || length < sizeof(shm::ShmObjectHeader)) { + content.allDiscardable = false; + content.blocking.push_back(name + " (no object header)"); + continue; + } + const auto *header = static_cast(iterator->value()); + if (header == nullptr) { + content.allDiscardable = false; + content.blocking.push_back(name + " (no object header)"); + continue; + } + if (!header->isCompatible() || !header->isReady()) { + content.allDiscardable = false; + content.blocking.push_back(name + " (unreadable object header)"); + continue; + } + if (header->persistenceClass() == shm::ShmPersistence::Persistent) { + content.allDiscardable = false; + content.blocking.push_back(name + " (persistent)"); + } + } + return content; + } + /** @param file mapped segment * @return the layout version stored in the segment; empty for a segment that * was never used, 0 for one written before the marker existed */ @@ -176,13 +240,15 @@ std::map> SharedMemorySe boost::interprocess::open_or_create, filePath.c_str(), segment->getSize()); const std::optional storedVersion = layoutVersionOf(*handle->m_managedMappedFile); if (storedVersion.has_value() && *storedVersion != kSegmentLayoutVersion) { - // The objects in a segment have no layout version of their own, so - // their bytes cannot be reinterpreted safely - but only the operator - // knows whether the contents can be recomputed. So the operator - // decides: an explicit migrator, an explicit policy, or - // LINTEL_SHM_RECREATE=1. Without either of them nothing is discarded. - // See docs/shm-migration-strategy.md. + // The layout marker covers the library's bookkeeping. On top of that, + // every object carries its own persistence class. A segment whose + // objects are all Volatile or Derived may be recreated without asking + // the operator; a Persistent object (or one without a readable header, + // i.e. written before object headers existed) keeps the old fail-safe + // behaviour. + // See docs/shm-payload-abi-plan.md. bool mayDiscard = false; + const SegmentContent content = inspectContent(*handle->m_managedMappedFile); if (shm::SegmentLayoutPolicy::hasMigrator()) { LOG_WARN( "shared memory segment '{}' was written with layout {} (current {}) - asking the registered " @@ -196,19 +262,31 @@ std::map> SharedMemorySe throw std::runtime_error("the migrator refused to hand over shared memory segment '" + segment->getName() + "' at '" + filePath + "'"); } + } else if (*storedVersion < kSegmentLayoutVersion && content.allDiscardable) { + mayDiscard = true; + LOG_WARN("recreating shared memory segment '{}' at '{}': all {} object(s) are volatile or derived", + segment->getName(), + filePath, + content.objects); } else if (shm::SegmentLayoutPolicy::policy() == shm::LayoutMismatchPolicy::Recreate || recreateAllowed()) { mayDiscard = true; } if (!mayDiscard) { + std::string blocking; + for (const std::string &name : content.blocking) { + if (!blocking.empty()) { blocking.append(", "); } + blocking.append(name); + } throw std::runtime_error("shared memory segment '" + segment->getName() + "' at '" + filePath - + "' was written with an older object layout and cannot be opened. Delete the " - "file to start with an empty segment, register a migrator " + + "' was written with an older object layout and cannot be opened" + + (blocking.empty() ? std::string() : " (blocking objects: " + blocking + ")") + + ". Delete the file to start with an empty segment, register a migrator " "(lintel::shm::SegmentLayoutPolicy) or set LINTEL_SHM_RECREATE=1 to have it " "discarded automatically."); } LOG_WARN( - "shared memory segment '{}' was written with an older object layout - recreating '{}' because " - "LINTEL_SHM_RECREATE is set (its contents are discarded, not migrated)", + "shared memory segment '{}' was written with an older object layout - recreating '{}' " + "(its contents are discarded, not migrated)", segment->getName(), filePath); handle->m_managedMappedFile.reset(); diff --git a/src/features/base/services/ShmObject.cpp b/src/features/base/services/ShmObject.cpp new file mode 100644 index 00000000..a000be95 --- /dev/null +++ b/src/features/base/services/ShmObject.cpp @@ -0,0 +1,434 @@ +#include "lintel/core/shm/ShmObject.h" + +#include "lintel/core/models/SharedMemorySegment.h" +#include "lintel/core/services/LoggerService.h" +#include "lintel/core/shm/BoostSegmentAllocator.h" +#include "lintel/core/shm/ShmSegmentAccessor.h" +#include "lintel/features/base/services/SharedMemoryService.h" +#include "lintel/features/base/services/ShmConstructors.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace lintel { +namespace shm { + namespace { + + /** Process wide migration registry state. */ + struct MigrationRegistry + { + std::mutex mutex; + std::map> steps; + }; + + /** @return the registry of this process */ + MigrationRegistry &migrationRegistry() + { + static MigrationRegistry instance; + return instance; + } + + /** @param typeId payload type + * @return a copy of the registered steps, so the callback cannot deadlock + * on the registry mutex if it registers another migration */ + std::vector stepsOf(std::uint32_t typeId) + { + MigrationRegistry ®istry = migrationRegistry(); + std::lock_guard lock(registry.mutex); + const auto found = registry.steps.find(typeId); + return found == registry.steps.end() ? std::vector{} : found->second; + } + + /** @param steps registered chain + * @param version current version + * @return the step that advances from version, or nullptr */ + const ShmPayloadMigrationStep *stepFrom(const std::vector &steps, std::uint16_t version) + { + const auto found = std::find_if(steps.begin(), steps.end(), [version](const ShmPayloadMigrationStep &step) { + return step.fromVersion == version; + }); + return found == steps.end() ? nullptr : &*found; + } + + /** Allocate a payload block and align it inside the block. + * @param segment allocator of the segment + * @param layout requested payload layout + * @return pointer to the aligned payload; the raw block is released on + * failure */ + void *allocatePayload(ShmSegmentAccessor &segment, ShmObject::Layout layout) + { + const std::size_t bytes = layout.size + layout.alignment - 1u; + void *raw = segment.allocate(bytes); + if (raw == nullptr) { throw std::bad_alloc(); } + void *aligned = raw; + std::size_t space = bytes; + if (std::align(layout.alignment, layout.size, aligned, space) == nullptr) { + segment.deallocate(raw); + throw std::bad_alloc(); + } + return aligned; + } + + /** Release the payload a header points at, if any. + * @param header object header + * @param segment allocator of the segment */ + void releasePayload(ShmObjectHeader &header, ShmSegmentAccessor &segment) noexcept + { + if (header.payloadOffset == 0 || header.payloadBytes == 0) { + header.payloadOffset = 0; + header.payloadBytes = 0; + header.state = ShmObjectHeader::kStateConstructing; + return; + } + void *payload = payloadAddressOf(&header); + try { + segment.deallocate(payload); + } catch (...) { + // The segment is being torn down or the block was already released; + // leaking is the only safe option here and the segment owns the bytes + // anyway. + } + header.payloadOffset = 0; + header.payloadBytes = 0; + header.state = ShmObjectHeader::kStateConstructing; + } + + /** Publish payload metadata into a header and run the payload constructor. + * The caller holds the segment's construct lock. + * @param header header inside the segment + * @param segment allocator of the segment + * @param layout payload layout + * @param typeId payload type id + * @param options registration options + * @param init payload constructor + * @param initArgument constructor arguments */ + void initializePayload(ShmObjectHeader &header, + ShmSegmentAccessor &segment, + ShmObject::Layout layout, + std::uint32_t typeId, + const ShmObjectOptions &options, + ShmObject::InitFunction init, + ShmObject::InitArgument initArgument) + { + releasePayload(header, segment); + void *payload = allocatePayload(segment, layout); + const std::intptr_t offset = reinterpret_cast(payload) - reinterpret_cast(&header); + try { + init(payload, initArgument); + } catch (...) { + segment.deallocate(payload); + header.payloadOffset = 0; + header.payloadBytes = 0; + throw; + } + header.magic = ShmObjectHeader::kMagic; + header.headerVersion = ShmObjectHeader::kHeaderVersion; + header.flags = 0; + header.payloadTypeId = typeId; + header.payloadVersion = options.payloadVersion; + header.persistence = static_cast(options.persistence); + header.reservedByte = 0; + header.payloadBytes = layout.size; + header.payloadAlignment = layout.alignment; + header.payloadOffset = offset; + header.state = ShmObjectHeader::kStateReady; + } + + /** Why an existing object cannot be used by this build. */ + std::string incompatibilityOf(const ShmObjectHeader &header, const ShmObject::Layout &layout, std::uint32_t typeId) + { + if (!header.isCompatible()) { return "its header layout is not compatible with this build"; } + if (header.payloadTypeId != typeId) { + return "it stores payload type " + std::to_string(header.payloadTypeId) + " but this build expects " + + std::to_string(typeId); + } + if (header.payloadBytes != layout.size) { + return "its payload is " + std::to_string(header.payloadBytes) + " bytes but this build expects " + + std::to_string(layout.size); + } + return "its payload is not initialised"; + } + + /** + * Locate the named object header, creating or migrating it as needed. + * + * Everything runs under the segment's construct lock, exactly like the + * former ShmCellRecord path: the process wide mutex serializes against a + * remap of this process, the cross-process semaphore against every other + * process. + */ + ShmObjectHeader *findRecordOf(SharedMemoryService &service, + const std::shared_ptr &segment, + const std::string &name, + ShmObject::Layout layout, + std::uint32_t typeId, + const ShmObjectOptions &options, + ShmObject::InitFunction init, + ShmObject::InitArgument initArgument) + { + std::lock_guard lock(service.segmentsMutexOf()); + auto &handle = service.segmentOf(segment->getName()); + SegmentSemaphoreGuard gate(handle); + if (handle.m_segmentAllocator == nullptr) { + handle.m_segmentAllocator = + std::make_shared(*handle.m_managedMappedFile->get_segment_manager()); + } + ShmSegmentAccessor accessor(*handle.m_segmentAllocator); + // A pre-header object can be smaller than the header; never read past its + // allocation. The layout marker normally catches such segments before + // this point, but the check keeps a hand-stamped segment from crashing. + const auto existing = handle.m_managedMappedFile->find(name.c_str()); + if (existing.first != nullptr && existing.second < sizeof(ShmObjectHeader)) { + throw std::runtime_error("shared memory object '" + name + "' of segment '" + segment->getName() + + "' is too small to carry an object header"); + } + auto *header = handle.m_managedMappedFile->find_or_construct(name.c_str())(); + + // A header in the constructing state is either brand new (default + // constructed by find_or_construct) or left over by a process that died + // while creating/migrating. The semaphore serializes us against every + // live process, so taking it over is safe - but only when the bytes + // actually carry our header magic; otherwise this is a pre-header object. + if (header->state == ShmObjectHeader::kStateConstructing && header->isCompatible()) { + initializePayload(*header, accessor, layout, typeId, options, init, initArgument); + return header; + } + if (!header->isCompatible()) { + // An object written before the header existed, or by a build whose + // header layout this one cannot read. Its persistence class is unknown, + // so it must be treated as authoritative. + throw std::runtime_error("shared memory object '" + name + "' of segment '" + segment->getName() + + "' has no compatible object header (layout mismatch); " + "register a migrator or recreate the segment"); + } + if (!header->isReady()) { + throw std::runtime_error("shared memory object '" + name + "' of segment '" + segment->getName() + + "' is still being initialised by another process"); + } + if (header->payloadVersion == options.payloadVersion) { + if (header->payloadTypeId != typeId || header->payloadBytes != layout.size) { + const std::string reason = incompatibilityOf(*header, layout, typeId); + if (header->persistenceClass() == ShmPersistence::Persistent) { + throw std::invalid_argument( + "shared memory object '" + name + "' of segment '" + segment->getName() + "' is incompatible: " + reason); + } + LOG_WARN("recreating shared memory object '{}' of segment '{}': {} (persistence allows it)", + name, + segment->getName(), + reason); + initializePayload(*header, accessor, layout, typeId, options, init, initArgument); + return header; + } + return header; + } + if (header->payloadVersion > options.payloadVersion) { + throw std::runtime_error("shared memory object '" + name + "' of segment '" + segment->getName() + + "' was written with payload version " + std::to_string(header->payloadVersion) + + ", but this build only knows version " + std::to_string(options.payloadVersion)); + } + + // Older payload. A registered chain wins; otherwise the persistence class + // decides whether the object may be recreated. + if (ShmPayloadMigrations::canMigrate(typeId, header->payloadVersion, options.payloadVersion)) { + LOG_WARN("migrating shared memory object '{}' of segment '{}' from payload version {} to {}", + name, + segment->getName(), + header->payloadVersion, + options.payloadVersion); + const std::vector migrated = ShmPayloadMigrations::migrate(typeId, + header->payloadVersion, + options.payloadVersion, + payloadAddressOf(header), + static_cast(header->payloadBytes), + layout.size); + void *payload = allocatePayload(accessor, layout); + std::memcpy(payload, migrated.data(), layout.size); + // Publish the new payload first, then release the old block: a process + // that dies in between leaks the old block, but never sees a header + // that points at freed memory. + void *oldPayload = payloadAddressOf(header); + header->payloadOffset = reinterpret_cast(payload) - reinterpret_cast(header); + header->payloadBytes = layout.size; + header->payloadAlignment = layout.alignment; + header->payloadVersion = options.payloadVersion; + header->payloadTypeId = typeId; + header->state = ShmObjectHeader::kStateReady; + try { + accessor.deallocate(oldPayload); + } catch (...) { + // See releasePayload(): the segment owns the bytes. + } + return header; + } + + if (header->persistenceClass() == ShmPersistence::Persistent) { + throw std::runtime_error("shared memory object '" + name + "' of segment '" + segment->getName() + + "' is persistent and was written with payload version " + + std::to_string(header->payloadVersion) + " (current " + + std::to_string(options.payloadVersion) + + "); register a payload migration or remove the segment"); + } + LOG_WARN( + "recreating shared memory object '{}' of segment '{}': payload version {} is not migratable, " + "persistence is {}", + name, + segment->getName(), + header->payloadVersion, + header->persistenceClass() == ShmPersistence::Volatile ? "volatile" : "derived"); + initializePayload(*header, accessor, layout, typeId, options, init, initArgument); + return header; + } + + }// namespace + + void ShmPayloadMigrations::registerStep(std::uint32_t typeId, ShmPayloadMigrationStep step) + { + if (step.fromVersion >= step.toVersion) { + throw std::invalid_argument("a payload migration must advance the version"); + } + if (!step.apply) { throw std::invalid_argument("a payload migration needs a callback"); } + MigrationRegistry ®istry = migrationRegistry(); + std::lock_guard lock(registry.mutex); + std::vector &steps = registry.steps[typeId]; + const auto found = std::find_if(steps.begin(), steps.end(), [&step](const ShmPayloadMigrationStep &existing) { + return existing.fromVersion == step.fromVersion; + }); + if (found == steps.end()) { + steps.push_back(std::move(step)); + } else { + *found = std::move(step); + } + } + + bool ShmPayloadMigrations::canMigrate(std::uint32_t typeId, std::uint16_t fromVersion, std::uint16_t toVersion) + { + if (fromVersion >= toVersion) { return fromVersion == toVersion; } + const std::vector steps = stepsOf(typeId); + std::uint16_t version = fromVersion; + while (version < toVersion) { + const ShmPayloadMigrationStep *step = stepFrom(steps, version); + if (step == nullptr || step->toVersion <= version) { return false; } + version = step->toVersion; + } + return true; + } + + std::vector ShmPayloadMigrations::migrate(std::uint32_t typeId, + std::uint16_t fromVersion, + std::uint16_t toVersion, + const void *source, + std::size_t sourceBytes, + std::size_t finalBytes) + { + if (fromVersion >= toVersion) { + std::vector unchanged(sourceBytes); + if (sourceBytes > 0 && source != nullptr) { std::memcpy(unchanged.data(), source, sourceBytes); } + return unchanged; + } + const std::vector steps = stepsOf(typeId); + std::vector current(sourceBytes); + if (sourceBytes > 0 && source != nullptr) { std::memcpy(current.data(), source, sourceBytes); } + std::uint16_t version = fromVersion; + while (version < toVersion) { + const ShmPayloadMigrationStep *step = stepFrom(steps, version); + if (step == nullptr || step->toVersion <= version) { + throw std::runtime_error( + "no payload migration from version " + std::to_string(version) + " for type " + std::to_string(typeId)); + } + if (step->sourceBytes != 0 && step->sourceBytes != current.size()) { + throw std::runtime_error("payload migration from version " + std::to_string(version) + " expects " + + std::to_string(step->sourceBytes) + " bytes but got " + + std::to_string(current.size())); + } + const std::size_t targetBytes = + step->targetBytes != 0 ? step->targetBytes : (step->toVersion == toVersion ? finalBytes : 0); + if (targetBytes == 0) { + throw std::runtime_error( + "payload migration from version " + std::to_string(version) + " does not name an intermediate size"); + } + std::vector next(targetBytes); + step->apply(current.data(), current.size(), next.data(), next.size()); + current = std::move(next); + version = step->toVersion; + } + if (current.size() != finalBytes) { + throw std::runtime_error("payload migration ended with " + std::to_string(current.size()) + " bytes but " + + std::to_string(finalBytes) + " were expected"); + } + return current; + } + + void ShmPayloadMigrations::reset() noexcept + { + MigrationRegistry ®istry = migrationRegistry(); + std::lock_guard lock(registry.mutex); + registry.steps.clear(); + } + + ShmObject::ShmObject(std::shared_ptr service, + std::shared_ptr segment, + std::string name, + Layout layout, + std::uint32_t typeId, + ShmObjectOptions options, + InitFunction init, + std::shared_ptr initArgument) + : m_service(std::move(service)), m_segment(std::move(segment)), m_name(std::move(name)), m_payloadSize(layout.size), + m_alignment(layout.alignment), m_typeId(typeId), m_options(options), m_init(init), + m_initArgument(std::move(initArgument)) + { + if (m_service == nullptr) { throw std::invalid_argument("ShmObject needs a shared memory service"); } + if (m_segment == nullptr) { + throw std::invalid_argument("ShmObject '" + m_name + "' needs a shared memory segment"); + } + if (m_name.empty()) { throw std::invalid_argument("ShmObject name must not be empty"); } + if (m_payloadSize == 0) { throw std::invalid_argument("ShmObject '" + m_name + "' payload must not be empty"); } + if (m_init == nullptr) { throw std::invalid_argument("ShmObject '" + m_name + "' needs a payload constructor"); } + if (m_initArgument == nullptr) { + throw std::invalid_argument("ShmObject '" + m_name + "' needs constructor arguments"); + } + if (m_alignment == 0 || m_alignment > kMaxAlignment || (m_alignment & (m_alignment - 1u)) != 0) { + throw std::invalid_argument("ShmObject '" + m_name + "' alignment must be a power of two of at most " + + std::to_string(kMaxAlignment) + " bytes"); + } + } + + void *ShmObject::resolve() const + { + const std::size_t observed = m_service->getGeneration(); + if (m_record != nullptr && observed == m_generation) { + auto *header = static_cast(m_record); + return payloadAddressOf(header); + } + auto *header = findRecordOf(*m_service, + m_segment, + m_name, + Layout{ m_payloadSize, m_alignment }, + m_typeId, + m_options, + m_init, + ShmObject::InitArgument{ m_initArgument.get() }); + m_record = header; + m_generation = observed; + return payloadAddressOf(header); + } + + const ShmObjectHeader &ShmObject::header() const + { + static_cast(resolve()); + return *static_cast(m_record); + } + + bool ShmObject::isStale() const { return m_record == nullptr || m_service->getGeneration() != m_generation; } + + std::size_t ShmObject::generation() const { return m_generation; } + +}// namespace shm +}// namespace lintel diff --git a/src/features/property/repositories/SharedMemoryPropertyRepository.cpp b/src/features/property/repositories/SharedMemoryPropertyRepository.cpp index a033031b..88a8a70c 100644 --- a/src/features/property/repositories/SharedMemoryPropertyRepository.cpp +++ b/src/features/property/repositories/SharedMemoryPropertyRepository.cpp @@ -1,6 +1,7 @@ #include "lintel/features/property/repositories/SharedMemoryPropertyRepository.h" #include +#include #include #include @@ -22,7 +23,12 @@ SharedMemoryPropertyRepository::SharedMemoryPropertyRepository( sharedMemorySegmentManager->of("shm_property"), m_version, std::string{ UUID }, - PropertyDataDto::getSize()), + PropertyDataDto::getSize(), + // The mirror is derived state: the properties can be rebuilt from their + // source of truth, so the library may recreate the map instead of + // aborting the start. The payload version is the first version of the + // layout this repository writes. + shm::ShmObjectOptions{ .payloadVersion = 1, .persistence = shm::ShmPersistence::Derived }), PropertyRepository(PropertyRepositoryType::SHM_REPOSITORY, configuration), m_processName(std::move(processName)), m_upgradableMutex(boost::interprocess::open_or_create, "shm_property_mutex_upgradable") {} @@ -136,11 +142,15 @@ void SharedMemoryPropertyRepository::deleteOf(const std::vector struct FirstIsObjectOptions : std::false_type { - /** payload size in bytes (> 0) */ - std::size_t size = 0; - /** payload alignment in bytes (power of two, <= kMaxAlignment) */ - std::size_t alignment = 1; }; - - /** Type-erased pointer to the payload constructor's arguments. Wrapping the - * raw pointer in a distinct type keeps the two pointer parameters of - * InitFunction from being swapped by mistake. */ - struct InitArgument + template + struct FirstIsObjectOptions : std::is_same, ShmObjectOptions> { - const void *value = nullptr; }; - /** Runs the payload constructor of T. Invoked exactly once - by the process - * that creates the cell, while the segment's construct lock is held - and - * never by a process that only finds the cell again. - * @param payload raw, not yet constructed payload memory - * @param initArgument type erased constructor arguments; the binding keeps - * them alive for as long as it exists */ - using InitFunction = void (*)(void *payload, InitArgument initArgument); - - /** Bind to the named cell without touching the segment yet; the first - * resolve() finds the cell or creates it (record + payload constructor). - * @param service shared memory service owning the segment - * @param segment segment the cell lives in - * @param name name of the cell inside the segment - * @param layout payload size and alignment - * @param init payload constructor, run when the cell is created - * @param initArgument constructor arguments for init, kept alive by this binding - * @throws std::invalid_argument on null service/segment, empty name, empty - * payload, unsupported alignment or missing constructor */ - ShmCell(std::shared_ptr service, - std::shared_ptr segment, - std::string name, - Layout layout, - InitFunction init, - std::shared_ptr initArgument); - - ShmCell(const ShmCell &) = delete; - ShmCell &operator=(const ShmCell &) = delete; - - /** Locate the payload, creating the cell on first use and re-reading the - * record after the segment was remapped (generation change). - * @return pointer to the payload, valid until the next remap of the segment */ - [[nodiscard]] void *resolve() const; - - /** @return true when the segment was remapped since the last resolve(), - * i.e. the cached record address has to be refreshed */ - [[nodiscard]] bool isStale() const; - - /** @return the service generation observed by the last resolve() */ - [[nodiscard]] std::size_t generation() const; - - /** @return name of the cell inside the segment */ - [[nodiscard]] const std::string &getName() const { return m_name; } - - /** @return segment the cell lives in */ - [[nodiscard]] const std::shared_ptr &getSegment() const { return m_segment; } - -private: - std::shared_ptr m_service; - std::shared_ptr m_segment; - std::string m_name; - std::size_t m_payloadSize; - std::size_t m_alignment; - InitFunction m_init; - std::shared_ptr m_initArgument; - /** Record inside the segment; opaque because its type (ShmCellRecord) is an - * implementation detail of SharedMemoryRepository.cpp. Mutable, because - * resolving refreshes the cache from const read access. */ - mutable void *m_record = nullptr; - mutable std::size_t m_generation = 0; -}; - -namespace shm { - /** * Typed repository for a single object of type T inside a shared memory * segment (Object/Array/RingBuffer/Queue storage). @@ -156,7 +55,9 @@ namespace shm { * base class of features/base/repositories/SharedMemoryRepository.h. * * The header is boost free: the boost interprocess find_or_construct lives - * in ShmCell / SharedMemoryRepository.cpp. + * in ShmObject / ShmObject.cpp. The on-disk object is a ShmObjectHeader plus + * a separate payload allocation; see ShmObjectHeader.h for the layout and + * migration contract. * * Requirements on T: usable in shared memory (no process local pointers, no * throwing destructor), identical in every process that uses the same cell @@ -170,7 +71,7 @@ namespace shm { "the payload alignment of a shared memory repository must not exceed one page"); public: - /** Find or create the cell holding one T. + /** Find or create the cell holding one T with default options. * @param service shared memory service of the segment * @param segment segment the object lives in * @param name name of the object inside the segment @@ -178,14 +79,37 @@ namespace shm { * @throws std::invalid_argument on invalid arguments * @throws ShmSegmentNotFound when the service does not know the segment */ template + requires(!FirstIsObjectOptions::value) SharedMemoryRepository(std::shared_ptr service, std::shared_ptr segment, std::string name, Args &&...args) + : SharedMemoryRepository(std::move(service), + std::move(segment), + std::move(name), + ShmObjectOptions{}, + std::forward(args)...) + {} + + /** Find or create the cell holding one T with explicit registration options. + * + * @param service shared memory service of the segment + * @param segment segment the object lives in + * @param name name of the object inside the segment + * @param options payload version, persistence class and optional type id + * @param args constructor arguments, used only when the cell is created */ + template + SharedMemoryRepository(std::shared_ptr service, + std::shared_ptr segment, + std::string name, + ShmObjectOptions options, + Args &&...args) : m_cell(std::move(service), std::move(segment), std::move(name), ShmCell::Layout{ sizeof(T), alignof(T) }, + options.payloadTypeId != 0 ? options.payloadTypeId : payloadTypeIdOf(), + options, &SharedMemoryRepository::template construct, std::make_shared...>>(std::forward(args)...)) { @@ -216,10 +140,30 @@ namespace shm { * already registered with another payload type * @throws ShmSegmentNotFound when the service does not know the segment */ template + requires(!FirstIsObjectOptions::value) static std::shared_ptr> of(std::shared_ptr service, std::shared_ptr segment, std::string name, Args &&...args) + { + return of( + std::move(service), std::move(segment), std::move(name), ShmObjectOptions{}, std::forward(args)...); + } + + /** Bind to the named cell with explicit registration options. + * + * @param service shared memory service of the segment + * @param segment segment the object lives in + * @param name name of the object inside the segment + * @param options payload version, persistence class and optional type id + * @param args constructor arguments, used only when the cell is created + * @return the repository bound to the cell */ + template + static std::shared_ptr> of(std::shared_ptr service, + std::shared_ptr segment, + std::string name, + ShmObjectOptions options, + Args &&...args) { if (service == nullptr || segment == nullptr) { throw std::invalid_argument("shared memory repository factory needs a service and a segment"); @@ -231,7 +175,7 @@ namespace shm { std::shared_ptr> existing = registry.getAs>(key); if (existing != nullptr) { return existing; } std::shared_ptr> repository = std::make_shared>( - std::move(service), std::move(segment), std::move(name), std::forward(args)...); + std::move(service), std::move(segment), std::move(name), options, std::forward(args)...); registry.registerRepository(key, repository); return repository; } @@ -266,6 +210,15 @@ namespace shm { /** @return segment the object lives in */ [[nodiscard]] const std::shared_ptr &getSegment() const { return m_cell.getSegment(); } + /** @return the stable header of the object inside the segment */ + [[nodiscard]] const ShmObjectHeader &getHeader() const { return m_cell.header(); } + + /** @return persistence class recorded in the segment */ + [[nodiscard]] ShmPersistence getPersistence() const { return m_cell.persistence(); } + + /** @return payload version recorded in the segment */ + [[nodiscard]] std::uint16_t getPayloadVersion() const { return m_cell.payloadVersion(); } + private: /** Placement-construct the payload from the stored constructor arguments. * diff --git a/src/include/lintel/core/shm/ShmObject.h b/src/include/lintel/core/shm/ShmObject.h new file mode 100644 index 00000000..d9648cd3 --- /dev/null +++ b/src/include/lintel/core/shm/ShmObject.h @@ -0,0 +1,120 @@ +#ifndef LINTEL_SHM_OBJECT_H +#define LINTEL_SHM_OBJECT_H + +#include +#include +#include +#include + +#include "lintel/core/shm/ShmObjectHeader.h" + +namespace lintel { + +class SharedMemoryService; +class SharedMemorySegment; + +namespace shm { + + /** + * Process local binding of one named, versioned object inside a segment. + * + * The named object in the segment is a ShmObjectHeader; the payload itself is + * a separate allocation, addressed by an offset relative to the header. The + * binding resolves the header, creates the payload on first use and migrates + * it when the stored payloadVersion differs from the version this build + * expects. + * + * The header is boost free: finding/constructing it, allocating the payload + * and running the migration live in the out-of-line methods in + * src/features/base/services/ShmObject.cpp. + * + * A binding lives as long as its owner (a repository or a construct helper). + * When the segment is remapped, resolve() re-reads the header from the new + * mapping; the payload pointer is only valid until that remap. + */ + class ShmObject + { + public: + /** Highest payload alignment the relative addressing supports. */ + static constexpr std::size_t kMaxAlignment = 4096; + + /** Payload size and alignment. */ + struct Layout + { + /** payload size in bytes (> 0) */ + std::size_t size = 0; + /** payload alignment in bytes (power of two, <= kMaxAlignment) */ + std::size_t alignment = 1; + }; + + /** Type-erased pointer to the payload constructor's arguments. */ + struct InitArgument + { + const void *value = nullptr; + }; + + /** Runs the payload constructor of T exactly once, when the object is + * created or recreated. + * @param payload raw, not yet constructed payload memory + * @param initArgument type erased constructor arguments */ + using InitFunction = void (*)(void *payload, InitArgument initArgument); + + /** Bind to the named object without touching the segment yet. */ + ShmObject(std::shared_ptr service, + std::shared_ptr segment, + std::string name, + Layout layout, + std::uint32_t typeId, + ShmObjectOptions options, + InitFunction init, + std::shared_ptr initArgument); + + ShmObject(const ShmObject &) = delete; + ShmObject &operator=(const ShmObject &) = delete; + + /** Locate the payload, creating or migrating it on first use and + * re-reading the header after the segment was remapped. + * @return pointer to the payload, valid until the next remap */ + [[nodiscard]] void *resolve() const; + + /** @return the header of the object; resolves the object first */ + [[nodiscard]] const ShmObjectHeader &header() const; + + /** @return persistence class recorded in the segment */ + [[nodiscard]] ShmPersistence persistence() const { return header().persistenceClass(); } + + /** @return payload version recorded in the segment */ + [[nodiscard]] std::uint16_t payloadVersion() const { return header().payloadVersion; } + + /** @return true when the segment was remapped since the last resolve */ + [[nodiscard]] bool isStale() const; + + /** @return the service generation observed by the last resolve */ + [[nodiscard]] std::size_t generation() const; + + /** @return name of the object inside the segment */ + [[nodiscard]] const std::string &getName() const { return m_name; } + + /** @return segment the object lives in */ + [[nodiscard]] const std::shared_ptr &getSegment() const { return m_segment; } + + private: + std::shared_ptr m_service; + std::shared_ptr m_segment; + std::string m_name; + std::size_t m_payloadSize; + std::size_t m_alignment; + std::uint32_t m_typeId; + ShmObjectOptions m_options; + InitFunction m_init; + std::shared_ptr m_initArgument; + /** Header inside the segment; opaque because creating it is an + * implementation detail of ShmObject.cpp. */ + mutable void *m_record = nullptr; + mutable std::size_t m_generation = 0; + }; + +}// namespace shm +}// namespace lintel + +#endif// LINTEL_SHM_OBJECT_H diff --git a/src/include/lintel/core/shm/ShmObjectHeader.h b/src/include/lintel/core/shm/ShmObjectHeader.h new file mode 100644 index 00000000..606a2516 --- /dev/null +++ b/src/include/lintel/core/shm/ShmObjectHeader.h @@ -0,0 +1,237 @@ +#ifndef LINTEL_SHM_OBJECT_HEADER_H +#define LINTEL_SHM_OBJECT_HEADER_H + +#include +#include +#include +#include +#include +#include +#include + +#include "lintel/core/utils/TypeName.h" + +namespace lintel { +namespace shm { + + /** + * What may happen to one object when the library cannot interpret its payload. + * + * The class is stored inside the segment (in ShmObjectHeader) and is the + * library's answer to "may I discard this without asking the operator?". + */ + enum class ShmPersistence : std::uint8_t { + /** Ephemeral state (event bus rings, message queues): losing it is defined + * behaviour, a new object is created instead. */ + Volatile = 0, + /** Reconstructable from a source of truth (database, XML): the owning + * service rebuilds it after the object was recreated. */ + Derived = 1, + /** Authoritative data: it has to be migrated, or the start aborts. */ + Persistent = 2, + }; + + /** Registration options of one named shared memory object. */ + struct ShmObjectOptions + { + /** Version of the payload type; bumped when T's layout or meaning changes. */ + std::uint16_t payloadVersion = 1; + /** What the library may do when the payload cannot be read. */ + ShmPersistence persistence = ShmPersistence::Persistent; + /** Explicit type id. 0 means "derive from the payload type name" (the + * default in the typed helpers and shared memory repositories). */ + std::uint32_t payloadTypeId = 0; + }; + + /** + * Stable header of every named object in a shared memory segment. + * + * The header sits at the first byte of the named object and is the only part + * whose layout the library promises to keep. The payload itself is a separate + * allocation of the segment, addressed by payloadOffset relative to the + * header - changing the library's bookkeeping therefore never moves the + * payload bytes. + * + * headerVersion covers this struct, payloadVersion covers the type stored in + * the payload. A header layout change is a separate, deliberate migration + * step; new fields must go into the reserved area until then. + * + * The struct is a plain POD: it is created by boost::interprocess inside the + * segment and must not contain pointers, virtual functions or non-trivial + * members. + */ + struct alignas(16) ShmObjectHeader + { + /** "LSHO" in ASCII. */ + static constexpr std::uint32_t kMagic = 0x4c53484fU; + /** Layout version of this header. */ + static constexpr std::uint16_t kHeaderVersion = 1; + /** The payload allocation is being initialised; not readable yet. */ + static constexpr std::uint32_t kStateConstructing = 1; + /** Header and payload are complete and readable. */ + static constexpr std::uint32_t kStateReady = 2; + + std::uint32_t magic = kMagic; + std::uint16_t headerVersion = kHeaderVersion; + std::uint16_t flags = 0; + /** FNV-1a id of the payload type name (see payloadTypeIdOf()). */ + std::uint32_t payloadTypeId = 0; + std::uint16_t payloadVersion = 0; + std::uint8_t persistence = static_cast(ShmPersistence::Persistent); + std::uint8_t reservedByte = 0; + std::uint32_t state = kStateConstructing; + std::uint32_t reservedWord = 0; + /** Size of the payload allocation in bytes. */ + std::uint64_t payloadBytes = 0; + /** Alignment of the payload allocation in bytes. */ + std::uint64_t payloadAlignment = 1; + /** Distance from this header to the payload; stable in every mapping. */ + std::int64_t payloadOffset = 0; + std::uint64_t reserved[2]{}; + + /** @return true when magic and header layout version match this build */ + [[nodiscard]] bool isCompatible() const noexcept { return magic == kMagic && headerVersion == kHeaderVersion; } + + /** @return true when the payload was completely initialised */ + [[nodiscard]] bool isReady() const noexcept { return state == kStateReady; } + + /** @return the persistence class stored in the header */ + [[nodiscard]] ShmPersistence persistenceClass() const noexcept { return static_cast(persistence); } + }; + + static_assert(sizeof(ShmObjectHeader) == 64, "ShmObjectHeader is part of the on-disk layout"); + static_assert(alignof(ShmObjectHeader) == 16, "ShmObjectHeader alignment is part of the on-disk layout"); + static_assert(std::is_trivially_copyable_v, "ShmObjectHeader must be a POD"); + + /** @param header a compatible, ready header + * @return address of the payload in this process' mapping */ + inline void *payloadAddressOf(ShmObjectHeader *header) noexcept + { + if (header == nullptr) { return nullptr; } + const std::uintptr_t address = + reinterpret_cast(header) + static_cast(header->payloadOffset); + return std::bit_cast(address); + } + + /** @param header a compatible, ready header + * @return address of the payload in this process' mapping */ + inline const void *payloadAddressOf(const ShmObjectHeader *header) noexcept + { + return payloadAddressOf(const_cast(header)); + } + + /** FNV-1a hash of a type name. Stable within one build and independent of + * the address space, which is what a shared header can store. */ + constexpr std::uint32_t fnv1a32(std::string_view text) noexcept + { + std::uint32_t hash = 2166136261U; + for (const char character : text) { + hash ^= static_cast(static_cast(character)); + hash *= 16777619U; + } + return hash; + } + + /** @tparam T payload type + * @return type id stored in ShmObjectHeader::payloadTypeId */ + template constexpr std::uint32_t payloadTypeIdOf() noexcept { return fnv1a32(type_name_full()); } + + /** + * One migration step of a payload type: reads a complete object of + * fromVersion and writes a complete object of toVersion. + * + * sourceBytes/targetBytes describe the exact sizes of the two intermediate + * representations. A targetBytes of 0 means "the final payload size" (the + * size the current build registers for T). + */ + struct ShmPayloadMigrationStep + { + std::uint16_t fromVersion = 0; + std::uint16_t toVersion = 0; + std::size_t sourceBytes = 0; + std::size_t targetBytes = 0; + std::function apply; + }; + + /** + * Process wide registry of payload migrations, keyed by type id. + * + * Steps are chained: a registered 1→2 and 2→3 step migrates a version 1 + * object to version 3 in one open. Registration happens during startup, + * before the first repository is constructed; lookup and migration run under + * the segment's construct lock. + */ + class ShmPayloadMigrations + { + public: + /** Register or replace one step. + * @param typeId id of the payload type + * @param step the step; a step without `apply` is rejected + * @throws std::invalid_argument on an invalid step */ + static void registerStep(std::uint32_t typeId, ShmPayloadMigrationStep step); + + /** @return true when a complete chain from fromVersion to toVersion exists */ + [[nodiscard]] static bool canMigrate(std::uint32_t typeId, std::uint16_t fromVersion, std::uint16_t toVersion); + + /** + * Run the chain and return the migrated payload bytes. + * + * @param typeId id of the payload type + * @param fromVersion version stored in the segment + * @param toVersion version of this build + * @param source old payload bytes + * @param sourceBytes size of the old payload + * @param finalBytes size the final payload must have (sizeof(T)) + * @return the migrated bytes, exactly finalBytes long + * @throws std::runtime_error when no chain exists or a step is broken + */ + [[nodiscard]] static std::vector migrate(std::uint32_t typeId, + std::uint16_t fromVersion, + std::uint16_t toVersion, + const void *source, + std::size_t sourceBytes, + std::size_t finalBytes); + + /** Drop every registered step. Intended for tests. */ + static void reset() noexcept; + }; + + /** + * Register a v(n) → v(n+1) migration for T. + * + * The callback receives the old payload bytes and a target buffer of the + * current payload size; it has to placement-construct (or copy) the current T + * into the target. + * + * @tparam T payload type + * @param fromVersion version stored in the segment + * @param toVersion version this build writes + * @param apply migration callback + * @param typeId type id; defaults to payloadTypeIdOf() + * @param sourceBytes expected size of the old payload (0 = use the header) + * @param targetBytes size after this step (defaults to sizeof(T)) + */ + template + void registerPayloadMigration(std::uint16_t fromVersion, + std::uint16_t toVersion, + std::function apply, + std::uint32_t typeId = payloadTypeIdOf(), + std::size_t sourceBytes = 0, + std::size_t targetBytes = sizeof(T)) + { + ShmPayloadMigrationStep step; + step.fromVersion = fromVersion; + step.toVersion = toVersion; + step.sourceBytes = sourceBytes; + step.targetBytes = targetBytes; + step.apply = [callback = std::move(apply)]( + const void *source, std::size_t sourceBytes, void *target, std::size_t /*targetBytes*/) { + if (callback) { callback(source, sourceBytes, target); } + }; + ShmPayloadMigrations::registerStep(typeId, std::move(step)); + } + +}// namespace shm +}// namespace lintel + +#endif// LINTEL_SHM_OBJECT_HEADER_H diff --git a/src/include/lintel/core/shm/ShmSegmentAccessor.h b/src/include/lintel/core/shm/ShmSegmentAccessor.h index de5509e8..5d7a1fa0 100644 --- a/src/include/lintel/core/shm/ShmSegmentAccessor.h +++ b/src/include/lintel/core/shm/ShmSegmentAccessor.h @@ -1,14 +1,21 @@ #ifndef LINTEL_SHMSEGMENTACCESSOR_H #define LINTEL_SHMSEGMENTACCESSOR_H +#include +#include #include +#include #include #include #include #include +#include #include #include #include +#include + +#include "lintel/core/shm/ShmObjectHeader.h" namespace lintel { /** @@ -43,6 +50,12 @@ class ShmSegmentAccessor /** Capacity of the stored segment name in bytes, including the terminating NUL. */ static constexpr std::size_t NAME_SIZE = 32; + /** Upper bound for waiting until the creator of a payload published it. */ + static constexpr std::chrono::milliseconds kPayloadInitTimeout{ 5000 }; + + /** Granularity of the wait for that publication. */ + static constexpr std::chrono::milliseconds kPayloadInitWaitSlice{ 1 }; + /** Process local allocator an accessor forwards to. */ class Allocator { @@ -153,6 +166,78 @@ class ShmSegmentAccessor return allocator()->findOrConstructArray(name, bytes); } + /** + * Find or construct the versioned payload of a named object. + * + * The named object in the segment is a shm::ShmObjectHeader (allocated by + * findOrConstructArray()), the payload itself is a separate allocation + * addressed by the header. That separation is what keeps the payload in place + * when the library's bookkeeping changes: a build that can read the header + * finds the same payload, a build that cannot leaves the bytes untouched and + * reports it. + * + * A freshly created header is claimed with a compare exchange, so exactly one + * process allocates and publishes the payload; everybody else waits for the + * header to become ready. A ready payload that this build cannot use is + * recreated (zeroed) when its persistence class allows it - Volatile or + * Derived - and rejected otherwise. No destructor of the old payload runs: + * the old type may not even exist in this build, so nested allocations of it + * stay in the segment until the segment is recreated. + * + * The alignment is limited to alignof(std::max_align_t): the segment + * allocator returns blocks aligned to that value, so the aligned payload is + * the block start and a later deallocate() on the payload pointer is valid. + * + * @param name name of the header object inside the segment + * @param bytes payload size in bytes (must be positive) + * @param alignment payload alignment in bytes (power of two, at most + * alignof(std::max_align_t)) + * @param typeId id of the payload type, stored in the header + * @param options payload version and persistence class + * @return pointer to the payload, or nullptr when the allocator does not + * support named objects + * @throws std::invalid_argument on an invalid request, or when a persistent + * object stores a payload this build cannot use + * @throws std::runtime_error when the named object was written by an + * incompatible build or its initialisation does not finish in time + */ + [[nodiscard]] void *findOrConstructPayload(const char *name, + std::size_t bytes, + std::size_t alignment, + std::uint32_t typeId, + shm::ShmObjectOptions options) const + { + validatePayloadRequest(name, bytes, alignment); + auto *header = static_cast(findOrConstructArray(name, sizeof(shm::ShmObjectHeader))); + if (header == nullptr) { return nullptr; } + + if (isBlankHeader(*header)) { + // Fresh header: the allocator zero initialised the block, so nothing was + // published yet. Claim it, so no second process allocates a payload of + // its own behind the same header. + std::atomic_ref state(header->state); + std::uint32_t expected = 0; + if (state.compare_exchange_strong( + expected, shm::ShmObjectHeader::kStateConstructing, std::memory_order_acq_rel, std::memory_order_acquire)) { + try { + initialisePayload(*header, bytes, alignment, typeId, options); + } catch (...) { + // Release the claim: a header stuck in "constructing" would make + // every later open wait for a payload that will never be published. + state.store(0u, std::memory_order_release); + throw; + } + return shm::payloadAddressOf(header); + } + // Another process claimed the header between our lookup and the claim: + // fall through and wait for its payload. + } + + awaitReadyHeader(*header, name); + reconcilePayload(*header, name, bytes, alignment, typeId, options); + return shm::payloadAddressOf(header); + } + /** @return the segment name this accessor identifies its segment by; empty for an unnamed accessor */ [[nodiscard]] std::string_view segmentName() const noexcept { return std::string_view(m_segmentName); } @@ -182,6 +267,173 @@ class ShmSegmentAccessor + " characters"); } + /** @param name name of the payload object + * @param bytes requested payload size + * @param alignment requested payload alignment + * @throws std::invalid_argument on a request the segment cannot serve */ + static void validatePayloadRequest(const char *name, std::size_t bytes, std::size_t alignment) + { + if (name == nullptr || name[0] == '\0') { + throw std::invalid_argument("a named shared memory payload needs a name"); + } + if (bytes == 0) { + throw std::invalid_argument("the payload of shared memory object '" + std::string(name) + "' must not be empty"); + } + if (alignment == 0 || (alignment & (alignment - 1u)) != 0) { + throw std::invalid_argument( + "the payload alignment of shared memory object '" + std::string(name) + "' must be a power of two"); + } + if (alignment > alignof(std::max_align_t)) { + // The segment allocator aligns to max_align_t, so a larger alignment would + // make the payload pointer differ from the allocated block start - and a + // later deallocate() on the payload pointer would then be invalid. + throw std::invalid_argument("the payload alignment of shared memory object '" + std::string(name) + + "' must not exceed " + std::to_string(alignof(std::max_align_t)) + " bytes"); + } + } + + /** @param header header of a named object + * @return true when the header was created but nothing was published yet; + * a header claimed by another process still counts as blank */ + static bool isBlankHeader(const shm::ShmObjectHeader &header) noexcept + { + return header.magic == 0 && header.headerVersion == 0 && header.payloadBytes == 0 && header.payloadOffset == 0; + } + + /** Wait until the creator of a payload published the header. + * + * The wait is bounded: a process that died while claiming the header leaves + * it in the constructing state forever, and taking it over could free a + * payload somebody else is about to publish. Failing loudly is the safe + * direction. + * + * @param header header of the object + * @param name name of the object, for the error message + * @throws std::runtime_error when the object has no compatible header or the + * publication does not finish within kPayloadInitTimeout */ + static void awaitReadyHeader(shm::ShmObjectHeader &header, const char *name) + { + const auto deadline = std::chrono::steady_clock::now() + kPayloadInitTimeout; + for (;;) { + if (header.isCompatible() && header.isReady()) { return; } + if (!isBlankHeader(header) && !header.isCompatible()) { + throw std::runtime_error("shared memory object '" + std::string(name) + + "' has no compatible object header (written by an older build or with a different " + "layout); register a migrator or recreate the segment"); + } + if (std::chrono::steady_clock::now() >= deadline) { + throw std::runtime_error("timed out waiting for the initialisation of shared memory object '" + + std::string(name) + "': the process that started to create it did not finish"); + } + std::this_thread::sleep_for(kPayloadInitWaitSlice); + } + } + + /** Validate a ready header against this build's request and recreate the + * payload when its persistence class allows that. + * + * @param header ready header of the object + * @param name name of the object, for the error message + * @param bytes payload size this build expects + * @param alignment payload alignment this build expects + * @param typeId payload type id this build expects + * @param options version and persistence class this build registers + * @throws std::invalid_argument when a persistent object is incompatible */ + void reconcilePayload(shm::ShmObjectHeader &header, + const char *name, + std::size_t bytes, + std::size_t alignment, + std::uint32_t typeId, + const shm::ShmObjectOptions &options) const + { + const bool samePayload = header.payloadVersion == options.payloadVersion && header.payloadTypeId == typeId + && header.payloadBytes == static_cast(bytes) + && header.payloadAlignment == static_cast(alignment); + if (samePayload) { return; } + if (header.persistenceClass() == shm::ShmPersistence::Persistent) { + throw std::invalid_argument("shared memory object '" + std::string(name) + "' is incompatible: it stores payload " + "type " + std::to_string(header.payloadTypeId) + " version " + + std::to_string(header.payloadVersion) + " with " + + std::to_string(header.payloadBytes) + " bytes, but this build expects type " + + std::to_string(typeId) + " version " + std::to_string(options.payloadVersion) + + " with " + std::to_string(bytes) + " bytes"); + } + // Volatile or Derived: recreate the payload zeroed. No destructor of the + // old payload runs - the old type may not exist in this build. + initialisePayload(header, bytes, alignment, typeId, options); + } + + /** Release the payload block a header describes, if any, and put the header + * back into the freshly created state. + * + * @param header header of the object */ + void releasePayload(shm::ShmObjectHeader &header) const noexcept + { + // Stop readers first: a header marked constructing is not dereferenced. + header.state = shm::ShmObjectHeader::kStateConstructing; + const bool hasPayload = header.payloadOffset != 0 && header.payloadBytes != 0; + if (hasPayload) { + void *payload = shm::payloadAddressOf(&header); + try { + deallocate(payload); + } catch (...) { + // The segment is being torn down or the block was already released; + // leaking is the only safe option and the segment owns the bytes. + } + } + // Back to the blank state of a freshly created header: a failed + // initialisation must not leave a header that every later open waits for. + header.magic = 0; + header.headerVersion = 0; + header.payloadTypeId = 0; + header.payloadVersion = 0; + header.payloadOffset = 0; + header.payloadBytes = 0; + header.payloadAlignment = 1; + } + + /** Allocate the payload, publish it into the header and mark the header ready. + * The caller claimed the header with state == kStateConstructing. + * + * @param header header of the object + * @param bytes payload size in bytes + * @param alignment payload alignment in bytes + * @param typeId payload type id + * @param options version and persistence class to record + * @throws std::bad_alloc when the segment cannot serve the allocation */ + void initialisePayload(shm::ShmObjectHeader &header, + std::size_t bytes, + std::size_t alignment, + std::uint32_t typeId, + const shm::ShmObjectOptions &options) const + { + releasePayload(header); + const std::size_t allocationBytes = bytes + alignment - 1u; + void *raw = allocate(allocationBytes); + if (raw == nullptr) { throw std::bad_alloc(); } + void *aligned = raw; + std::size_t space = allocationBytes; + if (std::align(alignment, bytes, aligned, space) == nullptr) { + deallocate(raw); + throw std::bad_alloc(); + } + // A fresh block carries whatever the segment held before: the payload must + // start zeroed, like the named array of the old layout did. + std::memset(aligned, 0, bytes); + header.magic = shm::ShmObjectHeader::kMagic; + header.headerVersion = shm::ShmObjectHeader::kHeaderVersion; + header.flags = 0; + header.payloadTypeId = typeId; + header.payloadVersion = options.payloadVersion; + header.persistence = static_cast(options.persistence); + header.reservedByte = 0; + header.payloadBytes = static_cast(bytes); + header.payloadAlignment = static_cast(alignment); + header.payloadOffset = reinterpret_cast(aligned) - reinterpret_cast(&header); + // Publish last: a reader that sees kStateReady sees every field above. + header.state = shm::ShmObjectHeader::kStateReady; + } + /** * Allocator to forward to: the one bound to this accessor's segment name in * this process, otherwise the one captured at construction time. diff --git a/src/include/lintel/core/utils/TypeName.h b/src/include/lintel/core/utils/TypeName.h index b264ff13..20fcd5f8 100644 --- a/src/include/lintel/core/utils/TypeName.h +++ b/src/include/lintel/core/utils/TypeName.h @@ -3,31 +3,45 @@ #include namespace lintel { -/** Return a human-readable name for the given type at compile time. +/** Return the fully qualified name of the given type at compile time. + * + * Unlike type_name(), namespaces are kept. Shared memory uses this form to + * derive a type id, because two types with the same unqualified name in + * different namespaces would otherwise share an id. + * * @tparam T the type to name - * @return the type name as a string view */ -template constexpr std::string_view type_name() + * @return the fully qualified type name as a string view */ +template constexpr std::string_view type_name_full() { std::string_view name, prefix, suffix; #ifdef __clang__ name = __PRETTY_FUNCTION__; - prefix = "std::string_view lintel::type_name() [T = "; + prefix = "std::string_view lintel::type_name_full() [T = "; suffix = "]"; #elif defined(__GNUC__) name = __PRETTY_FUNCTION__; - prefix = "constexpr std::string_view lintel::type_name() [with T = "; + prefix = "constexpr std::string_view lintel::type_name_full() [with T = "; suffix = "; std::string_view = std::basic_string_view]"; #elif defined(_MSC_VER) name = __FUNCSIG__; prefix = "class std::basic_string_view > " - "__cdecl lintel::type_name<"; + "__cdecl lintel::type_name_full<"; suffix = ">(void)"; #endif name.remove_prefix(prefix.size()); name.remove_suffix(suffix.size()); + return name; +} + +/** Return a human-readable name for the given type at compile time. + * @tparam T the type to name + * @return the unqualified type name as a string view */ +template constexpr std::string_view type_name() +{ + std::string_view name = type_name_full(); auto pos = name.rfind("::"); if (pos != std::string_view::npos) { name.remove_prefix(pos + 2); } return name; } -}// namespace lintel \ No newline at end of file +}// namespace lintel diff --git a/src/include/lintel/features/base/repositories/SharedMemoryRepository.h b/src/include/lintel/features/base/repositories/SharedMemoryRepository.h index 74aae786..b321775c 100644 --- a/src/include/lintel/features/base/repositories/SharedMemoryRepository.h +++ b/src/include/lintel/features/base/repositories/SharedMemoryRepository.h @@ -121,6 +121,7 @@ class SharedMemoryArrayRepository : public SharedMemoryRepository private: mutable std::array *m_array; mutable std::size_t m_generation; + shm::ShmObjectOptions m_options; protected: std::shared_ptr m_sharedMemoryService; @@ -131,7 +132,7 @@ class SharedMemoryArrayRepository : public SharedMemoryRepository const auto generation = m_sharedMemoryService->getGeneration(); if (generation != m_generation) { m_array = &shm::constructArray( - *m_sharedMemoryService, getSharedMemorySegment(), getSharedMemoryRepository()); + *m_sharedMemoryService, getSharedMemorySegment(), getSharedMemoryRepository(), m_options); m_generation = generation; } return *m_array; @@ -141,10 +142,11 @@ class SharedMemoryArrayRepository : public SharedMemoryRepository SharedMemoryArrayRepository(const std::shared_ptr &sharedMemoryService, const std::shared_ptr &segment, int32_t codeVersion, - std::string uuid) + std::string uuid, + shm::ShmObjectOptions options = {}) : SharedMemoryRepository(segment, sizeof(DATA), std::string(type_name()), codeVersion, uuid), - m_array(&shm::constructArray(*sharedMemoryService, segment, getSharedMemoryRepository())), - m_generation(sharedMemoryService->getGeneration()), m_sharedMemoryService(sharedMemoryService) + m_array(&shm::constructArray(*sharedMemoryService, segment, getSharedMemoryRepository(), options)), + m_generation(sharedMemoryService->getGeneration()), m_options(options), m_sharedMemoryService(sharedMemoryService) {} [[nodiscard]] SharedMemoryType getType() const override { return SharedMemoryType::Array; } @@ -186,6 +188,7 @@ class SharedMemoryVectorRepository : public SharedMemoryRepository boost::interprocess::allocator>; mutable Vector *m_vector; mutable std::size_t m_generation; + shm::ShmObjectOptions m_options; protected: std::shared_ptr m_sharedMemoryService; @@ -194,8 +197,8 @@ class SharedMemoryVectorRepository : public SharedMemoryRepository { const auto generation = m_sharedMemoryService->getGeneration(); if (generation != m_generation) { - m_vector = - &shm::constructVector(*m_sharedMemoryService, getSharedMemorySegment(), getSharedMemoryRepository()); + m_vector = &shm::constructVector( + *m_sharedMemoryService, getSharedMemorySegment(), getSharedMemoryRepository(), m_options); m_generation = generation; } return *m_vector; @@ -205,10 +208,11 @@ class SharedMemoryVectorRepository : public SharedMemoryRepository SharedMemoryVectorRepository(const std::shared_ptr &sharedMemoryService, const std::shared_ptr &segment, int32_t codeVersion, - std::string uuid) + std::string uuid, + shm::ShmObjectOptions options = {}) : SharedMemoryRepository(segment, sizeof(DATA), std::string(type_name()), codeVersion, uuid), - m_vector(&shm::constructVector(*sharedMemoryService, segment, getSharedMemoryRepository())), - m_generation(sharedMemoryService->getGeneration()), m_sharedMemoryService(sharedMemoryService) + m_vector(&shm::constructVector(*sharedMemoryService, segment, getSharedMemoryRepository(), options)), + m_generation(sharedMemoryService->getGeneration()), m_options(options), m_sharedMemoryService(sharedMemoryService) {} [[nodiscard]] SharedMemoryType getType() const override { return SharedMemoryType::Vector; } @@ -255,6 +259,7 @@ class SharedMemoryMapRepository : public SharedMemoryRepository boost::interprocess::managed_mapped_file::segment_manager>>; mutable Map *m_map; mutable std::size_t m_generation; + shm::ShmObjectOptions m_options; protected: std::shared_ptr m_sharedMemoryService; @@ -263,8 +268,8 @@ class SharedMemoryMapRepository : public SharedMemoryRepository { const auto generation = m_sharedMemoryService->getGeneration(); if (generation != m_generation) { - m_map = - &shm::constructMap(*m_sharedMemoryService, getSharedMemorySegment(), getSharedMemoryRepository()); + m_map = &shm::constructMap( + *m_sharedMemoryService, getSharedMemorySegment(), getSharedMemoryRepository(), m_options); m_generation = generation; } return *m_map; @@ -275,10 +280,11 @@ class SharedMemoryMapRepository : public SharedMemoryRepository const std::shared_ptr &segment, int32_t codeVersion, std::string uuid, - int32_t sizeOfData = sizeof(VALUE)) + int32_t sizeOfData = sizeof(VALUE), + shm::ShmObjectOptions options = {}) : SharedMemoryRepository(segment, sizeOfData, std::string(type_name()), codeVersion, uuid), - m_map(&shm::constructMap(*sharedMemoryService, segment, getSharedMemoryRepository())), - m_generation(sharedMemoryService->getGeneration()), m_sharedMemoryService(sharedMemoryService) + m_map(&shm::constructMap(*sharedMemoryService, segment, getSharedMemoryRepository(), options)), + m_generation(sharedMemoryService->getGeneration()), m_options(options), m_sharedMemoryService(sharedMemoryService) { static_assert(isSerializable()); static_assert(isSerializable()); @@ -334,6 +340,7 @@ class SharedMemoryObjectRepository : public SharedMemoryRepository private: mutable DATA *m_data; mutable std::size_t m_generation; + shm::ShmObjectOptions m_options; protected: std::shared_ptr m_sharedMemoryService; @@ -342,8 +349,8 @@ class SharedMemoryObjectRepository : public SharedMemoryRepository { const auto generation = m_sharedMemoryService->getGeneration(); if (generation != m_generation) { - m_data = - &shm::constructObject(*m_sharedMemoryService, getSharedMemorySegment(), getSharedMemoryRepository()); + m_data = &shm::constructObject( + *m_sharedMemoryService, getSharedMemorySegment(), getSharedMemoryRepository(), m_options); m_generation = generation; } return *m_data; @@ -353,10 +360,11 @@ class SharedMemoryObjectRepository : public SharedMemoryRepository SharedMemoryObjectRepository(const std::shared_ptr &sharedMemoryService, const std::shared_ptr &segment, int32_t codeVersion, - std::string uuid) + std::string uuid, + shm::ShmObjectOptions options = {}) : SharedMemoryRepository(segment, sizeof(DATA), std::string(type_name()), codeVersion, uuid), - m_data(&shm::constructObject(*sharedMemoryService, segment, getSharedMemoryRepository())), - m_generation(sharedMemoryService->getGeneration()), m_sharedMemoryService(sharedMemoryService) + m_data(&shm::constructObject(*sharedMemoryService, segment, getSharedMemoryRepository(), options)), + m_generation(sharedMemoryService->getGeneration()), m_options(options), m_sharedMemoryService(sharedMemoryService) {} [[nodiscard]] SharedMemoryType getType() const override { return SharedMemoryType::Object; } diff --git a/src/include/lintel/features/base/services/ShmConstructors.h b/src/include/lintel/features/base/services/ShmConstructors.h index 6f1eb701..12cc76b1 100644 --- a/src/include/lintel/features/base/services/ShmConstructors.h +++ b/src/include/lintel/features/base/services/ShmConstructors.h @@ -21,14 +21,18 @@ #include #include #include +#include #include #include #include #include +#include +#include #include #include "lintel/core/models/SharedMemorySegment.h" #include "lintel/core/platform/Platform.h" +#include "lintel/core/shm/ShmObject.h" #include "lintel/core/shm/ShmSegmentAccessor.h" #include "lintel/features/base/services/SharedMemoryService.h" @@ -432,90 +436,290 @@ namespace shm { } } + /** Payload types and payload constructors of the named-object helpers below. + * + * The named object itself is a ShmObjectHeader; the payload is a separate + * allocation of the segment. ShmObject stores its payload constructor as a + * plain function pointer, so every constructor here is a non-capturing + * function template instantiation and its arguments travel through a + * shared_ptr> resp. SegmentInitArguments. */ + namespace detail { + + /** Constructor arguments of a container payload: its allocator has to come + * from the segment manager of the mapping the object is created in, so the + * constructor reads that manager from the service instead of keeping a + * pointer that a remap would invalidate. */ + struct SegmentInitArguments + { + /** service mapping the segment; non-owning, because the binding that + * runs the payload constructor does not outlive the helper call */ + SharedMemoryService *service = nullptr; + /** segment the object is created in */ + std::shared_ptr segment; + }; + + /** @param arguments constructor arguments of a container payload + * @return segment manager of the mapping the object is created in + * @throws ShmSegmentNotFound if the service does not map the segment */ + inline SegmentManager *mappedSegmentManagerOf(const SegmentInitArguments &arguments) + { + // The payload constructor runs inside ShmObject::resolve(), which holds + // segmentsMutexOf() and the segment's construct semaphore: the mapping + // cannot be replaced while we run, and locking again would deadlock. + SharedMemorySegmentHandle &handle = arguments.service->segmentOf(arguments.segment->getName()); + return handle.m_managedMappedFile->get_segment_manager(); + } + + /** Placement-construct Object from a stored argument tuple. + * @tparam Object the object type + * @tparam Arguments the decayed constructor argument types + * @tparam Indices index sequence selecting the tuple elements + * @param payload raw payload memory + * @param arguments stored constructor arguments + * @param indices selects the tuple elements */ + template + void constructFromTuple(void *payload, + const std::tuple &arguments, + std::index_sequence /*indices*/) + { + ::new (payload) Object(std::get(arguments)...); + } + + /** Payload constructor of a single object with constructor arguments. + * @tparam Object the object type + * @tparam Args the constructor argument types + * @param payload raw payload memory + * @param initArgument the decayed argument tuple of the binding */ + template void initObjectPayload(void *payload, ShmObject::InitArgument initArgument) + { + using Arguments = std::tuple...>; + const auto &arguments = *static_cast(initArgument.value); + constructFromTuple(payload, arguments, std::index_sequence_for{}); + } + + /** Payload constructor of a fixed-size array. Value initialises the array, + * exactly like the former find_or_construct>() call. + * @tparam Object the element type + * @tparam Size the array size + * @param payload raw payload memory + * @param initArgument unused: an array takes no constructor arguments */ + template void initArrayPayload(void *payload, ShmObject::InitArgument initArgument) + { + static_cast(initArgument); + ::new (payload) std::array(); + } + + /** Payload constructor of a shared memory map. + * @tparam Key the map key type + * @tparam Value the map value type + * @param payload raw payload memory + * @param initArgument the SegmentInitArguments of the binding */ + template void initMapPayload(void *payload, ShmObject::InitArgument initArgument) + { + const auto &arguments = *static_cast(initArgument.value); + SegmentAllocator> allocator(mappedSegmentManagerOf(arguments)); + ::new (payload) Map(std::less(), allocator); + } + + /** Payload constructor of a shared memory vector. + * @tparam Object the element type + * @param payload raw payload memory + * @param initArgument the SegmentInitArguments of the binding */ + template void initVectorPayload(void *payload, ShmObject::InitArgument initArgument) + { + const auto &arguments = *static_cast(initArgument.value); + SegmentAllocator allocator(mappedSegmentManagerOf(arguments)); + ::new (payload) Vector(allocator); + } + + /** @param service service the caller keeps alive for the whole call + * @return non-owning shared_ptr to that service, for bindings that end + * with the helper call */ + inline std::shared_ptr serviceBindingOf(SharedMemoryService &service) + { + return std::shared_ptr(&service, [](SharedMemoryService *) {}); + } + + /** @tparam Payload the stored payload type + * @param options registration options of the object + * @return the explicit type id, or the id derived from the payload type */ + template constexpr std::uint32_t payloadTypeIdFor(const ShmObjectOptions &options) noexcept + { + return options.payloadTypeId != 0 ? options.payloadTypeId : shm::payloadTypeIdOf(); + } + + /** @param service service mapping the segment + * @param segment segment the object lives in + * @param name name of the object + * @param layout payload layout + * @param payloadTypeId type id stored in the header + * @param options registration options + * @param init payload constructor + * @param arguments constructor arguments of the payload constructor + * @return the payload of the object in the current mapping */ + template + Payload &resolvePayload(SharedMemoryService &service, + const std::shared_ptr &segment, + const std::string &name, + ShmObject::Layout layout, + std::uint32_t payloadTypeId, + const ShmObjectOptions &options, + ShmObject::InitFunction init, + std::shared_ptr arguments) + { + ShmObject object( + serviceBindingOf(service), segment, name, layout, payloadTypeId, options, init, std::move(arguments)); + return *static_cast(object.resolve()); + } + + }// namespace detail + /** Find or construct a fixed-size array in the given shared memory segment. + * + * The named object is a ShmObjectHeader; the array itself is a separate + * payload allocation of the segment. * @tparam Object the element type * @tparam Size the array size * @param service the shared memory service * @param segment the shared memory segment * @param name the name of the array object + * @param options payload version, persistence class and optional type id * @return reference to the constructed or found array * @throws ShmSegmentNotFound if the segment does not exist */ template std::array &constructArray(SharedMemoryService &service, const std::shared_ptr &segment, - const std::string &name) + const std::string &name, + ShmObjectOptions options = {}) { - std::lock_guard lock(service.segmentsMutexOf()); - auto &handle = service.segmentOf(segment->getName()); - SegmentSemaphoreGuard gate(handle); - return *(handle.m_managedMappedFile->find_or_construct>(name.c_str())()); + using Payload = std::array; + return detail::resolvePayload(service, + segment, + name, + ShmObject::Layout{ sizeof(Payload), alignof(Payload) }, + detail::payloadTypeIdFor(options), + options, + &detail::initArrayPayload, + std::make_shared(detail::SegmentInitArguments{ &service, segment })); } /** Find or construct a map in the given shared memory segment. + * + * The named object is a ShmObjectHeader; the map itself is a separate + * payload allocation of the segment. * @tparam Key the map key type * @tparam Value the map value type * @param service the shared memory service * @param segment the shared memory segment * @param name the name of the map object + * @param options payload version, persistence class and optional type id * @return reference to the constructed or found map * @throws ShmSegmentNotFound if the segment does not exist */ template Map &constructMap(SharedMemoryService &service, const std::shared_ptr &segment, - const std::string &name) + const std::string &name, + ShmObjectOptions options = {}) { - using pairType = std::pair; - using persistentMapAllocator = SegmentAllocator; - std::lock_guard lock(service.segmentsMutexOf()); - auto &handle = service.segmentOf(segment->getName()); - SegmentSemaphoreGuard gate(handle); - persistentMapAllocator allocator(handle.m_managedMappedFile->get_segment_manager()); - Map *map = - handle.m_managedMappedFile->find_or_construct>(name.c_str())(std::less(), allocator); - LOG_TRACE("map {}", map->size()); - return *map; + using Payload = Map; + Payload &map = detail::resolvePayload(service, + segment, + name, + ShmObject::Layout{ sizeof(Payload), alignof(Payload) }, + detail::payloadTypeIdFor(options), + options, + &detail::initMapPayload, + std::make_shared(detail::SegmentInitArguments{ &service, segment })); + LOG_TRACE("map {}", map.size()); + return map; } /** Find or construct a vector in the given shared memory segment. + * + * The named object is a ShmObjectHeader; the vector itself is a separate + * payload allocation of the segment. * @tparam Object the element type * @param service the shared memory service * @param segment the shared memory segment * @param name the name of the vector object + * @param options payload version, persistence class and optional type id * @return reference to the constructed or found vector * @throws ShmSegmentNotFound if the segment does not exist */ template Vector &constructVector(SharedMemoryService &service, const std::shared_ptr &segment, - const std::string &name) + const std::string &name, + ShmObjectOptions options = {}) { - using persistentVectorAllocator = SegmentAllocator; - std::lock_guard lock(service.segmentsMutexOf()); - auto &handle = service.segmentOf(segment->getName()); - SegmentSemaphoreGuard gate(handle); - persistentVectorAllocator allocator(handle.m_managedMappedFile->get_segment_manager()); - return *(handle.m_managedMappedFile->find_or_construct>(name.c_str())(allocator)); + using Payload = Vector; + return detail::resolvePayload(service, + segment, + name, + ShmObject::Layout{ sizeof(Payload), alignof(Payload) }, + detail::payloadTypeIdFor(options), + options, + &detail::initVectorPayload, + std::make_shared(detail::SegmentInitArguments{ &service, segment })); } /** Find or construct a single object in the given shared memory segment. + * + * The named object is a ShmObjectHeader; the object itself is a separate + * payload allocation of the segment. * @tparam Object the object type * @param service the shared memory service * @param segment the shared memory segment * @param name the name of the object + * @param options payload version, persistence class and optional type id * @return reference to the constructed or found object * @throws ShmSegmentNotFound if the segment does not exist */ template Object &constructObject(SharedMemoryService &service, const std::shared_ptr &segment, - const std::string &name) + const std::string &name, + ShmObjectOptions options = {}) { - std::lock_guard lock(service.segmentsMutexOf()); - auto &handle = service.segmentOf(segment->getName()); - SegmentSemaphoreGuard gate(handle); - return *(handle.m_managedMappedFile->find_or_construct(name.c_str())()); + return detail::resolvePayload(service, + segment, + name, + ShmObject::Layout{ sizeof(Object), alignof(Object) }, + detail::payloadTypeIdFor(options), + options, + &detail::initObjectPayload, + std::make_shared>()); + } + + /** Find or construct a single object with constructor arguments and explicit + * registration options in the given shared memory segment. + * @tparam Object the object type + * @tparam Args the constructor argument types + * @param service the shared memory service + * @param segment the shared memory segment + * @param name the name of the object + * @param options payload version, persistence class and optional type id + * @param args arguments forwarded to the constructor + * @return reference to the constructed or found object + * @throws ShmSegmentNotFound if the segment does not exist */ + template + Object &constructObjectWith(SharedMemoryService &service, + const std::shared_ptr &segment, + const std::string &name, + ShmObjectOptions options, + Args &&...args) + { + using Arguments = std::tuple...>; + return detail::resolvePayload(service, + segment, + name, + ShmObject::Layout{ sizeof(Object), alignof(Object) }, + detail::payloadTypeIdFor(options), + options, + &detail::initObjectPayload, + std::make_shared(std::forward(args)...)); } /** Find or construct a single object with constructor arguments in the - * given shared memory segment. + * given shared memory segment, using the default registration options. * @tparam Object the object type * @tparam Args the constructor argument types * @param service the shared memory service @@ -530,10 +734,7 @@ namespace shm { const std::string &name, Args &&...args) { - std::lock_guard lock(service.segmentsMutexOf()); - auto &handle = service.segmentOf(segment->getName()); - SegmentSemaphoreGuard gate(handle); - return *(handle.m_managedMappedFile->find_or_construct(name.c_str())(std::forward(args)...)); + return constructObjectWith(service, segment, name, ShmObjectOptions{}, std::forward(args)...); } /** Find or construct a string in the given shared memory segment. diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 044f1406..8741633a 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -59,6 +59,7 @@ ADD_EXECUTABLE(base_tests features/SharedMemorySegmentComponentTest.cpp features/SharedMemoryServiceTest.cpp features/SharedMemoryRepositoryTest.cpp + features/ShmObjectMigrationTest.cpp features/ShmViewTest.cpp features/SharedMemoryCleanupServiceTest.cpp features/UserDtoTest.cpp diff --git a/tests/features/SharedMemoryServiceTest.cpp b/tests/features/SharedMemoryServiceTest.cpp index 59579aec..f2761b08 100644 --- a/tests/features/SharedMemoryServiceTest.cpp +++ b/tests/features/SharedMemoryServiceTest.cpp @@ -615,7 +615,7 @@ TEST_CASE("SharedMemoryService recreates a segment when the policy allows it") REQUIRE(current.find("stale_object").first == nullptr); const auto marker = current.find("lintel_segment_layout"); REQUIRE(marker.first != nullptr); - REQUIRE(*marker.first == 2); + REQUIRE(*marker.first == 3); current.construct("survivor")(7); } @@ -682,7 +682,7 @@ TEST_CASE("SharedMemoryService lets a registered migrator decide what happens") REQUIRE(seen.path == path); REQUIRE(seen.found == 0);// a segment from before the marker existed - REQUIRE(seen.current == 2); + REQUIRE(seen.current == 3); REQUIRE(seen.value == 42); { diff --git a/tests/features/ShmObjectMigrationTest.cpp b/tests/features/ShmObjectMigrationTest.cpp new file mode 100644 index 00000000..f1fca542 --- /dev/null +++ b/tests/features/ShmObjectMigrationTest.cpp @@ -0,0 +1,259 @@ +#include +#include + +#include "../helpers/PlatformCompat.h" + +#include +#include +#include +#include +#include +#include +#include + +#include "lintel/core/configuration/Configuration.h" +#include "lintel/core/configuration/EnvironmentConfiguration.h" +#include "lintel/core/models/ProcessName.h" +#include "lintel/core/models/SharedMemorySegment.h" +#include "lintel/core/shm/SegmentLayoutPolicy.h" +#include "lintel/core/shm/SharedMemoryRepository.h" +#include "lintel/core/shm/ShmObjectHeader.h" +#include "lintel/core/utils/TypeName.h" +#include "lintel/features/base/configuration/SharedMemorySegmentComponent.h" +#include "lintel/features/base/configuration/SharedMemorySegmentEntry.h" +#include "lintel/features/base/services/SchedulerService.h" +#include "lintel/features/base/services/SharedMemorySegmentManager.h" +#include "lintel/features/base/services/SharedMemoryService.h" +#include "lintel/features/base/services/ShmConstructors.h" + +#include + +#include "../helpers/ScopedEnvironmentVariable.h" + +namespace lintel { +namespace { + + constexpr std::size_t kMigrationSegmentSize = 1u << 20; + + std::filesystem::path uniqueMigrationPath() + { + static unsigned int counter = 0; + return std::filesystem::temp_directory_path() + / ("shm_migration_test_" + std::to_string(::getpid()) + "_" + std::to_string(counter++) + ".bin"); + } + + // macOS caps POSIX semaphore names at 31 characters; the service appends + // "_remap_sem" (10), so the segment name stays short. + std::string uniqueMigrationSegmentName() + { + static unsigned int counter = 0; + return "shm_mig_" + std::to_string(::getpid()) + "_" + std::to_string(counter++); + } + + /** Payload whose first version only used the `value` field. */ + struct VersionedPayload + { + int value = 0; + int reserved = 0; + }; + + /** One real segment that can be closed and reopened, so a payload written by + * an older "build" (version 1) can be read by the current one (version 2). */ + struct MigrationFixture + { + ScopedEnvironmentVariable m_configDirectory{ "CONFIG_DIRECTORY", "/tmp/nonexistent_shm_migration_cfg" }; + std::filesystem::path path; + std::string segmentName; + std::shared_ptr segment; + std::shared_ptr scheduler; + std::shared_ptr segmentManager; + std::shared_ptr service; + + MigrationFixture() : path(uniqueMigrationPath()), segmentName(uniqueMigrationSegmentName()) + { + std::filesystem::remove(path); + segment = std::make_shared(path, segmentName, kMigrationSegmentSize); + auto config = std::make_shared( + std::vector>{}, std::make_shared()); + config->setEntries( + { std::make_shared(type_name(), segment) }); + segmentManager = std::make_shared(config); + scheduler = std::make_shared(std::make_shared("shm-migration-test")); + service = makeService(); + } + + ~MigrationFixture() + { + service.reset(); + scheduler.reset(); + SegmentLockOwner::removeOf(segmentName); + std::filesystem::remove(path); + boost::interprocess::named_semaphore::remove(shm::remapSemaphoreNameOf(segmentName).c_str()); + shm::ShmPayloadMigrations::reset(); + shm::SegmentLayoutPolicy::reset(); + } + + [[nodiscard]] std::shared_ptr makeService() const + { + return std::make_shared( + segmentManager, scheduler, std::make_shared("shm-migration-test")); + } + + /** Close and reopen the segment, like a process restart. */ + void restart() + { + service.reset(); + service = makeService(); + } + + /** Rewrite the segment marker to a value older than the current layout. */ + void stampLayoutVersion(std::uint64_t version) const + { + const std::string filePath = path.string(); + boost::interprocess::managed_mapped_file mapping(boost::interprocess::open_only, filePath.c_str()); + auto found = mapping.find("lintel_segment_layout"); + REQUIRE(found.first != nullptr); + *found.first = version; + } + }; + + [[nodiscard]] shm::ShmObjectOptions optionsOf(std::uint16_t version, + shm::ShmPersistence persistence = shm::ShmPersistence::Persistent) + { + return shm::ShmObjectOptions{ .payloadVersion = version, .persistence = persistence }; + } + +}// namespace + +TEST_CASE("ShmObjectHeader has a stable on-disk layout") +{ + REQUIRE(sizeof(shm::ShmObjectHeader) == 64); + REQUIRE(alignof(shm::ShmObjectHeader) == 16); + REQUIRE(shm::ShmObjectHeader::kMagic == 0x4c53484fU); + REQUIRE(shm::ShmObjectHeader{}.isCompatible()); + REQUIRE_FALSE(shm::ShmObjectHeader{}.isReady()); +} + +TEST_CASE("a typed repository migrates its payload through a registered chain") +{ + MigrationFixture fixture; + const shm::ShmObjectOptions versionOne = optionsOf(1); + { + shm::SharedMemoryRepository repository( + fixture.service, fixture.segment, "versioned", versionOne, VersionedPayload{ 42, 0 }); + repository.get().value = 42; + REQUIRE(repository.getPayloadVersion() == 1); + } + fixture.restart(); + + shm::registerPayloadMigration( + 1, 2, [](const void *source, std::size_t /*sourceBytes*/, void *target) { + const auto *oldValue = static_cast(source); + ::new (target) VersionedPayload{ oldValue->value * 2, 7 }; + }); + + shm::SharedMemoryRepository migrated( + fixture.service, fixture.segment, "versioned", optionsOf(2), VersionedPayload{}); + REQUIRE(migrated.get().value == 84); + REQUIRE(migrated.get().reserved == 7); + REQUIRE(migrated.getPayloadVersion() == 2); +} + +TEST_CASE("a persistent payload without a migration aborts the start") +{ + MigrationFixture fixture; + { + shm::SharedMemoryRepository repository( + fixture.service, fixture.segment, "persistent", optionsOf(1), VersionedPayload{ 42, 0 }); + repository.get().value = 42; + } + fixture.restart(); + shm::ShmPayloadMigrations::reset(); + + REQUIRE_THROWS_AS((shm::SharedMemoryRepository( + fixture.service, fixture.segment, "persistent", optionsOf(2), VersionedPayload{})), + std::runtime_error); +} + +TEST_CASE("a volatile payload is recreated when its version is unknown") +{ + MigrationFixture fixture; + { + shm::SharedMemoryRepository repository(fixture.service, + fixture.segment, + "volatile", + optionsOf(1, shm::ShmPersistence::Volatile), + VersionedPayload{ 42, 0 }); + repository.get().value = 42; + } + fixture.restart(); + shm::ShmPayloadMigrations::reset(); + + shm::SharedMemoryRepository recreated(fixture.service, + fixture.segment, + "volatile", + optionsOf(2, shm::ShmPersistence::Volatile), + VersionedPayload{ 7, 0 }); + REQUIRE(recreated.get().value == 7); + REQUIRE(recreated.getPayloadVersion() == 2); + REQUIRE(recreated.getPersistence() == shm::ShmPersistence::Volatile); +} + +TEST_CASE("a derived payload is recreated when its version is unknown") +{ + MigrationFixture fixture; + { + shm::SharedMemoryRepository repository(fixture.service, + fixture.segment, + "derived", + optionsOf(1, shm::ShmPersistence::Derived), + VersionedPayload{ 42, 0 }); + repository.get().value = 42; + } + fixture.restart(); + shm::ShmPayloadMigrations::reset(); + + shm::SharedMemoryRepository recreated( + fixture.service, fixture.segment, "derived", optionsOf(2, shm::ShmPersistence::Derived), VersionedPayload{ 11, 0 }); + REQUIRE(recreated.get().value == 11); + REQUIRE(recreated.getPersistence() == shm::ShmPersistence::Derived); +} + +TEST_CASE("a segment with only volatile objects is recreated automatically") +{ + MigrationFixture fixture; + { + shm::SharedMemoryRepository repository(fixture.service, + fixture.segment, + "auto-volatile", + optionsOf(1, shm::ShmPersistence::Volatile), + VersionedPayload{ 42, 0 }); + repository.get().value = 42; + } + fixture.service.reset(); + fixture.stampLayoutVersion(2); + + REQUIRE_NOTHROW(fixture.service = fixture.makeService()); + const std::string filePath = fixture.path.string(); + boost::interprocess::managed_mapped_file current(boost::interprocess::open_only, filePath.c_str()); + REQUIRE(current.find("auto-volatile").first == nullptr); + auto marker = current.find("lintel_segment_layout"); + REQUIRE(marker.first != nullptr); + REQUIRE(*marker.first == 3); +} + +TEST_CASE("a segment with a persistent object still refuses an automatic discard") +{ + MigrationFixture fixture; + { + shm::SharedMemoryRepository repository( + fixture.service, fixture.segment, "auto-persistent", optionsOf(1), VersionedPayload{ 42, 0 }); + repository.get().value = 42; + } + fixture.service.reset(); + fixture.stampLayoutVersion(2); + + REQUIRE_THROWS_AS((fixture.service = fixture.makeService()), std::runtime_error); +} + +}// namespace lintel diff --git a/tests/plugins/SharedMemoryBootstrapPluginTest.cpp b/tests/plugins/SharedMemoryBootstrapPluginTest.cpp index dbebbef8..cbcf6ccd 100644 --- a/tests/plugins/SharedMemoryBootstrapPluginTest.cpp +++ b/tests/plugins/SharedMemoryBootstrapPluginTest.cpp @@ -2,6 +2,7 @@ #include #include #include +#include #include #include @@ -18,6 +19,8 @@ #include "lintel/core/persistence/DatabaseConnectionConfigurations.h" #include "lintel/core/persistence/Statement.h" #include "lintel/core/shm/BoostSegmentAllocator.h" +#include "lintel/core/shm/ShmObjectHeader.h" +#include "lintel/core/shm/ShmSegmentAccessor.h" #include "lintel/core/utils/TypeName.h" #include "lintel/features/base/configuration/EventBusComponent.h" #include "lintel/features/base/configuration/EventBusEntry.h" @@ -120,7 +123,14 @@ namespace { { Segment mapping(boost::interprocess::open_only, path.string().c_str()); BoostSegmentAllocator allocator(*mapping.get_segment_manager()); - mapping.find_or_construct(name)(name, config, ShmSegmentAccessor(allocator)); + const ShmSegmentAccessor accessor(allocator); + void *payload = accessor.findOrConstructPayload(name, + sizeof(EventBus), + alignof(EventBus), + shm::payloadTypeIdOf(), + shm::ShmObjectOptions{ .payloadVersion = 1, .persistence = shm::ShmPersistence::Volatile }); + REQUIRE(payload != nullptr); + ::new (payload) EventBus(name, config, accessor); } std::shared_ptr sqliteEntry(const std::filesystem::path &dbPath) From c13fe62846e6c5f5842aea2bd0a363ae9d646d99 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dominik=20Kr=C3=BCmpelmann?= Date: Thu, 22 Oct 2026 20:06:43 +0200 Subject: [PATCH 2/4] fix(shm): never discard a reclassified persistent payload An object written as Volatile or Derived may be authoritative for a build that now registers it as Persistent. Require both the stored and the requested persistence class to allow discarding before recreating a payload. --- src/features/base/services/ShmObject.cpp | 18 +++++++++++++++--- .../lintel/core/shm/ShmSegmentAccessor.h | 18 +++++++++++++----- 2 files changed, 28 insertions(+), 8 deletions(-) diff --git a/src/features/base/services/ShmObject.cpp b/src/features/base/services/ShmObject.cpp index a000be95..a494ccb9 100644 --- a/src/features/base/services/ShmObject.cpp +++ b/src/features/base/services/ShmObject.cpp @@ -141,6 +141,18 @@ namespace shm { header.state = ShmObjectHeader::kStateReady; } + /** @param header existing object header + * @param options registration options of this build + * @return true when both the stored and the requested persistence class + * allow discarding the payload */ + bool mayDiscardPayload(const ShmObjectHeader &header, const ShmObjectOptions &options) noexcept + { + // An object the old build wrote as Volatile may still be authoritative for + // a build that now registers it as Persistent - never discard in that case. + return header.persistenceClass() != ShmPersistence::Persistent + && options.persistence != ShmPersistence::Persistent; + } + /** Why an existing object cannot be used by this build. */ std::string incompatibilityOf(const ShmObjectHeader &header, const ShmObject::Layout &layout, std::uint32_t typeId) { @@ -215,7 +227,7 @@ namespace shm { if (header->payloadVersion == options.payloadVersion) { if (header->payloadTypeId != typeId || header->payloadBytes != layout.size) { const std::string reason = incompatibilityOf(*header, layout, typeId); - if (header->persistenceClass() == ShmPersistence::Persistent) { + if (!mayDiscardPayload(*header, options)) { throw std::invalid_argument( "shared memory object '" + name + "' of segment '" + segment->getName() + "' is incompatible: " + reason); } @@ -268,9 +280,9 @@ namespace shm { return header; } - if (header->persistenceClass() == ShmPersistence::Persistent) { + if (!mayDiscardPayload(*header, options)) { throw std::runtime_error("shared memory object '" + name + "' of segment '" + segment->getName() - + "' is persistent and was written with payload version " + + "' may not be recreated and was written with payload version " + std::to_string(header->payloadVersion) + " (current " + std::to_string(options.payloadVersion) + "); register a payload migration or remove the segment"); diff --git a/src/include/lintel/core/shm/ShmSegmentAccessor.h b/src/include/lintel/core/shm/ShmSegmentAccessor.h index 5d7a1fa0..8f62f149 100644 --- a/src/include/lintel/core/shm/ShmSegmentAccessor.h +++ b/src/include/lintel/core/shm/ShmSegmentAccessor.h @@ -196,8 +196,9 @@ class ShmSegmentAccessor * @param options payload version and persistence class * @return pointer to the payload, or nullptr when the allocator does not * support named objects - * @throws std::invalid_argument on an invalid request, or when a persistent - * object stores a payload this build cannot use + * @throws std::invalid_argument on an invalid request, or when an + * incompatible payload may not be recreated because of its + * persistence class * @throws std::runtime_error when the named object was written by an * incompatible build or its initialisation does not finish in time */ @@ -338,7 +339,8 @@ class ShmSegmentAccessor * @param alignment payload alignment this build expects * @param typeId payload type id this build expects * @param options version and persistence class this build registers - * @throws std::invalid_argument when a persistent object is incompatible */ + * @throws std::invalid_argument when the persistence class does not allow + * recreating an incompatible payload */ void reconcilePayload(shm::ShmObjectHeader &header, const char *name, std::size_t bytes, @@ -350,13 +352,19 @@ class ShmSegmentAccessor && header.payloadBytes == static_cast(bytes) && header.payloadAlignment == static_cast(alignment); if (samePayload) { return; } - if (header.persistenceClass() == shm::ShmPersistence::Persistent) { + // Discard only when *both* the stored and the requested class allow it: an + // object the old build wrote as Volatile may still be authoritative for a + // build that registers it as Persistent. + const bool discardable = header.persistenceClass() != shm::ShmPersistence::Persistent + && options.persistence != shm::ShmPersistence::Persistent; + if (!discardable) { throw std::invalid_argument("shared memory object '" + std::string(name) + "' is incompatible: it stores payload " "type " + std::to_string(header.payloadTypeId) + " version " + std::to_string(header.payloadVersion) + " with " + std::to_string(header.payloadBytes) + " bytes, but this build expects type " + std::to_string(typeId) + " version " + std::to_string(options.payloadVersion) - + " with " + std::to_string(bytes) + " bytes"); + + " with " + std::to_string(bytes) + " bytes, and its persistence class does not allow " + "recreating it"); } // Volatile or Derived: recreate the payload zeroed. No destructor of the // old payload runs - the old type may not exist in this build. From e9fedb40a2bfcdd178743d5ba0d320856b1d9dd3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dominik=20Kr=C3=BCmpelmann?= Date: Thu, 22 Oct 2026 22:08:02 +0200 Subject: [PATCH 3/4] fix(shm): satisfy the clang-tidy gate for the payload ABI The clang-tidy job analyses changed files with an explicit --checks list, which re-enables bugprone-easily-swappable-parameters and reports the new shared-memory API. Fix the code instead of loosening the gate: - bundle the version pair, the two byte counts and the type id into ShmPayloadVersionRange, ShmPayloadMigrationInput and ShmPayloadMigrationRegistration, and the payload size/alignment into ShmPayloadLayout (ShmObject::Layout stays as an alias), - give SharedMemoryType a std::uint8_t base type, - log the swallowed deallocate errors instead of leaving empty catch blocks, and guard ~SegmentSemaphoreGuard against post() throwing, - name the temporary named_semaphore whose constructor creates the lock. clang-tidy: 0 findings in 17 analysed files; unit tests 1318/1318. --- CHANGELOG.md | 3 +- docs/shm-migration-strategy.md | 9 +- docs/shm-payload-abi-plan.md | 9 +- src/features/base/services/ShmObject.cpp | 68 +++++------ src/include/lintel/core/shm/ShmObject.h | 11 +- src/include/lintel/core/shm/ShmObjectHeader.h | 107 ++++++++++++------ .../lintel/core/shm/ShmSegmentAccessor.h | 37 +++--- .../repositories/SharedMemoryRepository.h | 3 +- .../features/base/services/ShmConstructors.h | 26 +++-- tests/features/ShmObjectMigrationTest.cpp | 8 +- 10 files changed, 170 insertions(+), 111 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index cf6471f1..abe2f8c2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,7 +12,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 named object in a segment now starts with a 64-byte `lintel::shm::ShmObjectHeader` (`headerVersion`, `payloadVersion`, type id, persistence class); the payload is a separate allocation addressed by a relative offset, so a change to the library's bookkeeping no longer - moves the payload. `shm::registerPayloadMigration(n, n+1, …)` chains per-type migrations, + moves the payload. `shm::registerPayloadMigration({ .fromVersion = n, .toVersion = n + 1, .apply = … })` + chains per-type migrations, and `shm::ShmObjectOptions { payloadVersion, persistence, payloadTypeId }` is accepted by `shm::SharedMemoryRepository`, the `shm::construct*` helpers and the JSON mirror repositories. `Volatile`/`Derived` objects may be recreated without operator input, `Persistent` (the default) diff --git a/docs/shm-migration-strategy.md b/docs/shm-migration-strategy.md index 56e23c73..300c8a64 100644 --- a/docs/shm-migration-strategy.md +++ b/docs/shm-migration-strategy.md @@ -118,10 +118,13 @@ EventBus und MessageQueue als `Volatile`. ### 5.2 Payload migrieren ```cpp -lintel::shm::registerPayloadMigration(1, 2, - [](const void *oldBytes, std::size_t oldSize, void *newPayload) { +lintel::shm::registerPayloadMigration({ + .fromVersion = 1, + .toVersion = 2, + .apply = [](const void *oldBytes, std::size_t oldSize, void *newPayload) { // alten Stand lesen und den neuen Typ an newPayload konstruieren - }); + }, +}); ``` Die Registry verkettet `1 → 2 → 3` automatisch. Der alte Payload bleibt diff --git a/docs/shm-payload-abi-plan.md b/docs/shm-payload-abi-plan.md index b5cf20ca..bb8478c1 100644 --- a/docs/shm-payload-abi-plan.md +++ b/docs/shm-payload-abi-plan.md @@ -90,10 +90,13 @@ lintel::shm::ShmObjectOptions options{ .persistence = lintel::shm::ShmPersistence::Persistent, }; -lintel::shm::registerPayloadMigration(1, 2, - [](const void *oldBytes, std::size_t oldSize, void *newPayload) { +lintel::shm::registerPayloadMigration({ + .fromVersion = 1, + .toVersion = 2, + .apply = [](const void *oldBytes, std::size_t oldSize, void *newPayload) { // alten Stand lesen, neuen Stand an newPayload konstruieren - }); + }, +}); lintel::shm::SharedMemoryRepository repository( service, segment, "my-dto", options, MyDto{}); diff --git a/src/features/base/services/ShmObject.cpp b/src/features/base/services/ShmObject.cpp index a494ccb9..8298b176 100644 --- a/src/features/base/services/ShmObject.cpp +++ b/src/features/base/services/ShmObject.cpp @@ -90,10 +90,11 @@ namespace shm { void *payload = payloadAddressOf(&header); try { segment.deallocate(payload); - } catch (...) { + } catch (const std::exception &error) { // The segment is being torn down or the block was already released; // leaking is the only safe option here and the segment owns the bytes // anyway. + LOG_DEBUG("cannot release a shared memory payload: {}", error.what()); } header.payloadOffset = 0; header.payloadBytes = 0; @@ -248,18 +249,20 @@ namespace shm { // Older payload. A registered chain wins; otherwise the persistence class // decides whether the object may be recreated. - if (ShmPayloadMigrations::canMigrate(typeId, header->payloadVersion, options.payloadVersion)) { + if (ShmPayloadMigrations::canMigrate( + typeId, { .fromVersion = header->payloadVersion, .toVersion = options.payloadVersion })) { LOG_WARN("migrating shared memory object '{}' of segment '{}' from payload version {} to {}", name, segment->getName(), header->payloadVersion, options.payloadVersion); - const std::vector migrated = ShmPayloadMigrations::migrate(typeId, - header->payloadVersion, - options.payloadVersion, - payloadAddressOf(header), - static_cast(header->payloadBytes), - layout.size); + const std::vector migrated = ShmPayloadMigrations::migrate({ + .typeId = typeId, + .versions = { .fromVersion = header->payloadVersion, .toVersion = options.payloadVersion }, + .source = payloadAddressOf(header), + .sourceBytes = static_cast(header->payloadBytes), + .targetBytes = layout.size, + }); void *payload = allocatePayload(accessor, layout); std::memcpy(payload, migrated.data(), layout.size); // Publish the new payload first, then release the old block: a process @@ -274,8 +277,9 @@ namespace shm { header->state = ShmObjectHeader::kStateReady; try { accessor.deallocate(oldPayload); - } catch (...) { + } catch (const std::exception &error) { // See releasePayload(): the segment owns the bytes. + LOG_DEBUG("cannot release the previous shared memory payload: {}", error.what()); } return header; } @@ -319,12 +323,12 @@ namespace shm { } } - bool ShmPayloadMigrations::canMigrate(std::uint32_t typeId, std::uint16_t fromVersion, std::uint16_t toVersion) + bool ShmPayloadMigrations::canMigrate(std::uint32_t typeId, ShmPayloadVersionRange versions) { - if (fromVersion >= toVersion) { return fromVersion == toVersion; } + if (versions.fromVersion >= versions.toVersion) { return versions.fromVersion == versions.toVersion; } const std::vector steps = stepsOf(typeId); - std::uint16_t version = fromVersion; - while (version < toVersion) { + std::uint16_t version = versions.fromVersion; + while (version < versions.toVersion) { const ShmPayloadMigrationStep *step = stepFrom(steps, version); if (step == nullptr || step->toVersion <= version) { return false; } version = step->toVersion; @@ -332,35 +336,35 @@ namespace shm { return true; } - std::vector ShmPayloadMigrations::migrate(std::uint32_t typeId, - std::uint16_t fromVersion, - std::uint16_t toVersion, - const void *source, - std::size_t sourceBytes, - std::size_t finalBytes) + std::vector ShmPayloadMigrations::migrate(const ShmPayloadMigrationInput &input) { - if (fromVersion >= toVersion) { - std::vector unchanged(sourceBytes); - if (sourceBytes > 0 && source != nullptr) { std::memcpy(unchanged.data(), source, sourceBytes); } + if (input.versions.fromVersion >= input.versions.toVersion) { + std::vector unchanged(input.sourceBytes); + if (input.sourceBytes > 0 && input.source != nullptr) { + std::memcpy(unchanged.data(), input.source, input.sourceBytes); + } return unchanged; } - const std::vector steps = stepsOf(typeId); - std::vector current(sourceBytes); - if (sourceBytes > 0 && source != nullptr) { std::memcpy(current.data(), source, sourceBytes); } - std::uint16_t version = fromVersion; - while (version < toVersion) { + const std::vector steps = stepsOf(input.typeId); + std::vector current(input.sourceBytes); + if (input.sourceBytes > 0 && input.source != nullptr) { + std::memcpy(current.data(), input.source, input.sourceBytes); + } + std::uint16_t version = input.versions.fromVersion; + while (version < input.versions.toVersion) { const ShmPayloadMigrationStep *step = stepFrom(steps, version); if (step == nullptr || step->toVersion <= version) { throw std::runtime_error( - "no payload migration from version " + std::to_string(version) + " for type " + std::to_string(typeId)); + "no payload migration from version " + std::to_string(version) + " for type " + std::to_string(input.typeId)); } if (step->sourceBytes != 0 && step->sourceBytes != current.size()) { throw std::runtime_error("payload migration from version " + std::to_string(version) + " expects " + std::to_string(step->sourceBytes) + " bytes but got " + std::to_string(current.size())); } - const std::size_t targetBytes = - step->targetBytes != 0 ? step->targetBytes : (step->toVersion == toVersion ? finalBytes : 0); + const std::size_t targetBytes = step->targetBytes != 0 + ? step->targetBytes + : (step->toVersion == input.versions.toVersion ? input.targetBytes : 0); if (targetBytes == 0) { throw std::runtime_error( "payload migration from version " + std::to_string(version) + " does not name an intermediate size"); @@ -370,9 +374,9 @@ namespace shm { current = std::move(next); version = step->toVersion; } - if (current.size() != finalBytes) { + if (current.size() != input.targetBytes) { throw std::runtime_error("payload migration ended with " + std::to_string(current.size()) + " bytes but " - + std::to_string(finalBytes) + " were expected"); + + std::to_string(input.targetBytes) + " were expected"); } return current; } diff --git a/src/include/lintel/core/shm/ShmObject.h b/src/include/lintel/core/shm/ShmObject.h index d9648cd3..fc80063a 100644 --- a/src/include/lintel/core/shm/ShmObject.h +++ b/src/include/lintel/core/shm/ShmObject.h @@ -38,14 +38,9 @@ namespace shm { /** Highest payload alignment the relative addressing supports. */ static constexpr std::size_t kMaxAlignment = 4096; - /** Payload size and alignment. */ - struct Layout - { - /** payload size in bytes (> 0) */ - std::size_t size = 0; - /** payload alignment in bytes (power of two, <= kMaxAlignment) */ - std::size_t alignment = 1; - }; + /** Payload size and alignment (a single aggregate so the two values cannot + * be swapped at a call site). */ + using Layout = ShmPayloadLayout; /** Type-erased pointer to the payload constructor's arguments. */ struct InitArgument diff --git a/src/include/lintel/core/shm/ShmObjectHeader.h b/src/include/lintel/core/shm/ShmObjectHeader.h index 606a2516..eb82cf6a 100644 --- a/src/include/lintel/core/shm/ShmObjectHeader.h +++ b/src/include/lintel/core/shm/ShmObjectHeader.h @@ -43,6 +43,44 @@ namespace shm { std::uint32_t payloadTypeId = 0; }; + /** Size and alignment of one payload allocation. + * + * A single aggregate instead of two adjacent sizes: the two values are easy + * to swap at a call site (`{ sizeof(T), alignof(T) }`), and a mistake there + * only shows up as a misaligned payload much later. */ + struct ShmPayloadLayout + { + /** payload size in bytes */ + std::size_t size = 0; + /** payload alignment in bytes (power of two) */ + std::size_t alignment = 1; + }; + + /** Payload version stored in the segment and the version this build writes. */ + struct ShmPayloadVersionRange + { + /** version written by the build that created the stored payload */ + std::uint16_t fromVersion = 0; + /** version this build expects and writes */ + std::uint16_t toVersion = 0; + }; + + /** Input of one payload migration: the stored representation and the size + * this build expects. */ + struct ShmPayloadMigrationInput + { + /** id of the payload type */ + std::uint32_t typeId = 0; + /** version currently stored and version this build writes */ + ShmPayloadVersionRange versions; + /** old payload bytes; nullptr is allowed when sourceBytes is 0 */ + const void *source = nullptr; + /** size of the old payload in bytes */ + std::size_t sourceBytes = 0; + /** size the migrated payload must have (sizeof(T)) */ + std::size_t targetBytes = 0; + }; + /** * Stable header of every named object in a shared memory segment. * @@ -171,31 +209,44 @@ namespace shm { static void registerStep(std::uint32_t typeId, ShmPayloadMigrationStep step); /** @return true when a complete chain from fromVersion to toVersion exists */ - [[nodiscard]] static bool canMigrate(std::uint32_t typeId, std::uint16_t fromVersion, std::uint16_t toVersion); + [[nodiscard]] static bool canMigrate(std::uint32_t typeId, ShmPayloadVersionRange versions); /** * Run the chain and return the migrated payload bytes. * - * @param typeId id of the payload type - * @param fromVersion version stored in the segment - * @param toVersion version of this build - * @param source old payload bytes - * @param sourceBytes size of the old payload - * @param finalBytes size the final payload must have (sizeof(T)) - * @return the migrated bytes, exactly finalBytes long + * @param input type id, version range, old payload bytes and target size + * @return the migrated bytes, exactly input.targetBytes long * @throws std::runtime_error when no chain exists or a step is broken */ - [[nodiscard]] static std::vector migrate(std::uint32_t typeId, - std::uint16_t fromVersion, - std::uint16_t toVersion, - const void *source, - std::size_t sourceBytes, - std::size_t finalBytes); + [[nodiscard]] static std::vector migrate(const ShmPayloadMigrationInput &input); /** Drop every registered step. Intended for tests. */ static void reset() noexcept; }; + /** + * Registration of one migration step for T. + * + * A single aggregate instead of six positional parameters: the version pair + * and the two byte counts are easy to swap, and a swapped pair would only + * surface as a corrupted payload long after registration. + */ + struct ShmPayloadMigrationRegistration + { + /** version stored in the segment */ + std::uint16_t fromVersion = 0; + /** version this build writes */ + std::uint16_t toVersion = 0; + /** places a T into the target buffer */ + std::function apply; + /** type id; 0 derives it from T */ + std::uint32_t typeId = 0; + /** expected size of the old payload; 0 uses the header */ + std::size_t sourceBytes = 0; + /** size after this step; 0 uses sizeof(T) */ + std::size_t targetBytes = 0; + }; + /** * Register a v(n) → v(n+1) migration for T. * @@ -204,31 +255,21 @@ namespace shm { * into the target. * * @tparam T payload type - * @param fromVersion version stored in the segment - * @param toVersion version this build writes - * @param apply migration callback - * @param typeId type id; defaults to payloadTypeIdOf() - * @param sourceBytes expected size of the old payload (0 = use the header) - * @param targetBytes size after this step (defaults to sizeof(T)) + * @param registration versions, callback and optional sizes/type id */ - template - void registerPayloadMigration(std::uint16_t fromVersion, - std::uint16_t toVersion, - std::function apply, - std::uint32_t typeId = payloadTypeIdOf(), - std::size_t sourceBytes = 0, - std::size_t targetBytes = sizeof(T)) + template void registerPayloadMigration(ShmPayloadMigrationRegistration registration) { ShmPayloadMigrationStep step; - step.fromVersion = fromVersion; - step.toVersion = toVersion; - step.sourceBytes = sourceBytes; - step.targetBytes = targetBytes; - step.apply = [callback = std::move(apply)]( + step.fromVersion = registration.fromVersion; + step.toVersion = registration.toVersion; + step.sourceBytes = registration.sourceBytes; + step.targetBytes = registration.targetBytes != 0 ? registration.targetBytes : sizeof(T); + step.apply = [callback = std::move(registration.apply)]( const void *source, std::size_t sourceBytes, void *target, std::size_t /*targetBytes*/) { if (callback) { callback(source, sourceBytes, target); } }; - ShmPayloadMigrations::registerStep(typeId, std::move(step)); + ShmPayloadMigrations::registerStep( + registration.typeId != 0 ? registration.typeId : payloadTypeIdOf(), std::move(step)); } }// namespace shm diff --git a/src/include/lintel/core/shm/ShmSegmentAccessor.h b/src/include/lintel/core/shm/ShmSegmentAccessor.h index 8f62f149..831c09d7 100644 --- a/src/include/lintel/core/shm/ShmSegmentAccessor.h +++ b/src/include/lintel/core/shm/ShmSegmentAccessor.h @@ -15,6 +15,7 @@ #include #include +#include "lintel/core/services/LoggerService.h" #include "lintel/core/shm/ShmObjectHeader.h" namespace lintel { @@ -208,7 +209,7 @@ class ShmSegmentAccessor std::uint32_t typeId, shm::ShmObjectOptions options) const { - validatePayloadRequest(name, bytes, alignment); + validatePayloadRequest(name, shm::ShmPayloadLayout{ bytes, alignment }); auto *header = static_cast(findOrConstructArray(name, sizeof(shm::ShmObjectHeader))); if (header == nullptr) { return nullptr; } @@ -221,7 +222,7 @@ class ShmSegmentAccessor if (state.compare_exchange_strong( expected, shm::ShmObjectHeader::kStateConstructing, std::memory_order_acq_rel, std::memory_order_acquire)) { try { - initialisePayload(*header, bytes, alignment, typeId, options); + initialisePayload(*header, shm::ShmPayloadLayout{ bytes, alignment }, typeId, options); } catch (...) { // Release the claim: a header stuck in "constructing" would make // every later open wait for a payload that will never be published. @@ -268,23 +269,22 @@ class ShmSegmentAccessor + " characters"); } - /** @param name name of the payload object - * @param bytes requested payload size - * @param alignment requested payload alignment + /** @param name name of the payload object + * @param layout requested payload size and alignment * @throws std::invalid_argument on a request the segment cannot serve */ - static void validatePayloadRequest(const char *name, std::size_t bytes, std::size_t alignment) + static void validatePayloadRequest(const char *name, shm::ShmPayloadLayout layout) { if (name == nullptr || name[0] == '\0') { throw std::invalid_argument("a named shared memory payload needs a name"); } - if (bytes == 0) { + if (layout.size == 0) { throw std::invalid_argument("the payload of shared memory object '" + std::string(name) + "' must not be empty"); } - if (alignment == 0 || (alignment & (alignment - 1u)) != 0) { + if (layout.alignment == 0 || (layout.alignment & (layout.alignment - 1u)) != 0) { throw std::invalid_argument( "the payload alignment of shared memory object '" + std::string(name) + "' must be a power of two"); } - if (alignment > alignof(std::max_align_t)) { + if (layout.alignment > alignof(std::max_align_t)) { // The segment allocator aligns to max_align_t, so a larger alignment would // make the payload pointer differ from the allocated block start - and a // later deallocate() on the payload pointer would then be invalid. @@ -368,7 +368,7 @@ class ShmSegmentAccessor } // Volatile or Derived: recreate the payload zeroed. No destructor of the // old payload runs - the old type may not exist in this build. - initialisePayload(header, bytes, alignment, typeId, options); + initialisePayload(header, shm::ShmPayloadLayout{ bytes, alignment }, typeId, options); } /** Release the payload block a header describes, if any, and put the header @@ -384,9 +384,10 @@ class ShmSegmentAccessor void *payload = shm::payloadAddressOf(&header); try { deallocate(payload); - } catch (...) { + } catch (const std::exception &error) { // The segment is being torn down or the block was already released; // leaking is the only safe option and the segment owns the bytes. + LOG_DEBUG("cannot release a shared memory payload: {}", error.what()); } } // Back to the blank state of a freshly created header: a failed @@ -403,18 +404,18 @@ class ShmSegmentAccessor /** Allocate the payload, publish it into the header and mark the header ready. * The caller claimed the header with state == kStateConstructing. * - * @param header header of the object - * @param bytes payload size in bytes - * @param alignment payload alignment in bytes - * @param typeId payload type id - * @param options version and persistence class to record + * @param header header of the object + * @param layout payload size and alignment in bytes + * @param typeId payload type id + * @param options version and persistence class to record * @throws std::bad_alloc when the segment cannot serve the allocation */ void initialisePayload(shm::ShmObjectHeader &header, - std::size_t bytes, - std::size_t alignment, + shm::ShmPayloadLayout layout, std::uint32_t typeId, const shm::ShmObjectOptions &options) const { + const std::size_t bytes = layout.size; + const std::size_t alignment = layout.alignment; releasePayload(header); const std::size_t allocationBytes = bytes + alignment - 1u; void *raw = allocate(allocationBytes); diff --git a/src/include/lintel/features/base/repositories/SharedMemoryRepository.h b/src/include/lintel/features/base/repositories/SharedMemoryRepository.h index b321775c..249f8b1f 100644 --- a/src/include/lintel/features/base/repositories/SharedMemoryRepository.h +++ b/src/include/lintel/features/base/repositories/SharedMemoryRepository.h @@ -3,6 +3,7 @@ #include +#include #include #include @@ -16,7 +17,7 @@ namespace lintel { /** Types of shared memory data structures. */ -enum SharedMemoryType { Map, Vector, Array, Object }; +enum SharedMemoryType : std::uint8_t { Map, Vector, Array, Object }; /** * Base class for shared-memory-backed repositories with JSON serialization. diff --git a/src/include/lintel/features/base/services/ShmConstructors.h b/src/include/lintel/features/base/services/ShmConstructors.h index 12cc76b1..c339fb0b 100644 --- a/src/include/lintel/features/base/services/ShmConstructors.h +++ b/src/include/lintel/features/base/services/ShmConstructors.h @@ -257,17 +257,23 @@ namespace shm { acquireOrThrow(timeout); } - ~SegmentSemaphoreGuard() + ~SegmentSemaphoreGuard() noexcept { // A timed-out guard never took the semaphore (acquireOrThrow cleared it): // it must not post on destruction - that would hand the lock to the next // waiter even though we never owned it. if (m_semaphore == nullptr) { return; } - // clearIf, not clear: the next holder writes its own record as soon as it - // owns the lock, so an unconditional clear could wipe a live holder and - // make the next recovery reclaim a lock that is in use. - if (m_owner != nullptr) { m_owner->clearIf(m_holderIdentity); } - m_semaphore->post(); + // post() reports a broken semaphore through an exception, but a destructor + // must not let it escape: log it and keep the segment usable. + try { + // clearIf, not clear: the next holder writes its own record as soon as it + // owns the lock, so an unconditional clear could wipe a live holder and + // make the next recovery reclaim a lock that is in use. + if (m_owner != nullptr) { m_owner->clearIf(m_holderIdentity); } + m_semaphore->post(); + } catch (const std::exception &error) { + LOG_ERROR("cannot release the construct lock of a shared memory segment: {}", error.what()); + } } SegmentSemaphoreGuard(const SegmentSemaphoreGuard &) = delete; @@ -413,8 +419,12 @@ namespace shm { semaphoreRemoved = boost::interprocess::named_semaphore::remove(semaphoreName.c_str()); // open_or_create: another process may have recreated the lock in between, - // in which case we open it instead of failing on a duplicate. - boost::interprocess::named_semaphore(boost::interprocess::open_or_create, semaphoreName.c_str(), 1u); + // in which case we open it instead of failing on a duplicate. The handle is + // only needed for its side effect - it creates the named semaphore - so it + // stays alive until the end of this scope and is then closed. + const boost::interprocess::named_semaphore recreated( + boost::interprocess::open_or_create, semaphoreName.c_str(), 1u); + static_cast(recreated); // clearIf, not clear: a process that took the lock in the meantime has // recorded itself and must keep that record. owner.clearIf(holder); diff --git a/tests/features/ShmObjectMigrationTest.cpp b/tests/features/ShmObjectMigrationTest.cpp index f1fca542..0cd87d23 100644 --- a/tests/features/ShmObjectMigrationTest.cpp +++ b/tests/features/ShmObjectMigrationTest.cpp @@ -147,10 +147,10 @@ TEST_CASE("a typed repository migrates its payload through a registered chain") fixture.restart(); shm::registerPayloadMigration( - 1, 2, [](const void *source, std::size_t /*sourceBytes*/, void *target) { - const auto *oldValue = static_cast(source); - ::new (target) VersionedPayload{ oldValue->value * 2, 7 }; - }); + { .fromVersion = 1, .toVersion = 2, .apply = [](const void *source, std::size_t /*sourceBytes*/, void *target) { + const auto *oldValue = static_cast(source); + ::new (target) VersionedPayload{ oldValue->value * 2, 7 }; + } }); shm::SharedMemoryRepository migrated( fixture.service, fixture.segment, "versioned", optionsOf(2), VersionedPayload{}); From e3d41dcbaf649d686fd8b0beb8e0c68d01c7df4e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dominik=20Kr=C3=BCmpelmann?= Date: Thu, 22 Oct 2026 22:18:33 +0200 Subject: [PATCH 4/4] docs(doxygen): document the remaining constructor parameters The Doxygen workflow runs with WARN_AS_ERROR and fails on master at the first undocumented parameter. Complete the parameter lists of EventBusEntry, LoggerConfiguration and MessageQueueService, and name the parameters of Allocator::findOrConstructArray so its @param entries resolve (the default implementation ignores them, hence [[maybe_unused]]). doxygen Doxyfile: exit 0, zero warnings. --- src/include/lintel/core/configuration/LoggerConfiguration.h | 3 ++- src/include/lintel/core/shm/ShmSegmentAccessor.h | 5 ++++- .../lintel/features/base/configuration/EventBusEntry.h | 5 ++++- .../lintel/features/base/services/MessageQueueService.h | 1 + 4 files changed, 11 insertions(+), 3 deletions(-) diff --git a/src/include/lintel/core/configuration/LoggerConfiguration.h b/src/include/lintel/core/configuration/LoggerConfiguration.h index 5612d45b..6cefe4c4 100644 --- a/src/include/lintel/core/configuration/LoggerConfiguration.h +++ b/src/include/lintel/core/configuration/LoggerConfiguration.h @@ -30,7 +30,8 @@ class LoggerConfiguration * @param pattern The log pattern * @param level The log level * @param isAsync Whether logging is asynchronous - * @param loggerSinks The sink configurations */ + * @param loggerSinks The sink configurations + * @param isJson Whether to format log messages as JSON */ LoggerConfiguration(std::string processName, std::string pattern, std::string level, diff --git a/src/include/lintel/core/shm/ShmSegmentAccessor.h b/src/include/lintel/core/shm/ShmSegmentAccessor.h index 831c09d7..d49611e2 100644 --- a/src/include/lintel/core/shm/ShmSegmentAccessor.h +++ b/src/include/lintel/core/shm/ShmSegmentAccessor.h @@ -90,7 +90,10 @@ class ShmSegmentAccessor * @param bytes size of the array in bytes (must be positive) * @return pointer to the array, or nullptr when named objects are not supported */ - virtual void *findOrConstructArray(const char * /*name*/, std::size_t /*bytes*/) { return nullptr; } + virtual void *findOrConstructArray([[maybe_unused]] const char *name, [[maybe_unused]] std::size_t bytes) + { + return nullptr; + } }; /** diff --git a/src/include/lintel/features/base/configuration/EventBusEntry.h b/src/include/lintel/features/base/configuration/EventBusEntry.h index 78bcc37c..1ede7059 100644 --- a/src/include/lintel/features/base/configuration/EventBusEntry.h +++ b/src/include/lintel/features/base/configuration/EventBusEntry.h @@ -50,7 +50,10 @@ class EventBusEntry : public Entry * @param component The configuration component name * @param eventBusName The bus name * @param segmentName The shared memory segment name - * @param capacities The bus/subscriber/topic/subscriber limits */ + * @param capacities The bus/subscriber/topic/subscriber limits + * @param pubPolicy What a producer does when the bus queue is full + * @param sticky Enable sticky/last-value events for topics + * @param stickyTtlMs TTL for sticky events in milliseconds */ EventBusEntry(const std::string_view &component, std::string eventBusName, std::string segmentName, diff --git a/src/include/lintel/features/base/services/MessageQueueService.h b/src/include/lintel/features/base/services/MessageQueueService.h index 81d806a8..b28a25a9 100644 --- a/src/include/lintel/features/base/services/MessageQueueService.h +++ b/src/include/lintel/features/base/services/MessageQueueService.h @@ -42,6 +42,7 @@ class MessageQueueService : public AbstractService * @param configuration parsed bootstrap configuration * @param processName name of the current process * @param messageQueueRepository repository holding the queue metadata + * @param schedulerService scheduler that runs the queue heartbeat (may be null) */ MessageQueueService(const std::shared_ptr &configuration, std::shared_ptr processName,