From cfb0ff1422553395ddafddba6333991014be91cc Mon Sep 17 00:00:00 2001 From: snkmcb Date: Sun, 13 Sep 2026 00:39:09 +0900 Subject: [PATCH 1/2] feat: add recoverable owned tile spool workspaces --- CHANGELOG.md | 4 + docs/roadmap/implementation-status.md | 7 +- docs/roadmap/streaming-and-tiling.md | 8 +- libs/usd-pointcloud-authoring/README.md | 12 +++ .../include/usdgeo/PointCloudLayer.h | 1 + .../src/PointCloudStreaming.cpp | 86 +++++++++++++++---- .../tests/test_usd_authoring.cpp | 82 ++++++++---------- libs/usd-pointcloud-tiling/README.md | 6 +- tools/usd-pointcloud-convert/main.cpp | 3 +- 9 files changed, 139 insertions(+), 70 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4808f67..9d7ab06 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,10 @@ All notable changes to this project are documented here. ### Added +- Owned, recoverable tile-spool workspaces through + `PointCloudPayloadOptions::spoolDirectory`; interrupted marked workspaces + are recovered on the next tiled authoring run, and the converter scopes its + workspace to the conversion transaction. - Local `.copc` and `.copc.laz` inputs in `usd-pointcloud-convert`, including deterministic generated-cache publication and reuse. - `PointCloudPayloadOptions::owner`, `PointCloudPayloadOwner`, and a diff --git a/docs/roadmap/implementation-status.md b/docs/roadmap/implementation-status.md index abc74f0..6adf4ff 100644 --- a/docs/roadmap/implementation-status.md +++ b/docs/roadmap/implementation-status.md @@ -426,9 +426,10 @@ remain be explicit. - [x] Materialize generated-cache hits as a generation named by the cache entry, reusing an intact copy without writing and replacing a damaged one -- [ ] Give tile spools an owned, recoverable working location instead of - timestamped directories in the system temporary directory, which an - interrupted process leaves behind +- [x] Give tile spools an owned, recoverable working location instead of + timestamped directories in the system temporary directory. The authoring + options select the workspace, marked leftovers are recovered on the next + invocation, and conversion ties the workspace to its transaction marker. #### Research - runtime streaming (no release gate) diff --git a/docs/roadmap/streaming-and-tiling.md b/docs/roadmap/streaming-and-tiling.md index bbebb2a..b2b4ee1 100644 --- a/docs/roadmap/streaming-and-tiling.md +++ b/docs/roadmap/streaming-and-tiling.md @@ -256,8 +256,12 @@ The spool system must define: - detection of incomplete spools; - deterministic iteration order. -Temporary output is isolated in a dedicated working directory and removed on -successful completion unless a debug-retention option is enabled. +Temporary output is isolated in a caller-owned dedicated working directory and +removed on completion. The authoring path writes a marker before creating tile +spools; a marked directory left by an interrupted process is removed before +the next run, while an unmarked existing directory is refused. The converter +places this workspace below its transaction marker so transaction recovery +owns it as well. ## 7. Payload output requirements diff --git a/libs/usd-pointcloud-authoring/README.md b/libs/usd-pointcloud-authoring/README.md index 6d03102..b3f3efc 100644 --- a/libs/usd-pointcloud-authoring/README.md +++ b/libs/usd-pointcloud-authoring/README.md @@ -162,6 +162,15 @@ beside it before the superseded one is removed. Nothing outside the owner's directory is touched. The layout is stated in the [file-format argument contract](../../docs/architecture/FILE_FORMAT_ARGUMENTS.md#generated-payload-ownership). +`PointCloudPayloadOptions::spoolDirectory` optionally names the exclusive +working directory for tiled stream spools. When it is empty, the authoring +library creates a `.spool` directory below the payload directory, scoped by +the payload owner when one is present. The directory contains a marker before +spooling starts; a later invocation removes a marked directory left by an +interrupted run, and successful or failed authoring removes it on exit. An +existing directory without the marker is refused so authoring never deletes +unrelated files. + ## Coordinate-space assumptions Input positions are **source-space** `usdgeo::Vec3d`. This module applies @@ -212,6 +221,9 @@ ctest --test-dir build/cy2026-windows-x86_64-py313-usd -C Release ` by source tile, and reconstructs one tile at a time before payload authoring. The router overload also accepts a planned `PointBudgetTileRouter`, allowing adaptive leaf plans to use the same bounded spool and payload path. + Spools live in the caller-owned `spoolDirectory` or an owner-scoped + `.spool` directory below the payload directory; marked leftovers are + recovered on the next invocation. - Large-corpus RSS measurement for generated, LAS, and LAZ inputs is available through the explicit `usdPointCloudAuthoring_stream_benchmark` target. It is disabled by default; diff --git a/libs/usd-pointcloud-authoring/include/usdgeo/PointCloudLayer.h b/libs/usd-pointcloud-authoring/include/usdgeo/PointCloudLayer.h index ad1f053..4018df7 100644 --- a/libs/usd-pointcloud-authoring/include/usdgeo/PointCloudLayer.h +++ b/libs/usd-pointcloud-authoring/include/usdgeo/PointCloudLayer.h @@ -140,6 +140,7 @@ struct PointCloudPayloadOptions { // superseded generations of the same owner are removed. Nothing outside // the owner's directory is touched, and the value is never persisted. std::string owner; + std::string spoolDirectory; }; // The payload owner for a FileFormat read: which source the layer reads and diff --git a/libs/usd-pointcloud-authoring/src/PointCloudStreaming.cpp b/libs/usd-pointcloud-authoring/src/PointCloudStreaming.cpp index 74c4b7a..3e5c103 100644 --- a/libs/usd-pointcloud-authoring/src/PointCloudStreaming.cpp +++ b/libs/usd-pointcloud-authoring/src/PointCloudStreaming.cpp @@ -1,12 +1,13 @@ #include "usdgeo/PointCloudLayer.h" #include "TiledPayloadAuthoring.h" +#include "usdgeo/CacheKey.h" #include #include -#include #include #include +#include #include #include #include @@ -208,23 +209,66 @@ bool PrepareData(const usdpointcloud::SpoolSchema& schema, return true; } +constexpr char kSpoolMarker[] = "USDGEO_SPOOL_WORKSPACE_V1"; + std::filesystem::path MakeSpoolDirectory( + const PointCloudPayloadOptions& options, std::vector& diagnostics) { - const auto stamp = std::chrono::steady_clock::now().time_since_epoch().count(); - const auto base = std::filesystem::temp_directory_path() / - ("usdgeo_point_spool_" + std::to_string(stamp)); + auto directory = options.spoolDirectory.empty() + ? std::filesystem::path(options.directory) / ".spool" + : std::filesystem::path(options.spoolDirectory); + if (options.spoolDirectory.empty() && !options.owner.empty()) { + directory /= usdgeo::StableCacheKey({{"payloadOwner", options.owner}}); + } + if (directory.empty()) { + AddError(diagnostics, DiagnosticCode::DecodeFailure, + "unable to determine point spool directory"); + return {}; + } std::error_code error; - for (std::uint32_t suffix = 0; suffix < 1000; ++suffix) { - const auto directory = suffix == 0 - ? base - : std::filesystem::path( - base.string() + "_" + std::to_string(suffix)); - if (std::filesystem::create_directory(directory, error)) return directory; - error.clear(); + if (std::filesystem::exists(directory, error)) { + if (error || !std::filesystem::is_directory(directory, error)) { + AddError(diagnostics, DiagnosticCode::DecodeFailure, + "point spool directory is not a directory"); + return {}; + } + const auto marker = directory / ".usdgeo-spool"; + std::ifstream markerFile(marker, std::ios::binary); + std::string markerValue; + const auto markerIsFile = + std::filesystem::is_regular_file(marker, error) && !error; + const auto markerIsValid = markerIsFile && + static_cast(markerFile) && + static_cast(std::getline( + markerFile, markerValue)) && + markerValue == kSpoolMarker; + markerFile.close(); + if (!markerIsValid) { + AddError(diagnostics, DiagnosticCode::DecodeFailure, + "point spool directory is not recoverable"); + return {}; + } + std::filesystem::remove_all(directory, error); + if (error) { + AddError(diagnostics, DiagnosticCode::DecodeFailure, + "unable to recover point spool directory"); + return {}; + } + } + std::filesystem::create_directories(directory, error); + if (error) { + AddError(diagnostics, DiagnosticCode::DecodeFailure, + "unable to create point spool directory"); + return {}; + } + std::ofstream marker(directory / ".usdgeo-spool", std::ios::binary); + if (!marker || !(marker << kSpoolMarker << '\n')) { + std::filesystem::remove_all(directory, error); + AddError(diagnostics, DiagnosticCode::DecodeFailure, + "unable to initialize point spool directory"); + return {}; } - AddError(diagnostics, DiagnosticCode::DecodeFailure, - "unable to create point spool directory"); - return {}; + return directory; } struct TileSpool { @@ -252,7 +296,15 @@ bool AuthorPointCloudTiledAssetFromStream( return false; } - const auto spoolDirectory = MakeSpoolDirectory(diagnostics); + std::error_code payloadDirectoryError; + const auto payloadDirectoryExisted = std::filesystem::exists( + std::filesystem::path(options.directory), payloadDirectoryError); + if (payloadDirectoryError) { + AddError(diagnostics, DiagnosticCode::DecodeFailure, + "unable to inspect point payload directory"); + return false; + } + const auto spoolDirectory = MakeSpoolDirectory(options, diagnostics); if (spoolDirectory.empty()) return false; std::map spools; std::size_t bufferedBytes = 0; @@ -263,6 +315,10 @@ bool AuthorPointCloudTiledAssetFromStream( std::vector cleanupDiagnostics; usdpointcloud::RemoveSpoolDirectory(spoolDirectory, cleanupDiagnostics); payloads.Rollback(); + if (!payloadDirectoryExisted) { + std::error_code removeError; + std::filesystem::remove(options.directory, removeError); + } }; const auto isCancelled = [&]() { return options.isCancelled && options.isCancelled(); diff --git a/libs/usd-pointcloud-authoring/tests/test_usd_authoring.cpp b/libs/usd-pointcloud-authoring/tests/test_usd_authoring.cpp index 86df377..1c918b0 100644 --- a/libs/usd-pointcloud-authoring/tests/test_usd_authoring.cpp +++ b/libs/usd-pointcloud-authoring/tests/test_usd_authoring.cpp @@ -125,47 +125,24 @@ std::map ReadDirectoryFiles( return files; } -std::set ListPointSpoolDirectories() { - std::set directories; +bool HasPointSpoolFile(const std::filesystem::path& directory) { std::error_code error; - const auto temporaryDirectory = std::filesystem::temp_directory_path(error); - Check(!error); + if (!std::filesystem::is_directory(directory, error)) { + Check(!error); + return false; + } for (const auto& entry : std::filesystem::directory_iterator( - temporaryDirectory, error)) { + directory, error)) { Check(!error); const auto name = entry.path().filename().string(); - const auto isDirectory = entry.is_directory(error); - Check(!error); - if (isDirectory && - name.rfind("usdgeo_point_spool_", 0) == 0) { - directories.insert(entry.path()); + if (entry.is_regular_file(error) && name.rfind("tile_", 0) == 0 && + entry.path().extension() == ".bin") { + return true; } + Check(!error); error.clear(); } Check(!error); - return directories; -} - -bool HasNewPointSpoolFile( - const std::set& existingDirectories) { - for (const auto& directory : ListPointSpoolDirectories()) { - if (existingDirectories.count(directory) != 0) continue; - std::error_code error; - for (const auto& entry : std::filesystem::directory_iterator( - directory, error)) { - Check(!error); - const auto name = entry.path().filename().string(); - const auto isRegularFile = entry.is_regular_file(error); - Check(!error); - if (isRegularFile && - name.rfind("tile_", 0) == 0 && - entry.path().extension() == ".bin") { - return true; - } - error.clear(); - } - Check(!error); - } return false; } @@ -200,9 +177,9 @@ class GeneratedPointStream final : public usdpointcloud::PointStream { public: GeneratedPointStream( std::size_t pointCount, - std::set existingSpoolDirectories) + std::filesystem::path spoolDirectory) : pointCount_(pointCount), - existingSpoolDirectories_(std::move(existingSpoolDirectories)) {} + spoolDirectory_(std::move(spoolDirectory)) {} usdpointcloud::PointStreamStatus ReadNext( usdpointcloud::PointChunk& chunk, @@ -212,8 +189,7 @@ class GeneratedPointStream final : public usdpointcloud::PointStream { if (index_ == pointCount_) { return usdpointcloud::PointStreamStatus::End; } - sawSpoolFile_ = sawSpoolFile_ || - HasNewPointSpoolFile(existingSpoolDirectories_); + sawSpoolFile_ = sawSpoolFile_ || HasPointSpoolFile(spoolDirectory_); const auto tileIndex = index_ % 32; const auto pointInTile = index_ / 32; data.positions = {{static_cast(tileIndex * 128 + 1), @@ -231,7 +207,7 @@ class GeneratedPointStream final : public usdpointcloud::PointStream { private: std::size_t pointCount_ = 0; std::size_t index_ = 0; - std::set existingSpoolDirectories_; + std::filesystem::path spoolDirectory_; bool sawSpoolFile_ = false; }; @@ -1080,17 +1056,25 @@ void TestGeneratedStreamTiledPayloadAuthoring() { usdgeo::GeoReference reference; reference.epsgCode = 26910; - const auto spoolDirectoriesBefore = ListPointSpoolDirectories(); - GeneratedPointStream stream(131072, spoolDirectoriesBefore); const auto payloadDirectory = std::filesystem::temp_directory_path() / "usd_geo_generated_payloads"; const auto rootLayerPath = payloadDirectory / "PointCloud.usda"; + const auto spoolDirectory = payloadDirectory / "spool-work"; std::filesystem::remove_all(payloadDirectory); + std::filesystem::create_directories(spoolDirectory); + { + std::ofstream marker(spoolDirectory / ".usdgeo-spool", std::ios::binary); + marker << "USDGEO_SPOOL_WORKSPACE_V1\n"; + std::ofstream stale(spoolDirectory / "tile_stale.bin"); + stale << "stale"; + } + GeneratedPointStream stream(131072, spoolDirectory); std::vector diagnostics; std::vector manifestEntries; - const usdgeo::PointCloudPayloadOptions options{ + usdgeo::PointCloudPayloadOptions options{ payloadDirectory.string(), rootLayerPath.string(), 1024, {}, {}, &manifestEntries}; + options.spoolDirectory = spoolDirectory.string(); Check(usdgeo::AuthorPointCloudTiledAssetFromStream( layer.operator->(), "/PointCloud", stream, reference, {128.0, 0}, options, diagnostics)); @@ -1112,7 +1096,7 @@ void TestGeneratedStreamTiledPayloadAuthoring() { Check(layer->GetPrimAtPath(pxr::SdfPath( "/PointCloud/Tiles/Tile_L0_p31_p0_p0")) != nullptr); Check(stream.SawSpoolFile()); - Check(ListPointSpoolDirectories() == spoolDirectoriesBefore); + Check(!std::filesystem::exists(spoolDirectory)); std::filesystem::remove_all(payloadDirectory); } @@ -1131,20 +1115,21 @@ void TestStreamCancellationCleansSpools() { const auto payloadDirectory = std::filesystem::temp_directory_path() / "usd_geo_cancelled_payloads"; std::filesystem::remove_all(payloadDirectory); - const auto spoolDirectoriesBefore = ListPointSpoolDirectories(); + const auto spoolDirectory = payloadDirectory / "spool-work"; int cancellationChecks = 0; std::vector diagnostics; - const usdgeo::PointCloudPayloadOptions options{ + usdgeo::PointCloudPayloadOptions options{ payloadDirectory.string(), (payloadDirectory / "PointCloud.usda").string(), 1, [&cancellationChecks]() { return ++cancellationChecks >= 3; }}; + options.spoolDirectory = spoolDirectory.string(); Check(!usdgeo::AuthorPointCloudTiledAssetFromStream( layer.operator->(), "/PointCloud", stream, reference, {1.0, 0}, options, diagnostics)); Check(!diagnostics.empty()); - Check(ListPointSpoolDirectories() == spoolDirectoriesBefore); + Check(!std::filesystem::exists(spoolDirectory)); Check(!std::filesystem::exists(payloadDirectory)); } @@ -1165,20 +1150,21 @@ void TestStreamCancellationDuringSpoolReadCleansSpools() { std::filesystem::temp_directory_path() / "usd_geo_cancelled_spool_read_payloads"; std::filesystem::remove_all(payloadDirectory); - const auto spoolDirectoriesBefore = ListPointSpoolDirectories(); + const auto spoolDirectory = payloadDirectory / "spool-work"; int cancellationChecks = 0; std::vector diagnostics; - const usdgeo::PointCloudPayloadOptions options{ + usdgeo::PointCloudPayloadOptions options{ payloadDirectory.string(), (payloadDirectory / "PointCloud.usda").string(), 1, [&cancellationChecks]() { return ++cancellationChecks >= 6; }}; + options.spoolDirectory = spoolDirectory.string(); Check(!usdgeo::AuthorPointCloudTiledAssetFromStream( layer.operator->(), "/PointCloud", stream, reference, {1.0, 0}, options, diagnostics)); Check(!diagnostics.empty()); - Check(ListPointSpoolDirectories() == spoolDirectoriesBefore); + Check(!std::filesystem::exists(spoolDirectory)); Check(!std::filesystem::exists(payloadDirectory)); } diff --git a/libs/usd-pointcloud-tiling/README.md b/libs/usd-pointcloud-tiling/README.md index 5cf4115..4aaa835 100644 --- a/libs/usd-pointcloud-tiling/README.md +++ b/libs/usd-pointcloud-tiling/README.md @@ -37,7 +37,11 @@ reject missing or truncated footers. Attribute values are serialized using the scalar types declared by the schema. The writer buffers records up to its configured byte threshold before flushing; streaming callers may combine multiple writers under a total working-set budget. Callers own the working -directory and can remove it with `RemoveSpoolDirectory`. +directory and can remove it with `RemoveSpoolDirectory`. The OpenUSD +authoring bridge places stream spools in a caller-selected directory or an +owner-scoped `.spool` directory, marks it before writing, and removes a +marked leftover before the next run so an interrupted process can be +recovered without scanning the system temporary directory. `PointTileManifest` records the tile ID, LOD level, source bounds, point count, and portable payload path for generated tile assets. Its v1 serializer sorts diff --git a/tools/usd-pointcloud-convert/main.cpp b/tools/usd-pointcloud-convert/main.cpp index 0eed9b1..034356e 100644 --- a/tools/usd-pointcloud-convert/main.cpp +++ b/tools/usd-pointcloud-convert/main.cpp @@ -1078,10 +1078,11 @@ int main(int argc, char** argv) { } std::vector tileManifestEntries; - const usdgeo::PointCloudPayloadOptions payloadOptions{ + usdgeo::PointCloudPayloadOptions payloadOptions{ generationPayloadDirectory.string(), generationRootPath.string(), request.tileMemoryLimitBytes, request.readOptions.isCancelled, {}, &tileManifestEntries}; + payloadOptions.spoolDirectory = (transactionPath / "spool").string(); std::unique_ptr plannedStream; const auto authored = [&]() { if (!request.maxPointsPerTile) { From 92a5e087160f5bee991015606dae38ce4fd94178 Mon Sep 17 00:00:00 2001 From: snkmcb Date: Sun, 13 Sep 2026 00:46:14 +0900 Subject: [PATCH 2/2] fix: remove owned spool parents on rollback --- libs/usd-pointcloud-authoring/src/PointCloudStreaming.cpp | 3 +++ libs/usd-pointcloud-authoring/tests/test_usd_authoring.cpp | 4 ++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/libs/usd-pointcloud-authoring/src/PointCloudStreaming.cpp b/libs/usd-pointcloud-authoring/src/PointCloudStreaming.cpp index 3e5c103..8107a0d 100644 --- a/libs/usd-pointcloud-authoring/src/PointCloudStreaming.cpp +++ b/libs/usd-pointcloud-authoring/src/PointCloudStreaming.cpp @@ -317,6 +317,9 @@ bool AuthorPointCloudTiledAssetFromStream( payloads.Rollback(); if (!payloadDirectoryExisted) { std::error_code removeError; + std::filesystem::remove( + std::filesystem::path(options.directory) / ".spool", + removeError); std::filesystem::remove(options.directory, removeError); } }; diff --git a/libs/usd-pointcloud-authoring/tests/test_usd_authoring.cpp b/libs/usd-pointcloud-authoring/tests/test_usd_authoring.cpp index 1c918b0..ba004d5 100644 --- a/libs/usd-pointcloud-authoring/tests/test_usd_authoring.cpp +++ b/libs/usd-pointcloud-authoring/tests/test_usd_authoring.cpp @@ -177,9 +177,9 @@ class GeneratedPointStream final : public usdpointcloud::PointStream { public: GeneratedPointStream( std::size_t pointCount, - std::filesystem::path spoolDirectory) + std::filesystem::path spoolDirectory) : pointCount_(pointCount), - spoolDirectory_(std::move(spoolDirectory)) {} + spoolDirectory_(std::move(spoolDirectory)) {} usdpointcloud::PointStreamStatus ReadNext( usdpointcloud::PointChunk& chunk,