From 20ffe3dc6dce09f23a1f86d38c3fa4450facc3b3 Mon Sep 17 00:00:00 2001 From: snkmcb Date: Fri, 4 Sep 2026 23:58:55 +0900 Subject: [PATCH 1/2] feat: support resolver-backed COPC conversion --- docs/architecture/RESOLVER_SOURCE.md | 15 +- docs/guides/BUILDING.md | 24 ++- docs/reference/CAPABILITY_MATRIX.md | 8 +- docs/roadmap/implementation-status.md | 23 +-- docs/roadmap/infrastructure-maturity.md | 15 +- plugins/pointcloud-copc/CMakeLists.txt | 12 ++ plugins/pointcloud-copc/README.md | 8 +- .../pointcloud-copc/openstrata.plugin.yaml | 3 + .../tests/fixtures/conformance.copc | Bin 0 -> 813 bytes tools/usd-pointcloud-convert/main.cpp | 183 ++++++++++++++---- .../test_resolver_conversion.cmake | 83 ++++++++ 11 files changed, 295 insertions(+), 79 deletions(-) create mode 100644 plugins/pointcloud-copc/tests/fixtures/conformance.copc create mode 100644 tools/usd-pointcloud-convert/test_resolver_conversion.cmake diff --git a/docs/architecture/RESOLVER_SOURCE.md b/docs/architecture/RESOLVER_SOURCE.md index 13501b0..8b143d9 100644 --- a/docs/architecture/RESOLVER_SOURCE.md +++ b/docs/architecture/RESOLVER_SOURCE.md @@ -175,14 +175,13 @@ other outcome through the categories in §4. `usd-pointcloud-convert` remains the production path for deterministic, long-running payload generation, and it is the only thing that publishes a -generated entry. **Not implemented (`v0.10.0`):** it accepts `.las` and `.laz` -local inputs only. No COPC input, and no resolver-addressable identifier, -reaches it, so no COPC read — local or resolver-backed — has an entry to hit in -a normal workflow. The lookup side is complete and covered; the generation side -for COPC and for resolver-addressable inputs is future work. When it lands it -computes the same resolver-neutral identity, populates and reuses the generated -cache only under stable identity, and may record the normalized identity class -in manifest or debug metadata — never the transport secrets covered in §2.3. +generated entry. It accepts local `.las`, `.laz`, `.copc`, and `.copc.laz` +inputs, plus resolver-addressable COPC identifiers. Resolver inputs are opened +through the active `ArResolver` and the same resolver-neutral identity is used +for cache layout. Generated-cache publication and reuse are enabled only for +`Stable` identity; `Unstable` and `Unavailable` inputs still convert from the +source but do not publish generated entries. Conversion manifests use a fixed +resolver-source label and never persist the identifier or validation token. ### 3.3 Cache ownership boundary — Implemented (`v0.10.0`) diff --git a/docs/guides/BUILDING.md b/docs/guides/BUILDING.md index 13ecf5d..bf9e442 100644 --- a/docs/guides/BUILDING.md +++ b/docs/guides/BUILDING.md @@ -74,10 +74,10 @@ A per-bundle `ost plugin build` does not define `USDGEO_BUILD_TESTS`, so the CTest integration targets are built only by the workspace `ost build`. Run both when changing a plugin. -The bundles do not all declare OST test fixtures yet, so `ost plugin test` currently -reports the L3 `usdcat.read` and L4 `python.stage_open` checks as skipped. That -gap is tracked in -[implementation status](../roadmap/implementation-status.md). +All four product bundles declare OST smoke fixtures, so `ost plugin test` +exercises the L3 `usdcat.read` and L4 `python.stage_open` checks for LAS, LAZ, +COPC, and PLY. The workspace CTest integration targets remain separate and are +built by `ost build`. ## Explicit tiled conversion @@ -119,6 +119,22 @@ An existing committed cache entry is materialized without decoding the source again. The cache is derived data; the source file and normalized arguments remain the authority for invalidation. +Resolver-addressable COPC identifiers use the active OpenUSD resolver. The +resolver must be composed at runtime through `PXR_PLUGINPATH_NAME` and must +provide efficient random-access `ArAsset` reads. Generated-cache publication +and reuse require a `Stable` resolver identity; an unstable or unavailable +identity still converts from the source without publishing a generated entry: + +```powershell +$env:PXR_PLUGINPATH_NAME = C:\path\to\resolver\resources +$env:PXR_AR_DEFAULT_RESOLVER = MyResolver +usd-pointcloud-convert ` + http://example.invalid/pointcloud.copc ` + C:\path\to\output\PointCloud.usda ` + --tile-size 128 ` + --cache-root C:\path\to\pointcloud-cache +``` + Direct LAS, LAZ, and COPC FileFormat reads can use the same cache entries by setting `USDGEO_CACHE_ROOT` in the host process. A committed hit loads the cached root before point decoding. The environment variable is storage diff --git a/docs/reference/CAPABILITY_MATRIX.md b/docs/reference/CAPABILITY_MATRIX.md index 096270c..a02a27f 100644 --- a/docs/reference/CAPABILITY_MATRIX.md +++ b/docs/reference/CAPABILITY_MATRIX.md @@ -36,7 +36,7 @@ uses the same reader and authoring contracts for long-running generation. | Local COPC conversion and generated-cache publication | Supported | `usd-pointcloud-convert` accepts `.copc` and `.copc.laz`, reuses the COPC point stream for fixed-grid or adaptive tiled generation, and publishes entries compatible with local COPC FileFormat cache lookup | | Resolver-backed COPC reads | Supported | The plugin adapts an `ArAsset` opened through the active `ArResolver` to the project-owned random-access source; remote tiled reads require an absolute local `payloadDirectory`. Transport, authentication, retries, and raw byte caching belong to the resolver | | Resolver-backed generated-cache lookup | Supported | COPC extracts resolver-neutral identity through the shared adapter and reuses a committed entry only for `Stable` identity. Incomplete and corrupted entries are invalidated, and a changed validation token regenerates rather than hitting the superseded entry. Recorded against an external resolver in the [resolver read baseline](RESOLVER_BASELINE.md); the contract is the [resolver-backed source contract](../architecture/RESOLVER_SOURCE.md) | -| Resolver-backed generated-cache generation | Not supported | The converter publishes local COPC entries but does not yet accept resolver-addressable identifiers, so it cannot publish against resolver-provided identity | +| Resolver-backed generated-cache generation | Supported | `usd-pointcloud-convert` accepts resolver-addressable COPC identifiers, opens them through the active resolver, and publishes/reuses entries only for `Stable` identity; unstable and unavailable identities convert without cache publication | | Generated-cache decision diagnostics | Supported | `usdgeo::cache::CacheDecision` publishes seven stable, transport-neutral categories; COPC projects them onto `COPC009`-`COPC012` and every message names its category | ## LAS Versions @@ -249,9 +249,9 @@ in the [tile and LOD contract](../architecture/LOD.md). through `USDGEO_CACHE_ROOT`. Reuse requires either a stable local filesystem identity or a `Stable` resolver identity; `Unstable` and `Unavailable` identity fails closed and reads from the source. The conversion tool - publishes entries for local LAS, LAZ, and COPC paths, but does not yet accept - resolver-addressable identifiers or publish against resolver-provided - identity. + publishes entries for local LAS, LAZ, and COPC paths plus resolver-addressable + COPC identifiers when the resolver provides a `Stable` identity; unstable + and unavailable resolver identities remain conversion-only. - No HTTP client, cloud SDK, authentication flow, retry policy, or raw byte-range cache exists here. Resolver-backed reads consume whatever the active `ArResolver` provides, and no resolver implementation is a build-time diff --git a/docs/roadmap/implementation-status.md b/docs/roadmap/implementation-status.md index 894ebad..2b6ebb5 100644 --- a/docs/roadmap/implementation-status.md +++ b/docs/roadmap/implementation-status.md @@ -380,25 +380,26 @@ Phase 6 — validation and baselines: - [x] Record remote hit ratios and `bytes fetched / source size` baselines. A metadata open costs 0.001486 of the asset in three requests; a full read costs exactly 1.0 in 277 requests. A generated-cache hit ratio for COPC is - not measurable end to end, because `usd-pointcloud-convert` publishes - entries for `.las` and `.laz` local inputs only and no COPC read has an - entry to hit; the reuse decision itself is Tier 1 covered + covered by the resolver-backed converter miss-to-hit test for stable + identity; the external Tier 2 record remains a source-read baseline and + does not claim a generated-cache ratio #### Open after `v0.10.0` -- [ ] Declare an OST smoke fixture for the `pointcloud-copc` bundle. It is the - only product bundle without one, so its L3 `usdcat.read` and L4 - `python.stage_open` checks skip and its bundle cells verify discovery - only. +- [x] Declare an OST smoke fixture for the `pointcloud-copc` bundle. Its + component-owned conformance fixture now exercises the L3 `usdcat.read` + and L4 `python.stage_open` checks alongside the other product bundles. - [x] Publish generated cache entries for local COPC inputs. `usd-pointcloud-convert` accepts `.copc` and `.copc.laz`, uses the shared COPC point stream for fixed-grid and adaptive generation, and publishes - entries compatible with local COPC FileFormat lookup. Remote publication - remains blocked on resolver-addressable converter inputs below. -- [ ] Accept resolver-addressable identifiers in `usd-pointcloud-convert`, as - described in the + entries compatible with local COPC FileFormat lookup. Resolver-backed + publication is covered below and requires a stable resolver identity. +- [x] Accept resolver-addressable COPC identifiers in + `usd-pointcloud-convert`, as described in the [resolver-backed source contract](../architecture/RESOLVER_SOURCE.md). + Stable resolver identities publish and reuse generated entries; unstable + and unavailable identities convert without cache publication. #### Research - runtime streaming (no release gate) diff --git a/docs/roadmap/infrastructure-maturity.md b/docs/roadmap/infrastructure-maturity.md index c40b023..9ce9e8a 100644 --- a/docs/roadmap/infrastructure-maturity.md +++ b/docs/roadmap/infrastructure-maturity.md @@ -486,13 +486,14 @@ isolated under `tests/`, and recorded remote baselines including `bytes fetched / source size` against the released `usd-http-resolver` `v0.4.0`. -One thing the gate asked for is recorded as absent rather than as met: a -generated-cache *hit ratio* for a remote source. The converter now publishes -entries for local `.copc` and `.copc.laz` inputs, but it does not yet accept a -resolver-addressable identifier or publish against resolver-provided identity. -The lookup side, local COPC publication, reuse rules, and every decision -diagnostic are complete and covered; resolver-addressable conversion is the -remaining step that makes the remote ratio measurable. +The converter now accepts resolver-addressable COPC identifiers and publishes +entries against resolver-provided identity when it is `Stable`. Tier 1 covers +the resolver-backed miss-to-hit path and verifies that resolver identifiers and +validation tokens do not enter conversion manifests. `Unstable` and +`Unavailable` identities remain conversion-only, so a remote cache hit ratio is +measurable only for resolvers that provide stable validation metadata; the Tier +2 baseline currently records resolver read selectivity rather than a generated +cache ratio. ### Research - Runtime Streaming diff --git a/plugins/pointcloud-copc/CMakeLists.txt b/plugins/pointcloud-copc/CMakeLists.txt index 57b0b4f..fad8476 100644 --- a/plugins/pointcloud-copc/CMakeLists.txt +++ b/plugins/pointcloud-copc/CMakeLists.txt @@ -127,6 +127,18 @@ if(USDGEO_BUILD_TESTS) usdgeo_configure_openusd_test_runtime(pointcloudCopc_integration) set_property(TEST pointcloudCopc_integration APPEND PROPERTY ENVIRONMENT "PXR_AR_DEFAULT_RESOLVER=HttpResolver") + if(TARGET usd-pointcloud-convert) + add_dependencies(usd-pointcloud-convert HttpResolver) + add_test(NAME usdPointCloudConvert_resolver_copc + COMMAND ${CMAKE_COMMAND} + -Dconverter=$ + -Dfixture=${CMAKE_CURRENT_BINARY_DIR}/resolver-conformance.copc + -Dfixture_generator=$ + -Dtest_root=${CMAKE_CURRENT_BINARY_DIR}/resolver-conversion + -Dresolver_plugin_path=${CMAKE_CURRENT_SOURCE_DIR}/../../tests/plugins/httpresolver/plugin/resources/httpresolver + -P ${CMAKE_CURRENT_SOURCE_DIR}/../../tools/usd-pointcloud-convert/test_resolver_conversion.cmake) + usdgeo_configure_openusd_test_runtime(usdPointCloudConvert_resolver_copc) + endif() endif() openstrata_configure_plugin( diff --git a/plugins/pointcloud-copc/README.md b/plugins/pointcloud-copc/README.md index 1aa9a4d..49673f9 100644 --- a/plugins/pointcloud-copc/README.md +++ b/plugins/pointcloud-copc/README.md @@ -31,10 +31,10 @@ absolute local `payloadDirectory`. Generated-USDC cache lookup is enabled for a stable local filesystem identity and for a `Stable` resolver identity. `Unstable` and `Unavailable` identity -fails closed: the read proceeds from the source and says why. Nothing here -publishes an entry; `usd-pointcloud-convert` publishes entries for local -`.copc` and `.copc.laz` inputs. Resolver-backed publication remains unavailable -until the converter accepts resolver-addressable identifiers. +fails closed: the read proceeds from the source and says why. This plugin does +not publish entries; `usd-pointcloud-convert` publishes entries for local +`.copc` and `.copc.laz` inputs and resolver-addressable COPC identifiers when +the resolver supplies a `Stable` identity. Every cache decision is reported through four codes that project the stable categories `usdgeo::cache` publishes, each message naming its exact category: diff --git a/plugins/pointcloud-copc/openstrata.plugin.yaml b/plugins/pointcloud-copc/openstrata.plugin.yaml index 784065f..e7eb865 100644 --- a/plugins/pointcloud-copc/openstrata.plugin.yaml +++ b/plugins/pointcloud-copc/openstrata.plugin.yaml @@ -11,3 +11,6 @@ requires: capabilities: [usd-stage-read] usd: plug_info: plugin/resources/pointcloud-copc/plugInfo.json +tests: + smoke: + - tests/fixtures/conformance.copc diff --git a/plugins/pointcloud-copc/tests/fixtures/conformance.copc b/plugins/pointcloud-copc/tests/fixtures/conformance.copc new file mode 100644 index 0000000000000000000000000000000000000000..92345397f9e1938f64c2394879076364f37f660c GIT binary patch literal 813 zcmeZq40dC{0vK7a2$C#R&bXL~fdPmZ*yI?PacHj=S?B(+s>L3i1}k^ycYx6IA#}lV z2ps?uhteKUc?T$60#yf74>klr;BpH>1+7`h`31?yb}%w5U~u*ia8}67OUp-=KxcbE zee44DX%WPE8u!S1*_q7&0 +#include #include #include @@ -111,6 +115,8 @@ InputFormat DetectInputFormat(const std::filesystem::path& path) { std::unique_ptr OpenPointStream( InputFormat format, const std::filesystem::path& inputPath, + const std::shared_ptr& randomAccessSource, + const std::string& sourceName, const usdpointcloud::PointReadOptions& options, usdlas::LasHeader& header, std::vector& diagnostics) { @@ -123,6 +129,14 @@ std::unique_ptr OpenPointStream( inputPath.string(), options, header, diagnostics); } if (format == InputFormat::Copc) { + if (randomAccessSource) { + usdcopc::CopcHeader copcHeader; + auto stream = usdcopc::OpenCopcPointStream( + randomAccessSource, options, copcHeader, diagnostics, + sourceName); + header = copcHeader.las; + return stream; + } usdcopc::CopcHeader copcHeader; auto stream = usdcopc::OpenCopcPointStream( inputPath.string(), options, copcHeader, diagnostics); @@ -132,6 +146,91 @@ std::unique_ptr OpenPointStream( return nullptr; } +struct ConverterSource { + std::string identifier; + std::string sourceName; + std::filesystem::path localPath; + std::shared_ptr asset; + std::shared_ptr randomAccessSource; + usdgeo::cache::SourceIdentity identity; + usdgeo::cache::ResolverIdentityStability resolverStability = + usdgeo::cache::ResolverIdentityStability::Unavailable; + bool resolverBacked = false; +}; + +bool IsResolverIdentifier(const std::string& identifier) { + return identifier.find("://") != std::string::npos; +} + +std::filesystem::path FormatPath(const std::string& identifier) { + const auto queryStart = identifier.find_first_of("?#"); + const auto withoutQuery = identifier.substr(0, queryStart); + const auto slash = withoutQuery.find_last_of("/\\"); + return std::filesystem::path( + slash == std::string::npos ? withoutQuery + : withoutQuery.substr(slash + 1)); +} + +bool PrepareSource(const std::string& identifier, + InputFormat format, + ConverterSource& source, + std::string& errorMessage) { + source = {}; + source.identifier = identifier; + if (!IsResolverIdentifier(identifier)) { + source.localPath = std::filesystem::absolute( + std::filesystem::path(identifier)); + source.sourceName = source.localPath.string(); + if (!usdgeo::cache::TryBuildLocalSourceIdentity( + source.localPath, source.identity, errorMessage)) { + return false; + } + return true; + } + + if (format != InputFormat::Copc) { + errorMessage = + "resolver-addressable conversion currently supports COPC only"; + return false; + } + const auto resolvedPath = pxr::ArGetResolver().Resolve(identifier); + if (resolvedPath.GetPathString().empty()) { + errorMessage = "active resolver could not resolve COPC asset: " + + identifier; + return false; + } + source.asset = pxr::ArGetResolver().OpenAsset(resolvedPath); + if (!source.asset) { + errorMessage = "active resolver could not open COPC asset: " + + resolvedPath.GetPathString(); + return false; + } + source.resolverBacked = true; + source.sourceName = resolvedPath.GetPathString(); + source.randomAccessSource = + std::make_shared( + source.asset, source.sourceName); + std::string identityError; + usdgeo::TryBuildResolverSourceIdentity( + pxr::ArGetResolver(), identifier, resolvedPath, *source.asset, + source.identity, source.resolverStability, identityError); + return true; +} + +bool RefreshSourceIdentity(const ConverterSource& source, + usdgeo::cache::SourceIdentity& identity, + std::string& errorMessage) { + if (!source.resolverBacked) { + return usdgeo::cache::TryBuildLocalSourceIdentity( + source.localPath, identity, errorMessage); + } + usdgeo::cache::ResolverIdentityStability stability; + return usdgeo::TryBuildResolverSourceIdentity( + pxr::ArGetResolver(), source.identifier, + pxr::ArResolvedPath(source.sourceName), *source.asset, identity, + stability, errorMessage); +} + bool SameSourceIdentity(const usdgeo::cache::SourceIdentity& left, const usdgeo::cache::SourceIdentity& right) { return left.identifier == right.identifier && @@ -354,17 +453,14 @@ bool CopyDirectory(const std::filesystem::path& source, } bool BuildCacheDescriptor( - const std::filesystem::path& inputPath, + const usdgeo::cache::SourceIdentity& sourceIdentity, InputFormat inputFormat, const usdgeo::GeoReference& reference, const usdpointcloud::PointReadRequest& request, const std::string& tilePlanKey, usdgeo::cache::Descriptor& descriptor, std::string& errorMessage) { - if (!usdgeo::cache::TryBuildLocalSourceIdentity( - inputPath, descriptor.source, errorMessage)) { - return false; - } + descriptor.source = sourceIdentity; descriptor.pluginVersion = "usd-pointcloud-plugins-0.3.0-display-v3"; descriptor.parserVersion = inputFormat == InputFormat::Las ? "las-reader-1" @@ -500,19 +596,13 @@ bool PublishCacheEntry(const usdgeo::cache::Layout& temporaryLayout, } bool WriteManifest(const std::filesystem::path& manifestPath, - const std::filesystem::path& inputPath, + const std::string& inputName, + std::uintmax_t inputSize, const std::filesystem::path& outputPath, const std::filesystem::path& payloadDirectory, const usdpointcloud::PointReadRequest& request, std::string& errorMessage) { std::error_code error; - const auto inputSize = std::filesystem::file_size(inputPath, error); - if (error) { - errorMessage = "unable to inspect input for manifest: " + - error.message(); - return false; - } - std::vector payloads; for (std::filesystem::recursive_directory_iterator iterator( payloadDirectory, error), end; @@ -541,7 +631,7 @@ bool WriteManifest(const std::filesystem::path& manifestPath, return false; } output << "format=usd-pointcloud-manifest-v1\n" - << "input.file=" << inputPath.filename().generic_string() << "\n" + << "input.file=" << inputName << "\n" << "input.sizeBytes=" << inputSize << "\n" << "output.root=" << outputPath.filename().generic_string() << "\n" << "output.payloadDirectory=" @@ -630,10 +720,10 @@ int main(int argc, char** argv) { return 2; } - const std::filesystem::path inputPath = std::filesystem::absolute(argv[1]); + const std::string inputIdentifier = argv[1]; const std::filesystem::path outputPath = std::filesystem::absolute(argv[2]); const auto manifestPath = ManifestPath(outputPath); - const auto inputFormat = DetectInputFormat(inputPath); + const auto inputFormat = DetectInputFormat(FormatPath(inputIdentifier)); if (inputFormat == InputFormat::Unsupported) { std::cerr << "Input must have a .las, .laz, .copc, or .copc.laz extension\n"; return 2; @@ -643,6 +733,19 @@ int main(int argc, char** argv) { return 2; } + ConverterSource source; + std::string sourceError; + if (!PrepareSource(inputIdentifier, inputFormat, source, sourceError)) { + std::cerr << "Unable to prepare input source: " << sourceError << "\n"; + return 1; + } + const auto manifestInputName = source.resolverBacked + ? "resolver-addressed-source" + : source.localPath.filename().generic_string(); + const auto inputSize = source.resolverBacked + ? static_cast(source.asset->GetSize()) + : source.identity.sizeBytes; + std::map arguments; arguments.emplace("tile", "true"); bool hasTileSize = false; @@ -758,7 +861,8 @@ int main(int argc, char** argv) { usdlas::LasHeader header; auto stream = OpenPointStream( - inputFormat, inputPath, request.readOptions, header, diagnostics); + inputFormat, source.localPath, source.randomAccessSource, + source.sourceName, request.readOptions, header, diagnostics); if (!stream) { if (diagnostics.empty()) { std::cerr << "Unable to open point stream\n"; @@ -787,19 +891,12 @@ int main(int argc, char** argv) { usdgeo::cache::SourceIdentity planningIdentity; std::string tilePlanKey; if (request.maxPointsPerTile) { - std::string identityError; - if (!usdgeo::cache::TryBuildLocalSourceIdentity( - inputPath, planningIdentity, identityError)) { - diagnostics.push_back({usdgeo::DiagnosticCode::SourceOpenFailed, - usdgeo::Severity::Error, identityError}); - PrintDiagnostics(diagnostics); - return 1; - } + planningIdentity = source.identity; adaptiveStreamFactory = [&]() { usdlas::LasHeader passHeader; auto passStream = OpenPointStream( - inputFormat, inputPath, request.readOptions, passHeader, - diagnostics); + inputFormat, source.localPath, source.randomAccessSource, + source.sourceName, request.readOptions, passHeader, diagnostics); if (passStream && !request.attributes.empty()) { passStream = std::make_unique( std::move(passStream), request.attributes); @@ -823,7 +920,7 @@ int main(int argc, char** argv) { } } - const bool cacheEnabled = !cacheRoot.empty(); + const bool cacheEnabled = !cacheRoot.empty() && source.identity.IsValid(); usdgeo::cache::Layout cacheLayout; usdgeo::cache::Layout cacheGenerationLayout; bool cacheHit = false; @@ -831,7 +928,7 @@ int main(int argc, char** argv) { if (cacheEnabled) { usdgeo::cache::Descriptor descriptor; std::string cacheError; - if (!BuildCacheDescriptor(inputPath, inputFormat, reference, request, + if (!BuildCacheDescriptor(source.identity, inputFormat, reference, request, tilePlanKey, descriptor, cacheError)) { std::cerr << "Unable to build cache layout: " << cacheError << "\n"; return 1; @@ -921,7 +1018,8 @@ int main(int argc, char** argv) { std::string materializeError; if (!MaterializeCache(cacheLayout, temporaryRootPath, payloadDirectory, materializeError) || - !WriteManifest(temporaryManifestPath, inputPath, outputPath, + !WriteManifest(temporaryManifestPath, manifestInputName, inputSize, + outputPath, payloadDirectory, request, materializeError)) { cleanup(); std::cerr << "Unable to materialize cache hit: " @@ -983,16 +1081,17 @@ int main(int argc, char** argv) { if (!adaptiveStreamFactory) { return false; } - std::string identityError; usdgeo::cache::SourceIdentity authoringIdentity; - if (!usdgeo::cache::TryBuildLocalSourceIdentity( - inputPath, authoringIdentity, identityError) || - !SameSourceIdentity(planningIdentity, authoringIdentity)) { - diagnostics.push_back({ - usdgeo::DiagnosticCode::DecodeFailure, - usdgeo::Severity::Error, - "input changed during adaptive tile planning"}); - return false; + if (source.identity.IsValid()) { + std::string identityError; + if (!RefreshSourceIdentity(source, authoringIdentity, identityError) || + !SameSourceIdentity(planningIdentity, authoringIdentity)) { + diagnostics.push_back({ + usdgeo::DiagnosticCode::DecodeFailure, + usdgeo::Severity::Error, + "input changed during adaptive tile planning"}); + return false; + } } plannedStream = adaptiveStreamFactory(); if (!plannedStream) return false; @@ -1042,7 +1141,8 @@ int main(int argc, char** argv) { cacheEnabled ? cacheGenerationLayout.manifest : temporaryManifestPath; const auto generatedOutputPath = cacheEnabled ? cacheGenerationLayout.rootLayer : outputPath; - if (!WriteManifest(generatedManifestPath, inputPath, generatedOutputPath, + if (!WriteManifest(generatedManifestPath, manifestInputName, inputSize, + generatedOutputPath, generationPayloadDirectory, request, manifestError)) { std::cerr << "Unable to write conversion manifest: " << manifestError << "\n"; @@ -1061,7 +1161,8 @@ int main(int argc, char** argv) { layer = nullptr; if (!MaterializeCache(cacheLayout, temporaryRootPath, payloadDirectory, manifestError) || - !WriteManifest(temporaryManifestPath, inputPath, outputPath, + !WriteManifest(temporaryManifestPath, manifestInputName, inputSize, + outputPath, payloadDirectory, request, manifestError)) { cleanup(); std::cerr << "Unable to materialize generated cache: " diff --git a/tools/usd-pointcloud-convert/test_resolver_conversion.cmake b/tools/usd-pointcloud-convert/test_resolver_conversion.cmake new file mode 100644 index 0000000..fca0a96 --- /dev/null +++ b/tools/usd-pointcloud-convert/test_resolver_conversion.cmake @@ -0,0 +1,83 @@ +if(NOT DEFINED converter OR NOT DEFINED fixture OR + NOT DEFINED fixture_generator OR NOT DEFINED test_root OR + NOT DEFINED resolver_plugin_path) + message(FATAL_ERROR "resolver conversion test arguments are incomplete") +endif() + +file(REMOVE_RECURSE "${test_root}") +execute_process( + COMMAND "${fixture_generator}" --write-fixture "${fixture}" + RESULT_VARIABLE fixture_result + ERROR_VARIABLE fixture_error) +if(NOT fixture_result EQUAL 0) + message(FATAL_ERROR + "Resolver COPC fixture generation failed: ${fixture_error}") +endif() + +set(cache_root "${test_root}/cache") +foreach(pass IN ITEMS first second) + set(output_directory "${test_root}/${pass}") + execute_process( + COMMAND "${CMAKE_COMMAND}" -E env + "PXR_AR_DEFAULT_RESOLVER=HttpResolver" + "PXR_PLUGINPATH_NAME=${resolver_plugin_path}" + "USDGEOCOPC_TEST_ASSET=${fixture}" + "${converter}" "http://memory.copc" + "${output_directory}/PointCloud.usda" + --tile-size 1 --memory-limit 1024 + --cache-root "${cache_root}" + RESULT_VARIABLE convert_result + OUTPUT_VARIABLE convert_output + ERROR_VARIABLE convert_error) + if(NOT convert_result EQUAL 0 OR NOT convert_error STREQUAL "" OR + NOT EXISTS "${output_directory}/PointCloud.usda" OR + NOT EXISTS "${output_directory}/payloads/tiles.manifest") + message(FATAL_ERROR + "Resolver COPC ${pass} conversion failed: ${convert_output}${convert_error}") + endif() + if(pass STREQUAL "first") + string(FIND "${convert_output}" + "Cache lookups: 1, hits: 0, misses: 1" cache_stats_position) + else() + string(FIND "${convert_output}" + "Cache lookups: 1, hits: 1, misses: 0" cache_stats_position) + endif() + if(cache_stats_position EQUAL -1) + message(FATAL_ERROR + "Resolver COPC ${pass} has unexpected cache statistics: ${convert_output}") + endif() + + file(READ "${output_directory}/PointCloud.usda.manifest" manifest) + string(FIND "${manifest}" "memory.copc" identifier_position) + string(FIND "${manifest}" "test-fnv1a64" token_position) + if(NOT identifier_position EQUAL -1 OR NOT token_position EQUAL -1) + message(FATAL_ERROR + "Resolver identifier or validation token leaked into manifest") + endif() +endforeach() + +set(unstable_output_directory "${test_root}/unstable") +execute_process( + COMMAND "${CMAKE_COMMAND}" -E env + "PXR_AR_DEFAULT_RESOLVER=HttpResolver" + "PXR_PLUGINPATH_NAME=${resolver_plugin_path}" + "USDGEOCOPC_TEST_ASSET=${fixture}" + "USDGEOCOPC_TEST_IDENTITY=unstable" + "${converter}" "http://memory.copc" + "${unstable_output_directory}/PointCloud.usda" + --tile-size 1 --memory-limit 1024 + --cache-root "${cache_root}" + RESULT_VARIABLE unstable_result + OUTPUT_VARIABLE unstable_output + ERROR_VARIABLE unstable_error) +if(NOT unstable_result EQUAL 0 OR NOT unstable_error STREQUAL "" OR + NOT EXISTS "${unstable_output_directory}/PointCloud.usda" OR + NOT EXISTS "${unstable_output_directory}/payloads/tiles.manifest") + message(FATAL_ERROR + "Unstable resolver COPC conversion failed: ${unstable_output}${unstable_error}") +endif() +string(FIND "${unstable_output}" "Cache lookups:" unstable_cache_position) +if(NOT unstable_cache_position EQUAL -1) + message(FATAL_ERROR + "Unstable resolver identity unexpectedly enabled generated cache") +endif() From 813865caf3b4a9c7242d08a6ea83759d75f18715 Mon Sep 17 00:00:00 2001 From: snkmcb Date: Sat, 5 Sep 2026 00:00:55 +0900 Subject: [PATCH 2/2] fix: redact resolver identifiers from converter errors --- tools/usd-pointcloud-convert/main.cpp | 26 ++++++++++++++----- .../test_resolver_conversion.cmake | 19 ++++++++++++++ 2 files changed, 38 insertions(+), 7 deletions(-) diff --git a/tools/usd-pointcloud-convert/main.cpp b/tools/usd-pointcloud-convert/main.cpp index 7ea6597..d439bb4 100644 --- a/tools/usd-pointcloud-convert/main.cpp +++ b/tools/usd-pointcloud-convert/main.cpp @@ -148,6 +148,7 @@ std::unique_ptr OpenPointStream( struct ConverterSource { std::string identifier; + std::string resolvedPath; std::string sourceName; std::filesystem::path localPath; std::shared_ptr asset; @@ -159,7 +160,19 @@ struct ConverterSource { }; bool IsResolverIdentifier(const std::string& identifier) { - return identifier.find("://") != std::string::npos; + const auto colon = identifier.find(':'); + if (colon == std::string::npos) { + return false; + } + if (colon == 1 && + ((identifier[0] >= 'A' && identifier[0] <= 'Z') || + (identifier[0] >= 'a' && identifier[0] <= 'z')) && + identifier.size() > 2 && + (identifier[2] == '/' || identifier[2] == '\\')) { + return false; + } + const auto pathSeparator = identifier.find_first_of("/\\"); + return pathSeparator == std::string::npos || colon < pathSeparator; } std::filesystem::path FormatPath(const std::string& identifier) { @@ -195,18 +208,17 @@ bool PrepareSource(const std::string& identifier, } const auto resolvedPath = pxr::ArGetResolver().Resolve(identifier); if (resolvedPath.GetPathString().empty()) { - errorMessage = "active resolver could not resolve COPC asset: " + - identifier; + errorMessage = "active resolver could not resolve COPC asset"; return false; } source.asset = pxr::ArGetResolver().OpenAsset(resolvedPath); if (!source.asset) { - errorMessage = "active resolver could not open COPC asset: " + - resolvedPath.GetPathString(); + errorMessage = "active resolver could not open COPC asset"; return false; } source.resolverBacked = true; - source.sourceName = resolvedPath.GetPathString(); + source.resolvedPath = resolvedPath.GetPathString(); + source.sourceName = "resolver-addressed-source"; source.randomAccessSource = std::make_shared( source.asset, source.sourceName); @@ -227,7 +239,7 @@ bool RefreshSourceIdentity(const ConverterSource& source, usdgeo::cache::ResolverIdentityStability stability; return usdgeo::TryBuildResolverSourceIdentity( pxr::ArGetResolver(), source.identifier, - pxr::ArResolvedPath(source.sourceName), *source.asset, identity, + pxr::ArResolvedPath(source.resolvedPath), *source.asset, identity, stability, errorMessage); } diff --git a/tools/usd-pointcloud-convert/test_resolver_conversion.cmake b/tools/usd-pointcloud-convert/test_resolver_conversion.cmake index fca0a96..838957b 100644 --- a/tools/usd-pointcloud-convert/test_resolver_conversion.cmake +++ b/tools/usd-pointcloud-convert/test_resolver_conversion.cmake @@ -81,3 +81,22 @@ if(NOT unstable_cache_position EQUAL -1) message(FATAL_ERROR "Unstable resolver identity unexpectedly enabled generated cache") endif() + +execute_process( + COMMAND "${CMAKE_COMMAND}" -E env + "PXR_AR_DEFAULT_RESOLVER=HttpResolver" + "PXR_PLUGINPATH_NAME=${resolver_plugin_path}" + "USDGEOCOPC_TEST_ASSET=${fixture}" + "${converter}" "http://missing.copc?signature=secret-token" + "${test_root}/missing/PointCloud.usda" + --tile-size 1 + RESULT_VARIABLE missing_result + OUTPUT_VARIABLE missing_output + ERROR_VARIABLE missing_error) +if(missing_result EQUAL 0) + message(FATAL_ERROR "Unresolvable resolver COPC unexpectedly converted") +endif() +string(FIND "${missing_error}" "secret-token" secret_position) +if(NOT secret_position EQUAL -1) + message(FATAL_ERROR "Resolver identifier leaked into an error message") +endif()