From 775fd593aad88a40f1fc49710edb905c620dbe57 Mon Sep 17 00:00:00 2001 From: Victor Zhang Date: Fri, 27 Mar 2026 12:46:40 -0700 Subject: [PATCH] Add CBOR trace parser helper and smoke test for compression trace error testing (#557) Summary: Add test infrastructure for E2E error propagation testing in compression tracing: - `TraceTestHelpers.hpp`: Header-only CBOR trace parser that decodes trace output from `CCtx::getLatestTrace()` into structured C++ types (`ParsedTrace`, `ParsedChunk`, `ParsedCodec`, `ParsedGraph`). Uses the a1cbor decode API with a test arena allocator. - `CompressTraceErrorTest.cpp`: Smoke test verifying the parse helper works on a real successful compression trace. - `COMPRESS_TRACE_ERROR_TEST_PLAN.md`: Plan document for implementing error propagation tests covering codec failures, graph failures, type conversion failures, segmenter failures, and deduplication logic. Differential Revision: D97033481 Pulled By: Victor-C-Zhang --- .../trace/CompressTraceErrorTest.cpp | 54 ++++ .../experimental/trace/StreamdumpTest.cpp | 4 +- .../experimental/trace/TraceTestHelpers.hpp | 231 ++++++++++++++++++ 3 files changed, 288 insertions(+), 1 deletion(-) create mode 100644 cpp/tests/experimental/trace/CompressTraceErrorTest.cpp create mode 100644 cpp/tests/experimental/trace/TraceTestHelpers.hpp diff --git a/cpp/tests/experimental/trace/CompressTraceErrorTest.cpp b/cpp/tests/experimental/trace/CompressTraceErrorTest.cpp new file mode 100644 index 000000000..ce79fe3c4 --- /dev/null +++ b/cpp/tests/experimental/trace/CompressTraceErrorTest.cpp @@ -0,0 +1,54 @@ +// (c) Meta Platforms, Inc. and affiliates. Confidential and proprietary. + +#include "openzl/zl_config.h" + +#if ZL_ALLOW_INTROSPECTION + +# include + +# include "cpp/tests/experimental/trace/TraceTestHelpers.hpp" +# include "openzl/cpp/CCtx.hpp" +# include "openzl/cpp/Compressor.hpp" + +using namespace ::testing; + +namespace openzl { + +// Smoke test: verify TraceTestHelpers can parse a successful compression trace. +TEST(CompressTraceErrorTest, ParseHelperSmokeTest) +{ + Compressor compressor; + compressor.setParameter(CParam::FormatVersion, ZL_MAX_FORMAT_VERSION); + compressor.selectStartingGraph(ZL_GRAPH_COMPRESS_GENERIC); + + std::vector data(1000, 42); + auto input = Input::refNumeric(data.data(), sizeof(int64_t), data.size()); + + CCtx cctx; + cctx.refCompressor(compressor); + cctx.writeTraces(true); + + auto compressed = cctx.compressOne(input); + auto traceResult = cctx.getLatestTrace(); + + ASSERT_FALSE(traceResult.first.empty()); + + auto parsed = test::parseTrace(traceResult.first); + ASSERT_TRUE(parsed.has_value()) << "Failed to parse trace CBOR"; + + EXPECT_EQ(parsed->operationType, 0) << "Should be compression"; + EXPECT_FALSE(parsed->chunks.empty()) << "Should have at least one chunk"; + + // At least one chunk should have codecs + bool foundCodecs = false; + for (const auto& chunk : parsed->chunks) { + if (!chunk.codecs.empty()) { + foundCodecs = true; + } + } + EXPECT_TRUE(foundCodecs) << "Should have codecs in at least one chunk"; +} + +} // namespace openzl + +#endif // ZL_ALLOW_INTROSPECTION diff --git a/cpp/tests/experimental/trace/StreamdumpTest.cpp b/cpp/tests/experimental/trace/StreamdumpTest.cpp index 81c434ad2..21a3f8ba8 100644 --- a/cpp/tests/experimental/trace/StreamdumpTest.cpp +++ b/cpp/tests/experimental/trace/StreamdumpTest.cpp @@ -1,6 +1,8 @@ // Copyright (c) Meta Platforms, Inc. and affiliates. -#ifdef ZL_ALLOW_INTROSPECTION +#include "openzl/zl_config.h" + +#if ZL_ALLOW_INTROSPECTION # include # include "openzl/cpp/CCtx.hpp" diff --git a/cpp/tests/experimental/trace/TraceTestHelpers.hpp b/cpp/tests/experimental/trace/TraceTestHelpers.hpp new file mode 100644 index 000000000..9d3c4557b --- /dev/null +++ b/cpp/tests/experimental/trace/TraceTestHelpers.hpp @@ -0,0 +1,231 @@ +// (c) Meta Platforms, Inc. and affiliates. Confidential and proprietary. + +#pragma once + +#include "openzl/cpp/poly/StringView.hpp" +#include "openzl/shared/a1cbor.h" + +#include +#include +#include +#include +#include + +namespace openzl::test { + +// Simple parsed representations of trace CBOR data for test assertions. + +struct ParsedCodec { + size_t chunkId{}; + std::string name; + bool cType{}; + int64_t cID{}; + int64_t cHeaderSize{}; + std::string cFailureString; // empty if not present + std::vector inputStreams; + std::vector outputStreams; +}; + +struct ParsedGraph { + size_t chunkId{}; + int64_t gType{}; + std::string gName; + std::string gFailureString; // empty if not present + std::vector codecIDs; +}; + +struct ParsedChunk { + size_t chunkId{}; + std::vector codecs; + std::vector graphs; +}; + +struct ParsedTrace { + int64_t libraryVersion{}; + int64_t frameVersion{}; + int64_t traceVersion{}; + int64_t operationType{}; + std::vector chunks; +}; + +namespace detail { + +// Arena allocator backed by a vector of unique_ptrs, matching the pattern +// used in test_a1cbor.cpp. +using Ptrs = std::vector>; + +inline void* testCalloc(void* opaque, size_t bytes) noexcept +{ + if (bytes == 0) { + return nullptr; + } + // Use nothrow new to avoid throwing in a noexcept function. + // The () value-initializes (zeros) the array, matching calloc semantics. + auto* raw = new (std::nothrow) uint8_t[bytes](); + if (raw == nullptr) { + return nullptr; + } + auto* ptrs = static_cast(opaque); + ptrs->push_back(std::unique_ptr(raw)); + return raw; +} + +inline std::string extractString(const A1C_Item* item) +{ + if (item == nullptr || item->type != A1C_ItemType_string) { + return ""; + } + return std::string(item->string.data, item->string.size); +} + +inline int64_t extractInt(const A1C_Item* item, int64_t defaultVal = 0) +{ + if (item == nullptr || item->type != A1C_ItemType_int64) { + return defaultVal; + } + return item->int64; +} + +inline bool extractBool(const A1C_Item* item, bool defaultVal = false) +{ + if (item == nullptr || item->type != A1C_ItemType_boolean) { + return defaultVal; + } + return item->boolean; +} + +inline std::vector extractIntArray(const A1C_Item* item) +{ + std::vector result; + if (item == nullptr || item->type != A1C_ItemType_array) { + return result; + } + for (size_t i = 0; i < item->array.size; ++i) { + A1C_Item* elem = A1C_Array_get(&item->array, i); + if (elem != nullptr && elem->type == A1C_ItemType_int64) { + result.push_back(elem->int64); + } + } + return result; +} + +inline ParsedCodec parseCodec(const A1C_Item* item) +{ + ParsedCodec codec; + if (item == nullptr || item->type != A1C_ItemType_map) { + return codec; + } + const A1C_Map* m = &item->map; + codec.chunkId = + static_cast(extractInt(A1C_Map_get_cstr(m, "chunkId"))); + codec.name = extractString(A1C_Map_get_cstr(m, "name")); + codec.cType = extractBool(A1C_Map_get_cstr(m, "cType")); + codec.cID = extractInt(A1C_Map_get_cstr(m, "cID")); + codec.cHeaderSize = extractInt(A1C_Map_get_cstr(m, "cHeaderSize")); + codec.cFailureString = extractString(A1C_Map_get_cstr(m, "cFailureString")); + codec.inputStreams = extractIntArray(A1C_Map_get_cstr(m, "inputStreams")); + codec.outputStreams = extractIntArray(A1C_Map_get_cstr(m, "outputStreams")); + return codec; +} + +inline ParsedGraph parseGraph(const A1C_Item* item) +{ + ParsedGraph graph; + if (item == nullptr || item->type != A1C_ItemType_map) { + return graph; + } + const A1C_Map* m = &item->map; + graph.chunkId = + static_cast(extractInt(A1C_Map_get_cstr(m, "chunkId"))); + graph.gType = extractInt(A1C_Map_get_cstr(m, "gType")); + graph.gName = extractString(A1C_Map_get_cstr(m, "gName")); + graph.gFailureString = extractString(A1C_Map_get_cstr(m, "gFailureString")); + graph.codecIDs = extractIntArray(A1C_Map_get_cstr(m, "codecIDs")); + return graph; +} + +inline ParsedChunk parseChunk(const A1C_Item* item) +{ + ParsedChunk chunk; + if (item == nullptr || item->type != A1C_ItemType_map) { + return chunk; + } + const A1C_Map* m = &item->map; + chunk.chunkId = + static_cast(extractInt(A1C_Map_get_cstr(m, "chunkId"))); + + const A1C_Item* codecsItem = A1C_Map_get_cstr(m, "codecs"); + if (codecsItem != nullptr && codecsItem->type == A1C_ItemType_array) { + for (size_t i = 0; i < codecsItem->array.size; ++i) { + chunk.codecs.push_back( + parseCodec(A1C_Array_get(&codecsItem->array, i))); + } + } + + const A1C_Item* graphsItem = A1C_Map_get_cstr(m, "graphs"); + if (graphsItem != nullptr && graphsItem->type == A1C_ItemType_array) { + for (size_t i = 0; i < graphsItem->array.size; ++i) { + chunk.graphs.push_back( + parseGraph(A1C_Array_get(&graphsItem->array, i))); + } + } + + return chunk; +} + +} // namespace detail + +/** + * Decodes a CBOR trace (as returned by CCtx::getLatestTrace().first) into a + * ParsedTrace struct for easy assertion in tests. + * + * Returns std::nullopt if decoding fails. + */ +inline std::optional parseTrace(poly::string_view traceData) +{ + if (traceData.empty()) { + return std::nullopt; + } + + detail::Ptrs ptrs; + A1C_Arena arena; + arena.calloc = detail::testCalloc; + arena.opaque = &ptrs; + + A1C_DecoderConfig config{}; + config.referenceSource = true; + + A1C_Decoder decoder; + A1C_Decoder_init(&decoder, arena, config); + + const A1C_Item* root = A1C_Decoder_decode( + &decoder, + reinterpret_cast(traceData.data()), + traceData.size()); + if (root == nullptr || root->type != A1C_ItemType_map) { + return std::nullopt; + } + + ParsedTrace trace; + const A1C_Map* m = &root->map; + trace.libraryVersion = + detail::extractInt(A1C_Map_get_cstr(m, "libraryVersion")); + trace.frameVersion = + detail::extractInt(A1C_Map_get_cstr(m, "frameVersion")); + trace.traceVersion = + detail::extractInt(A1C_Map_get_cstr(m, "traceVersion")); + trace.operationType = + detail::extractInt(A1C_Map_get_cstr(m, "operationType")); + + const A1C_Item* chunksItem = A1C_Map_get_cstr(m, "chunks"); + if (chunksItem != nullptr && chunksItem->type == A1C_ItemType_array) { + for (size_t i = 0; i < chunksItem->array.size; ++i) { + trace.chunks.push_back( + detail::parseChunk(A1C_Array_get(&chunksItem->array, i))); + } + } + + return trace; +} + +} // namespace openzl::test