From f93a9b84a9e7193c19506aaee55080ed59d61c45 Mon Sep 17 00:00:00 2001 From: Matt Millett Date: Fri, 2 Oct 2026 09:01:38 -0400 Subject: [PATCH 1/3] Harden compression and decompression against memory amplification and malformed blocks --- groups/ntc/ntca/ntca_compressionconfig.cpp | 28 +- groups/ntc/ntca/ntca_compressionconfig.h | 72 +- groups/ntc/ntcd/ntcd_compression.cpp | 167 ++++- groups/ntc/ntcd/ntcd_compression.h | 41 +- groups/ntc/ntcd/ntcd_compression.t.cpp | 618 ++++++++++++++++ groups/ntc/ntci/ntci_compression.cpp | 4 + groups/ntc/ntctlc/ntctlc_plugin.cpp | 790 +++++++++++++++++++-- groups/ntc/ntctlc/ntctlc_plugin.t.cpp | 657 +++++++++++++++++ 8 files changed, 2274 insertions(+), 103 deletions(-) diff --git a/groups/ntc/ntca/ntca_compressionconfig.cpp b/groups/ntc/ntca/ntca_compressionconfig.cpp index ffd9046a..ff7bb4a7 100644 --- a/groups/ntc/ntca/ntca_compressionconfig.cpp +++ b/groups/ntc/ntca/ntca_compressionconfig.cpp @@ -25,7 +25,9 @@ namespace ntca { bool CompressionConfig::equals(const CompressionConfig& other) const { - return (d_type == other.d_type && d_goal == other.d_goal); + return (d_type == other.d_type && d_goal == other.d_goal && + d_maxInflateSize == other.d_maxInflateSize && + d_maxDeflateSize == other.d_maxDeflateSize); } bool CompressionConfig::less(const CompressionConfig& other) const @@ -38,7 +40,23 @@ bool CompressionConfig::less(const CompressionConfig& other) const return false; } - return d_goal < other.d_goal; + if (d_goal < other.d_goal) { + return true; + } + + if (other.d_goal < d_goal) { + return false; + } + + if (d_maxInflateSize < other.d_maxInflateSize) { + return true; + } + + if (other.d_maxInflateSize < d_maxInflateSize) { + return false; + } + + return d_maxDeflateSize < other.d_maxDeflateSize; } bsl::ostream& CompressionConfig::print(bsl::ostream& stream, @@ -53,6 +71,12 @@ bsl::ostream& CompressionConfig::print(bsl::ostream& stream, if (d_goal.has_value()) { printer.printAttribute("goal", d_goal); } + if (d_maxInflateSize.has_value()) { + printer.printAttribute("maxInflateSize", d_maxInflateSize); + } + if (d_maxDeflateSize.has_value()) { + printer.printAttribute("maxDeflateSize", d_maxDeflateSize); + } printer.end(); return stream; } diff --git a/groups/ntc/ntca/ntca_compressionconfig.h b/groups/ntc/ntca/ntca_compressionconfig.h index 44959a70..c3dad672 100644 --- a/groups/ntc/ntca/ntca_compressionconfig.h +++ b/groups/ntc/ntca/ntca_compressionconfig.h @@ -45,6 +45,20 @@ namespace ntca { /// or speed over size. If not specified, the default value is a balanced goal /// that favors neither size nor speed. /// +/// @li @b maxInflateSize: +/// The maximum number of bytes appended to the result of a single inflate +/// operation. Inflate operations that would produce more than this number of +/// bytes fail with 'ntsa::Error::e_LIMIT'. If not specified, the limit is +/// INT_MAX. Note that this limit is currently honored only by the "RLE" +/// algorithm. +/// +/// @li @b maxDeflateSize: +/// The maximum number of bytes appended to the result of a single deflate +/// operation. Deflate operations that would produce more than this number of +/// bytes fail with 'ntsa::Error::e_LIMIT'. If not specified, the limit is +/// INT_MAX. Note that this limit is currently honored only by the "RLE" +/// algorithm. +/// /// @par Thread Safety /// This class is not thread safe. /// @@ -53,6 +67,8 @@ class CompressionConfig { bdlb::NullableValue d_type; bdlb::NullableValue d_goal; + bdlb::NullableValue d_maxInflateSize; + bdlb::NullableValue d_maxDeflateSize; public: /// Create new deflate options having the default value. @@ -78,12 +94,28 @@ class CompressionConfig /// Set the compression goal to the specified 'value'. void setGoal(ntca::CompressionGoal::Value value); + /// Set the maximum number of bytes appended to the result of a single + /// inflate operation to the specified 'value'. + void setMaxInflateSize(bsl::size_t value); + + /// Set the maximum number of bytes appended to the result of a single + /// deflate operation to the specified 'value'. + void setMaxDeflateSize(bsl::size_t value); + /// Return the compression algorithm. const bdlb::NullableValue& type() const; /// Return the compression goal. const bdlb::NullableValue& goal() const; + /// Return the maximum number of bytes appended to the result of a single + /// inflate operation. + const bdlb::NullableValue& maxInflateSize() const; + + /// Return the maximum number of bytes appended to the result of a single + /// deflate operation. + const bdlb::NullableValue& maxDeflateSize() const; + /// Return true if this object has the same value as the specified 'other' /// object, otherwise return false. bool equals(const CompressionConfig& other) const; @@ -158,6 +190,8 @@ NTSCFG_INLINE CompressionConfig::CompressionConfig() : d_type() , d_goal() +, d_maxInflateSize() +, d_maxDeflateSize() { } @@ -165,6 +199,8 @@ NTSCFG_INLINE CompressionConfig::CompressionConfig(const CompressionConfig& original) : d_type(original.d_type) , d_goal(original.d_goal) +, d_maxInflateSize(original.d_maxInflateSize) +, d_maxDeflateSize(original.d_maxDeflateSize) { } @@ -176,8 +212,10 @@ CompressionConfig::~CompressionConfig() NTSCFG_INLINE CompressionConfig& CompressionConfig::operator=(const CompressionConfig& other) { - d_goal = other.d_goal; - d_type = other.d_type; + d_goal = other.d_goal; + d_type = other.d_type; + d_maxInflateSize = other.d_maxInflateSize; + d_maxDeflateSize = other.d_maxDeflateSize; return *this; } @@ -186,6 +224,8 @@ void CompressionConfig::reset() { d_type.reset(); d_goal.reset(); + d_maxInflateSize.reset(); + d_maxDeflateSize.reset(); } NTSCFG_INLINE @@ -200,6 +240,18 @@ void CompressionConfig::setGoal(ntca::CompressionGoal::Value value) d_goal = value; } +NTSCFG_INLINE +void CompressionConfig::setMaxInflateSize(bsl::size_t value) +{ + d_maxInflateSize = value; +} + +NTSCFG_INLINE +void CompressionConfig::setMaxDeflateSize(bsl::size_t value) +{ + d_maxDeflateSize = value; +} + NTSCFG_INLINE const bdlb::NullableValue& CompressionConfig:: type() const @@ -214,12 +266,28 @@ const bdlb::NullableValue& CompressionConfig:: return d_goal; } +NTSCFG_INLINE +const bdlb::NullableValue& CompressionConfig::maxInflateSize() + const +{ + return d_maxInflateSize; +} + +NTSCFG_INLINE +const bdlb::NullableValue& CompressionConfig::maxDeflateSize() + const +{ + return d_maxDeflateSize; +} + template NTSCFG_INLINE void CompressionConfig::hash(HASH_ALGORITHM& algorithm) const { using bslh::hashAppend; hashAppend(algorithm, d_type); hashAppend(algorithm, d_goal); + hashAppend(algorithm, d_maxInflateSize); + hashAppend(algorithm, d_maxDeflateSize); } NTSCFG_INLINE diff --git a/groups/ntc/ntcd/ntcd_compression.cpp b/groups/ntc/ntcd/ntcd_compression.cpp index 5cdba40f..d9e55070 100644 --- a/groups/ntc/ntcd/ntcd_compression.cpp +++ b/groups/ntc/ntcd/ntcd_compression.cpp @@ -43,6 +43,7 @@ BSLS_IDENT_RCSID(ntcd_compression_cpp, "$Id$ $CSID$") #include #include +#include #include #include #include @@ -635,6 +636,39 @@ bsl::ostream& CompressionBlock::print(bsl::ostream& stream, << NTCI_LOG_STREAM_END; \ } while (false) +ntsa::Error CompressionEncoder::checkLimit( + const ntca::DeflateContext& context, + const bdlbb::Blob& result, + bsl::size_t size) const +{ + NTCI_LOG_CONTEXT(); + + const bsl::size_t numBytesDeflated = context.bytesWritten(); + + if (numBytesDeflated > d_maxDeflateSize || + size > d_maxDeflateSize - numBytesDeflated) + { + NTCD_COMPRESSION_ENCODER_RLE_LOG_ERROR( + "The operation deflates beyond the maximum size", + ntsa::Error(ntsa::Error::e_LIMIT)); + return ntsa::Error(ntsa::Error::e_LIMIT); + } + + BSLS_ASSERT(result.length() >= 0); + + if (size > static_cast(INT_MAX - result.length())) { + NTCD_COMPRESSION_ENCODER_RLE_LOG_ERROR( + "The operation deflates beyond the maximum blob length", + ntsa::Error(ntsa::Error::e_LIMIT)); + return ntsa::Error(ntsa::Error::e_LIMIT); + } + + return ntsa::Error(); +} + +const bsl::size_t CompressionEncoder::k_DEFAULT_MAX_DEFLATE_SIZE = + static_cast(INT_MAX); + CompressionEncoder::CompressionEncoder( const ntca::CompressionConfig& configuration, bslma::Allocator* basicAllocator) @@ -642,6 +676,8 @@ CompressionEncoder::CompressionEncoder( , d_frameHeaderPosition(0) , d_frameContentBytesTotal(0) , d_frameContentCrc(ntca::ChecksumType::e_CRC32) +, d_maxDeflateSize( + configuration.maxDeflateSize().value_or(k_DEFAULT_MAX_DEFLATE_SIZE)) , d_config(configuration) , d_allocator_p(bslma::Default::allocator(basicAllocator)) { @@ -671,6 +707,11 @@ ntsa::Error CompressionEncoder::deflateBegin( d_frameContentBytesTotal = 0; d_frameContentCrc.reset(ntca::ChecksumType::e_CRC32); + error = this->checkLimit(*context, *result, sizeof d_frameHeader); + if (error) { + return error; + } + bsl::size_t frameHeaderBytesEncoded = 0; error = d_frameHeader.encode(&frameHeaderBytesEncoded, result); if (error) { @@ -735,6 +776,11 @@ ntsa::Error CompressionEncoder::deflateNext( NTCD_COMPRESSION_ENCODER_RLE_LOG_BLOCK_RLE(block); + error = this->checkLimit(*context, *result, sizeof block); + if (error) { + return error; + } + bsl::size_t blockBytesEncoded = 0; error = block.encode(&blockBytesEncoded, result); if (error) { @@ -761,6 +807,13 @@ ntsa::Error CompressionEncoder::deflateNext( block, bsl::string_view((const char*)(mark), block.length())); + error = this->checkLimit(*context, + *result, + sizeof block + block.length()); + if (error) { + return error; + } + bsl::size_t blockBytesEncoded = 0; error = block.encode(&blockBytesEncoded, result); if (error) { @@ -798,6 +851,11 @@ ntsa::Error CompressionEncoder::deflateNext( NTCD_COMPRESSION_ENCODER_RLE_LOG_BLOCK_RLE(block); + error = this->checkLimit(*context, *result, sizeof block); + if (error) { + return error; + } + bsl::size_t blockBytesEncoded = 0; error = block.encode(&blockBytesEncoded, result); if (error) { @@ -816,8 +874,18 @@ ntsa::Error CompressionEncoder::deflateNext( block, bsl::string_view((const char*)(mark), block.length())); + error = this->checkLimit(*context, + *result, + sizeof block + block.length()); + if (error) { + return error; + } + bsl::size_t blockBytesEncoded = 0; - block.encode(&blockBytesEncoded, result); + error = block.encode(&blockBytesEncoded, result); + if (error) { + return error; + } d_frameContentBytesTotal += blockBytesEncoded; context->setBytesWritten(context->bytesWritten() + blockBytesEncoded); @@ -869,6 +937,11 @@ ntsa::Error CompressionEncoder::deflateEnd(ntca::DeflateContext* context, ntcd::CompressionFrameFooter frameFooter; frameFooter.setChecksum(d_frameContentCrc); + error = this->checkLimit(*context, *result, sizeof frameFooter); + if (error) { + return error; + } + error = frameFooter.encode(&frameFooterBytesEncoded, result); if (error) { return error; @@ -970,6 +1043,13 @@ ntsa::Error CompressionDecoder::process(ntca::InflateContext* context, continue; } else if (d_state == e_WANT_BLOCK) { + if (d_frameContentBytesNeeded < sizeof(ntcd::CompressionBlock)) { + NTCD_COMPRESSION_DECODER_RLE_LOG_ERROR( + "The frame content is too short to contain a block", + ntsa::Error(ntsa::Error::e_INVALID)); + return this->fail(ntsa::Error(ntsa::Error::e_INVALID)); + } + bsl::size_t numBytesDecoded = 0; error = d_block.decode(&numBytesDecoded, d_input); if (error) { @@ -989,6 +1069,15 @@ ntsa::Error CompressionDecoder::process(ntca::InflateContext* context, ntcs::BlobUtil::pop(&d_input, numBytesDecoded); d_frameContentBytesNeeded -= numBytesDecoded; + if (d_block.type() == ntcd::CompressionBlockType::e_RAW && + d_block.length() > d_frameContentBytesNeeded) + { + NTCD_COMPRESSION_DECODER_RLE_LOG_ERROR( + "The block payload exceeds the remaining frame content", + ntsa::Error(ntsa::Error::e_INVALID)); + return this->fail(ntsa::Error(ntsa::Error::e_INVALID)); + } + if (d_block.type() == ntcd::CompressionBlockType::e_RAW) { if (d_block.length() > 0) { d_state = e_WANT_BLOCK_PAYLOAD; @@ -997,6 +1086,14 @@ ntsa::Error CompressionDecoder::process(ntca::InflateContext* context, } else if (d_block.type() == ntcd::CompressionBlockType::e_RLE) { if (d_block.length() > 0) { + error = this->checkLimit(*context, + *result, + numBytesWritten, + d_block.length()); + if (error) { + return this->fail(error); + } + d_expansion.clear(); d_expansion.resize(d_block.length(), d_block.literal()); @@ -1007,7 +1104,7 @@ ntsa::Error CompressionDecoder::process(ntca::InflateContext* context, error = d_frameContentCrc.update(&d_expansion[0], d_expansion.size()); if (error) { - return error; + return this->fail(error); } d_expansion.clear(); @@ -1035,6 +1132,14 @@ ntsa::Error CompressionDecoder::process(ntca::InflateContext* context, } else if (d_state == e_WANT_BLOCK_PAYLOAD) { if (d_input.length() >= d_block.length()) { + error = this->checkLimit(*context, + *result, + numBytesWritten, + d_block.length()); + if (error) { + return this->fail(error); + } + bdlbb::BlobUtil::append(result, d_input, 0, @@ -1042,10 +1147,12 @@ ntsa::Error CompressionDecoder::process(ntca::InflateContext* context, error = d_frameContentCrc.update(d_input, d_block.length()); if (error) { - return error; + return this->fail(error); } ntcs::BlobUtil::pop(&d_input, d_block.length()); + + BSLS_ASSERT(d_block.length() <= d_frameContentBytesNeeded); d_frameContentBytesNeeded -= d_block.length(); numBytesWritten += d_block.length(); @@ -1106,17 +1213,6 @@ ntsa::Error CompressionDecoder::process(ntca::InflateContext* context, continue; } - else if (d_state == e_ERROR) { - if (d_error) { - return d_error; - } - else { - NTCD_COMPRESSION_DECODER_RLE_LOG_ERROR( - "The decoder is inoperable after a previous failure", - ntsa::Error(ntsa::Error::e_INVALID)); - return this->fail(ntsa::Error(ntsa::Error::e_INVALID)); - } - } else { NTCD_COMPRESSION_DECODER_RLE_LOG_ERROR( "The decoder is is an invalid state", @@ -1130,10 +1226,43 @@ ntsa::Error CompressionDecoder::process(ntca::InflateContext* context, return ntsa::Error(); } +ntsa::Error CompressionDecoder::checkLimit( + const ntca::InflateContext& context, + const bdlbb::Blob& result, + bsl::size_t numBytesWritten, + bsl::size_t size) const +{ + NTCI_LOG_CONTEXT(); + + const bsl::size_t numBytesInflated = + context.bytesWritten() + numBytesWritten; + + if (numBytesInflated > d_maxInflateSize || + size > d_maxInflateSize - numBytesInflated) + { + NTCD_COMPRESSION_DECODER_RLE_LOG_ERROR( + "The operation inflates beyond the maximum size", + ntsa::Error(ntsa::Error::e_LIMIT)); + return ntsa::Error(ntsa::Error::e_LIMIT); + } + + BSLS_ASSERT(result.length() >= 0); + + if (size > static_cast(INT_MAX - result.length())) { + NTCD_COMPRESSION_DECODER_RLE_LOG_ERROR( + "The operation inflates beyond the maximum blob length", + ntsa::Error(ntsa::Error::e_LIMIT)); + return ntsa::Error(ntsa::Error::e_LIMIT); + } + + return ntsa::Error(); +} + ntsa::Error CompressionDecoder::fail(ntsa::Error error) { - d_state = e_ERROR; - d_error = error; + d_state = e_WANT_FRAME_HEADER; + + bdlbb::BlobUtil::erase(&d_input, 0, d_input.length()); d_expansion.clear(); d_frameHeader.reset(); @@ -1144,6 +1273,9 @@ ntsa::Error CompressionDecoder::fail(ntsa::Error error) return error; } +const bsl::size_t CompressionDecoder::k_DEFAULT_MAX_INFLATE_SIZE = + static_cast(INT_MAX); + CompressionDecoder::CompressionDecoder( const ntca::CompressionConfig& configuration, bslma::Allocator* basicAllocator) @@ -1154,8 +1286,9 @@ CompressionDecoder::CompressionDecoder( , d_frameContentBytesNeeded(0) , d_frameContentCrc(ntca::ChecksumType::e_CRC32) , d_block() +, d_maxInflateSize( + configuration.maxInflateSize().value_or(k_DEFAULT_MAX_INFLATE_SIZE)) , d_config(configuration) -, d_error() , d_allocator_p(bslma::Default::allocator(basicAllocator)) { } diff --git a/groups/ntc/ntcd/ntcd_compression.h b/groups/ntc/ntcd/ntcd_compression.h index 0debe0dc..9f2423c3 100644 --- a/groups/ntc/ntcd/ntcd_compression.h +++ b/groups/ntc/ntcd/ntcd_compression.h @@ -602,6 +602,7 @@ class CompressionEncoder bsl::size_t d_frameHeaderPosition; bsl::size_t d_frameContentBytesTotal; ntca::Checksum d_frameContentCrc; + bsl::size_t d_maxDeflateSize; ntca::CompressionConfig d_config; bslma::Allocator* d_allocator_p; @@ -610,6 +611,15 @@ class CompressionEncoder CompressionEncoder& operator=(const CompressionEncoder&) BSLS_KEYWORD_DELETED; + private: + /// Return an error if appending the specified 'size' bytes to the + /// specified 'result' would exceed the maximum number of bytes allowed + /// to be deflated by the operation described by the specified 'context', + /// or would exceed the maximum length of the 'result'. + ntsa::Error checkLimit(const ntca::DeflateContext& context, + const bdlbb::Blob& result, + bsl::size_t size) const; + public: /// Create a new RLE encoder with to the specified 'configuration'. /// Optionally specify a 'basicAllocator' used to supply memory. If @@ -640,6 +650,10 @@ class CompressionEncoder ntsa::Error deflateEnd(ntca::DeflateContext* context, bdlbb::Blob* result, const ntca::DeflateOptions& options); + + /// The maximum number of bytes a single deflate operation may produce + /// when the maximum is not explicitly configured. + static const bsl::size_t k_DEFAULT_MAX_DEFLATE_SIZE; }; /// @internal @brief @@ -663,10 +677,7 @@ class CompressionDecoder e_WANT_BLOCK_PAYLOAD, /// The decoder wants to read the frame footer. - e_WANT_FRAME_FOOTER, - - /// The decoder encountered an error. - e_ERROR + e_WANT_FRAME_FOOTER }; State d_state; @@ -676,8 +687,8 @@ class CompressionDecoder bsl::size_t d_frameContentBytesNeeded; ntca::Checksum d_frameContentCrc; ntcd::CompressionBlock d_block; + bsl::size_t d_maxInflateSize; ntca::CompressionConfig d_config; - ntsa::Error d_error; bslma::Allocator* d_allocator_p; private: @@ -692,7 +703,21 @@ class CompressionDecoder bdlbb::Blob* result, const ntca::InflateOptions& options); - /// Fail the decoder with the specified 'error'. Return the 'error'. + /// Return an error if appending the specified 'size' bytes to the + /// specified 'result', in addition to the specified 'numBytesWritten' + /// bytes already appended but not yet reflected in the specified + /// 'context', would exceed the maximum number of bytes allowed to be + /// inflated by the operation described by the 'context', or would exceed + /// the maximum length of the 'result'. + ntsa::Error checkLimit(const ntca::InflateContext& context, + const bdlbb::Blob& result, + bsl::size_t numBytesWritten, + bsl::size_t size) const; + + /// Fail the current inflate operation with the specified 'error'. + /// Discard any buffered input and partially-decoded frame, and reset the + /// decoder to expect a new frame, so that subsequent, well-formed frames + /// may be decoded. Return the 'error'. ntsa::Error fail(ntsa::Error error); public: @@ -732,6 +757,10 @@ class CompressionDecoder ntsa::Error inflateEnd(ntca::InflateContext* context, bdlbb::Blob* result, const ntca::InflateOptions& options); + + /// The maximum number of bytes a single inflate operation may produce + /// when the maximum is not explicitly configured. + static const bsl::size_t k_DEFAULT_MAX_INFLATE_SIZE; }; /// @internal @brief diff --git a/groups/ntc/ntcd/ntcd_compression.t.cpp b/groups/ntc/ntcd/ntcd_compression.t.cpp index 05995d96..7f2d3aa4 100644 --- a/groups/ntc/ntcd/ntcd_compression.t.cpp +++ b/groups/ntc/ntcd/ntcd_compression.t.cpp @@ -24,8 +24,11 @@ BSLS_IDENT_RCSID(ntcd_compression_t_cpp, "$Id$ $CSID$") #include #include #include +#include #include +#include + // Uncomment to define the single buffer size tested. // #define NTCD_COMPRESSION_TEST_BUFFER_SIZE 32 @@ -51,10 +54,51 @@ class CompressionTest // in chunks, simulating the arrival of deflated data over a stream. static void verifyStreaming(); + // Verify the inflater rejects malformed frames and blocks without + // underflowing its state or over-appending inflated data. + static void verifyMalformed(); + + // Verify the inflater and deflater honor the configured maximum number of + // bytes inflated and deflated per operation. + static void verifyLimits(); + private: // Verify compression using the specified testing 'parameters'. static void verifyParameters(const Parameters& parameters); + // Append to the specified 'result' a frame header declaring the + // specified frame content 'length' and the specified 'checksum'. + static void appendFrameHeader(bdlbb::Blob* result, + bsl::uint32_t length, + const ntca::Checksum& checksum); + + // Append to the specified 'result' a block having the specified 'type', + // 'length', and 'literal'. + static void appendBlock(bdlbb::Blob* result, + ntcd::CompressionBlockType::Value type, + bsl::uint16_t length, + bsl::uint8_t literal); + + // Append to the specified 'result' a frame footer having the specified + // 'checksum'. + static void appendFrameFooter(bdlbb::Blob* result, + const ntca::Checksum& checksum); + + // Load into the specified 'result' a frame whose content is a run of 100 + // 'a' characters, followed by "bcdef", followed by a run of 50 'z' + // characters, and load into the specified 'expected' the inflated content + // of that frame. + static void generateFrame(bdlbb::Blob* result, bdlbb::Blob* expected); + + // Inflate the specified 'input' into the specified 'output' using a new + // RLE inflater. Assert the inflation fails with the specified 'expected' + // error and that no more than the specified 'maxOutputSize' bytes are + // inflated. Assert the inflater resets after the failure such that a + // subsequent, well-formed frame is inflated successfully. + static void verifyInflateFailure(const bdlbb::Blob& input, + ntsa::Error::Code expected, + bsl::size_t maxOutputSize); + // Dump the specified 'blob' to the log with the specified 'label'. static void dump(const char* label, const bdlbb::Blob& blob); }; @@ -541,6 +585,155 @@ void CompressionTest::verifyParameters(const Parameters& parameters) NTSCFG_TEST_EQ(comparison, 0); } +void CompressionTest::appendFrameHeader(bdlbb::Blob* result, + bsl::uint32_t length, + const ntca::Checksum& checksum) +{ + ntcd::CompressionFrameHeader frameHeader; + frameHeader.setLength(length); + frameHeader.setChecksum(checksum); + + bsl::size_t numBytesEncoded = 0; + ntsa::Error error = frameHeader.encode(&numBytesEncoded, result); + NTSCFG_TEST_OK(error); +} + +void CompressionTest::appendBlock(bdlbb::Blob* result, + ntcd::CompressionBlockType::Value type, + bsl::uint16_t length, + bsl::uint8_t literal) +{ + ntcd::CompressionBlock block; + block.setType(type); + block.setLength(length); + block.setLiteral(literal); + + bsl::size_t numBytesEncoded = 0; + ntsa::Error error = block.encode(&numBytesEncoded, result); + NTSCFG_TEST_OK(error); +} + +void CompressionTest::appendFrameFooter(bdlbb::Blob* result, + const ntca::Checksum& checksum) +{ + ntcd::CompressionFrameFooter frameFooter; + frameFooter.setChecksum(checksum); + + bsl::size_t numBytesEncoded = 0; + ntsa::Error error = frameFooter.encode(&numBytesEncoded, result); + NTSCFG_TEST_OK(error); +} + +void CompressionTest::verifyInflateFailure(const bdlbb::Blob& input, + ntsa::Error::Code expected, + bsl::size_t maxOutputSize) +{ + ntsa::Error error; + + bdlbb::PooledBlobBufferFactory blobBufferFactory(256, + NTSCFG_TEST_ALLOCATOR); + + ntca::CompressionConfig compressionConfig; + compressionConfig.setType(ntca::CompressionType::e_RLE); + compressionConfig.setGoal(ntca::CompressionGoal::e_BALANCED); + + bsl::shared_ptr dataPool; + dataPool.createInplace(NTSCFG_TEST_ALLOCATOR, + 256, + 256, + NTSCFG_TEST_ALLOCATOR); + + ntcd::Compression compression(compressionConfig, + dataPool, + NTSCFG_TEST_ALLOCATOR); + + ntcd::CompressionTest::dump("M", input); + + bdlbb::Blob inflatedData(&blobBufferFactory, NTSCFG_TEST_ALLOCATOR); + + ntca::InflateOptions inflateOptions; + ntca::InflateContext inflateContext; + + error = compression.inflate(&inflateContext, + &inflatedData, + input, + inflateOptions); + NTSCFG_TEST_ERROR(error, expected); + + NTSCFG_TEST_LE(static_cast(inflatedData.length()), + maxOutputSize); + + // The inflater resets after the failure, so a subsequent, well-formed + // frame is inflated successfully. + + bdlbb::Blob validFrame(&blobBufferFactory, NTSCFG_TEST_ALLOCATOR); + { + ntca::Checksum checksum(ntca::ChecksumType::e_CRC32); + error = checksum.update("qqq", 3); + NTSCFG_TEST_OK(error); + + ntcd::CompressionTest::appendFrameHeader( + &validFrame, + static_cast(sizeof(ntcd::CompressionBlock)), + checksum); + ntcd::CompressionTest::appendBlock(&validFrame, + ntcd::CompressionBlockType::e_RLE, + 3, + 'q'); + ntcd::CompressionTest::appendFrameFooter(&validFrame, checksum); + } + + bdlbb::Blob validInflatedData(&blobBufferFactory, NTSCFG_TEST_ALLOCATOR); + + error = compression.inflate(&inflateContext, + &validInflatedData, + validFrame, + inflateOptions); + NTSCFG_TEST_OK(error); + + bdlbb::Blob validExpected(&blobBufferFactory, NTSCFG_TEST_ALLOCATOR); + ntcs::BlobUtil::append(&validExpected, "qqq", 3); + + NTSCFG_TEST_EQ(bdlbb::BlobUtil::compare(validInflatedData, validExpected), + 0); +} + +void CompressionTest::generateFrame(bdlbb::Blob* result, bdlbb::Blob* expected) +{ + ntsa::Error error; + + const bsl::string data = bsl::string(100, 'a') + "bcdef" + + bsl::string(50, 'z'); + + ntca::Checksum checksum(ntca::ChecksumType::e_CRC32); + error = checksum.update(data.data(), data.size()); + NTSCFG_TEST_OK(error); + + const bsl::size_t k_BLOCK_SIZE = sizeof(ntcd::CompressionBlock); + + CompressionTest::appendFrameHeader( + result, + static_cast(k_BLOCK_SIZE + k_BLOCK_SIZE + 5 + + k_BLOCK_SIZE), + checksum); + CompressionTest::appendBlock(result, + ntcd::CompressionBlockType::e_RLE, + 100, + 'a'); + CompressionTest::appendBlock(result, + ntcd::CompressionBlockType::e_RAW, + 5, + 0); + ntcs::BlobUtil::append(result, "bcdef", 5); + CompressionTest::appendBlock(result, + ntcd::CompressionBlockType::e_RLE, + 50, + 'z'); + CompressionTest::appendFrameFooter(result, checksum); + + ntcs::BlobUtil::append(expected, data.data(), data.size()); +} + void CompressionTest::dump(const char* label, const bdlbb::Blob& blob) { NTCI_LOG_CONTEXT(); @@ -705,5 +898,430 @@ NTSCFG_TEST_FUNCTION(ntcd::CompressionTest::verifyStreaming) } } +NTSCFG_TEST_FUNCTION(ntcd::CompressionTest::verifyMalformed) +{ + ntsa::Error error; + + bdlbb::PooledBlobBufferFactory blobBufferFactory(256, + NTSCFG_TEST_ALLOCATOR); + + const bsl::size_t k_BLOCK_SIZE = sizeof(ntcd::CompressionBlock); + const bsl::size_t k_MAX_BLOCK_LENGTH = 65535; + + // Concern: a well-formed, hand-crafted frame inflates successfully, + // validating the frame construction used by the cases below. + + { + const char k_DATA[] = "aaabc"; + const bsl::size_t k_DATA_SIZE = sizeof k_DATA - 1; + + ntca::Checksum checksum(ntca::ChecksumType::e_CRC32); + error = checksum.update(k_DATA, k_DATA_SIZE); + NTSCFG_TEST_OK(error); + + bdlbb::Blob input(&blobBufferFactory, NTSCFG_TEST_ALLOCATOR); + + CompressionTest::appendFrameHeader( + &input, + static_cast(k_BLOCK_SIZE + k_BLOCK_SIZE + 2), + checksum); + CompressionTest::appendBlock(&input, + ntcd::CompressionBlockType::e_RLE, + 3, + 'a'); + CompressionTest::appendBlock(&input, + ntcd::CompressionBlockType::e_RAW, + 2, + 0); + ntcs::BlobUtil::append(&input, "bc", 2); + CompressionTest::appendFrameFooter(&input, checksum); + + ntca::CompressionConfig compressionConfig; + compressionConfig.setType(ntca::CompressionType::e_RLE); + compressionConfig.setGoal(ntca::CompressionGoal::e_BALANCED); + + bsl::shared_ptr dataPool; + dataPool.createInplace(NTSCFG_TEST_ALLOCATOR, + 256, + 256, + NTSCFG_TEST_ALLOCATOR); + + ntcd::Compression compression(compressionConfig, + dataPool, + NTSCFG_TEST_ALLOCATOR); + + bdlbb::Blob inflatedData(&blobBufferFactory, NTSCFG_TEST_ALLOCATOR); + + ntca::InflateOptions inflateOptions; + ntca::InflateContext inflateContext; + + error = compression.inflate(&inflateContext, + &inflatedData, + input, + inflateOptions); + NTSCFG_TEST_OK(error); + + bdlbb::Blob expected(&blobBufferFactory, NTSCFG_TEST_ALLOCATOR); + ntcs::BlobUtil::append(&expected, k_DATA, k_DATA_SIZE); + + NTSCFG_TEST_EQ(bdlbb::BlobUtil::compare(inflatedData, expected), 0); + } + + // Concern: a frame whose declared content length is too short to contain + // even a single block is rejected before any data is inflated, and the + // blocks that follow are not inflated. + + for (bsl::uint32_t length = 1; length < k_BLOCK_SIZE; ++length) { + ntca::Checksum checksum(ntca::ChecksumType::e_CRC32); + + bdlbb::Blob input(&blobBufferFactory, NTSCFG_TEST_ALLOCATOR); + + CompressionTest::appendFrameHeader(&input, length, checksum); + for (bsl::size_t i = 0; i < 16; ++i) { + CompressionTest::appendBlock( + &input, + ntcd::CompressionBlockType::e_RLE, + static_cast(k_MAX_BLOCK_LENGTH), + 'x'); + } + CompressionTest::appendFrameFooter(&input, checksum); + + CompressionTest::verifyInflateFailure(input, + ntsa::Error::e_INVALID, + 0); + } + + // Concern: a frame whose declared content length leaves a remainder too + // short to contain a block after a valid block is rejected, and the + // blocks that follow are not inflated. + + for (bsl::uint32_t remainder = 1; remainder < k_BLOCK_SIZE; ++remainder) { + ntca::Checksum checksum(ntca::ChecksumType::e_CRC32); + + bdlbb::Blob input(&blobBufferFactory, NTSCFG_TEST_ALLOCATOR); + + CompressionTest::appendFrameHeader( + &input, + static_cast(k_BLOCK_SIZE + remainder), + checksum); + for (bsl::size_t i = 0; i < 16; ++i) { + CompressionTest::appendBlock( + &input, + ntcd::CompressionBlockType::e_RLE, + static_cast(k_MAX_BLOCK_LENGTH), + 'x'); + } + CompressionTest::appendFrameFooter(&input, checksum); + + CompressionTest::verifyInflateFailure(input, + ntsa::Error::e_INVALID, + k_MAX_BLOCK_LENGTH); + } + + // Concern: a raw block whose payload length exceeds the remaining frame + // content is rejected before its payload is inflated. + + { + const bsl::size_t k_FRAME_PAYLOAD_SIZE = 8; + + const bsl::size_t k_BLOCK_LENGTH[] = {k_FRAME_PAYLOAD_SIZE + 1, + 100, + k_MAX_BLOCK_LENGTH}; + + const bsl::size_t k_NUM_BLOCK_LENGTHS = + sizeof k_BLOCK_LENGTH / sizeof k_BLOCK_LENGTH[0]; + + for (bsl::size_t i = 0; i < k_NUM_BLOCK_LENGTHS; ++i) { + ntca::Checksum checksum(ntca::ChecksumType::e_CRC32); + + bdlbb::Blob input(&blobBufferFactory, NTSCFG_TEST_ALLOCATOR); + + CompressionTest::appendFrameHeader( + &input, + static_cast(k_BLOCK_SIZE + + k_FRAME_PAYLOAD_SIZE), + checksum); + CompressionTest::appendBlock( + &input, + ntcd::CompressionBlockType::e_RAW, + static_cast(k_BLOCK_LENGTH[i]), + 0); + + bsl::vector payload(k_BLOCK_LENGTH[i], 'y'); + ntcs::BlobUtil::append(&input, &payload[0], payload.size()); + + CompressionTest::appendFrameFooter(&input, checksum); + + CompressionTest::verifyInflateFailure(input, + ntsa::Error::e_INVALID, + 0); + } + } + + // Concern: a raw block following a valid raw block whose payload length + // exceeds the remaining frame content is rejected before its payload is + // inflated. + + { + ntca::Checksum checksum(ntca::ChecksumType::e_CRC32); + + bdlbb::Blob input(&blobBufferFactory, NTSCFG_TEST_ALLOCATOR); + + CompressionTest::appendFrameHeader( + &input, + static_cast(k_BLOCK_SIZE + 3 + k_BLOCK_SIZE + 2), + checksum); + CompressionTest::appendBlock(&input, + ntcd::CompressionBlockType::e_RAW, + 3, + 0); + ntcs::BlobUtil::append(&input, "abc", 3); + CompressionTest::appendBlock(&input, + ntcd::CompressionBlockType::e_RAW, + 3, + 0); + ntcs::BlobUtil::append(&input, "def", 3); + CompressionTest::appendFrameFooter(&input, checksum); + + CompressionTest::verifyInflateFailure(input, + ntsa::Error::e_INVALID, + 3); + } +} + +NTSCFG_TEST_FUNCTION(ntcd::CompressionTest::verifyLimits) +{ + ntsa::Error error; + + bdlbb::PooledBlobBufferFactory blobBufferFactory(256, + NTSCFG_TEST_ALLOCATOR); + + bsl::shared_ptr dataPool; + dataPool.createInplace(NTSCFG_TEST_ALLOCATOR, + 256, + 256, + NTSCFG_TEST_ALLOCATOR); + + // Concern: the limits default to INT_MAX when not explicitly configured. + + NTSCFG_TEST_EQ(ntcd::CompressionDecoder::k_DEFAULT_MAX_INFLATE_SIZE, + static_cast(INT_MAX)); + + NTSCFG_TEST_EQ(ntcd::CompressionEncoder::k_DEFAULT_MAX_DEFLATE_SIZE, + static_cast(INT_MAX)); + + bdlbb::Blob frame(&blobBufferFactory, NTSCFG_TEST_ALLOCATOR); + bdlbb::Blob expected(&blobBufferFactory, NTSCFG_TEST_ALLOCATOR); + + CompressionTest::generateFrame(&frame, &expected); + + const bsl::size_t k_INFLATED_SIZE = + static_cast(expected.length()); + + // Concern: an inflate operation succeeds if and only if it produces no + // more than the maximum inflate size, and never appends more than the + // maximum inflate size. + + for (bsl::size_t limit = 0; limit <= k_INFLATED_SIZE + 1; ++limit) { + ntca::CompressionConfig compressionConfig; + compressionConfig.setType(ntca::CompressionType::e_RLE); + compressionConfig.setMaxInflateSize(limit); + + ntcd::Compression compression(compressionConfig, + dataPool, + NTSCFG_TEST_ALLOCATOR); + + bdlbb::Blob inflatedData(&blobBufferFactory, NTSCFG_TEST_ALLOCATOR); + + ntca::InflateOptions inflateOptions; + ntca::InflateContext inflateContext; + + error = compression.inflate(&inflateContext, + &inflatedData, + frame, + inflateOptions); + + if (limit >= k_INFLATED_SIZE) { + NTSCFG_TEST_OK(error); + NTSCFG_TEST_EQ(bdlbb::BlobUtil::compare(inflatedData, expected), + 0); + } + else { + NTSCFG_TEST_ERROR(error, ntsa::Error::e_LIMIT); + NTSCFG_TEST_LE(static_cast(inflatedData.length()), + limit); + + // The inflater resets after the failure, so inflating the same + // frame again fails again in the same way. + + const bsl::size_t inflatedLength = + static_cast(inflatedData.length()); + + error = compression.inflate(&inflateContext, + &inflatedData, + frame, + inflateOptions); + NTSCFG_TEST_ERROR(error, ntsa::Error::e_LIMIT); + NTSCFG_TEST_LE(static_cast(inflatedData.length()) - + inflatedLength, + limit); + } + } + + // Concern: the maximum inflate size applies to the entire operation, even + // when the operation is fed its input across many calls. + + for (bsl::size_t limit = 0; limit <= k_INFLATED_SIZE + 1; ++limit) { + ntca::CompressionConfig compressionConfig; + compressionConfig.setType(ntca::CompressionType::e_RLE); + compressionConfig.setMaxInflateSize(limit); + + ntcd::CompressionDecoder decoder(compressionConfig, + NTSCFG_TEST_ALLOCATOR); + + bdlbb::Blob inflatedData(&blobBufferFactory, NTSCFG_TEST_ALLOCATOR); + + ntca::InflateOptions inflateOptions; + ntca::InflateContext inflateContext; + + error = decoder.inflateBegin(&inflateContext, + &inflatedData, + inflateOptions); + NTSCFG_TEST_OK(error); + + bdlbb::Blob remaining = frame; + while (remaining.length() > 0 && !error) { + bdlbb::Blob chunk(&blobBufferFactory, NTSCFG_TEST_ALLOCATOR); + bdlbb::BlobUtil::append(&chunk, remaining, 0, 1); + bdlbb::BlobUtil::erase(&remaining, 0, 1); + + error = decoder.inflateNext(&inflateContext, + &inflatedData, + chunk, + inflateOptions); + } + + if (limit >= k_INFLATED_SIZE) { + NTSCFG_TEST_OK(error); + + error = decoder.inflateEnd(&inflateContext, + &inflatedData, + inflateOptions); + NTSCFG_TEST_OK(error); + + NTSCFG_TEST_EQ(bdlbb::BlobUtil::compare(inflatedData, expected), + 0); + } + else { + NTSCFG_TEST_ERROR(error, ntsa::Error::e_LIMIT); + NTSCFG_TEST_LE(static_cast(inflatedData.length()), + limit); + } + } + + // Concern: the maximum inflate size applies to each operation, not + // cumulatively across operations. + + { + ntca::CompressionConfig compressionConfig; + compressionConfig.setType(ntca::CompressionType::e_RLE); + compressionConfig.setMaxInflateSize(k_INFLATED_SIZE); + + ntcd::Compression compression(compressionConfig, + dataPool, + NTSCFG_TEST_ALLOCATOR); + + for (bsl::size_t i = 0; i < 4; ++i) { + bdlbb::Blob inflatedData(&blobBufferFactory, + NTSCFG_TEST_ALLOCATOR); + + ntca::InflateOptions inflateOptions; + ntca::InflateContext inflateContext; + + error = compression.inflate(&inflateContext, + &inflatedData, + frame, + inflateOptions); + NTSCFG_TEST_OK(error); + + NTSCFG_TEST_EQ(bdlbb::BlobUtil::compare(inflatedData, expected), + 0); + } + } + + // Concern: a deflate operation succeeds if and only if it produces no + // more than the maximum deflate size, and never appends more than the + // maximum deflate size. + + bsl::size_t deflatedSize = 0; + { + ntca::CompressionConfig compressionConfig; + compressionConfig.setType(ntca::CompressionType::e_RLE); + + ntcd::Compression compression(compressionConfig, + dataPool, + NTSCFG_TEST_ALLOCATOR); + + bdlbb::Blob deflatedData(&blobBufferFactory, NTSCFG_TEST_ALLOCATOR); + + ntca::DeflateOptions deflateOptions; + ntca::DeflateContext deflateContext; + + error = compression.deflate(&deflateContext, + &deflatedData, + expected, + deflateOptions); + NTSCFG_TEST_OK(error); + + deflatedSize = static_cast(deflatedData.length()); + } + + for (bsl::size_t limit = 0; limit <= deflatedSize + 1; ++limit) { + ntca::CompressionConfig compressionConfig; + compressionConfig.setType(ntca::CompressionType::e_RLE); + compressionConfig.setMaxDeflateSize(limit); + + ntcd::Compression compression(compressionConfig, + dataPool, + NTSCFG_TEST_ALLOCATOR); + + bdlbb::Blob deflatedData(&blobBufferFactory, NTSCFG_TEST_ALLOCATOR); + + ntca::DeflateOptions deflateOptions; + ntca::DeflateContext deflateContext; + + error = compression.deflate(&deflateContext, + &deflatedData, + expected, + deflateOptions); + + if (limit >= deflatedSize) { + NTSCFG_TEST_OK(error); + NTSCFG_TEST_EQ(static_cast(deflatedData.length()), + deflatedSize); + + bdlbb::Blob inflatedData(&blobBufferFactory, + NTSCFG_TEST_ALLOCATOR); + + ntca::InflateOptions inflateOptions; + ntca::InflateContext inflateContext; + + error = compression.inflate(&inflateContext, + &inflatedData, + deflatedData, + inflateOptions); + NTSCFG_TEST_OK(error); + + NTSCFG_TEST_EQ(bdlbb::BlobUtil::compare(inflatedData, expected), + 0); + } + else { + NTSCFG_TEST_ERROR(error, ntsa::Error::e_LIMIT); + NTSCFG_TEST_LE(static_cast(deflatedData.length()), + limit); + } + } +} + } // close namespace ntcd } // close namespace BloombergLP diff --git a/groups/ntc/ntci/ntci_compression.cpp b/groups/ntc/ntci/ntci_compression.cpp index bcacc1c0..43bed900 100644 --- a/groups/ntc/ntci/ntci_compression.cpp +++ b/groups/ntc/ntci/ntci_compression.cpp @@ -773,6 +773,10 @@ ntsa::Error Compression::inflate(ntca::InflateContext* context, error = ntsa::Error(ntsa::Error::e_NOT_IMPLEMENTED); } + if (error) { + return error; + } + error = this->inflateEnd(context, result, options); if (error) { return error; diff --git a/groups/ntc/ntctlc/ntctlc_plugin.cpp b/groups/ntc/ntctlc/ntctlc_plugin.cpp index 47b6efae..a68337da 100644 --- a/groups/ntc/ntctlc/ntctlc_plugin.cpp +++ b/groups/ntc/ntctlc/ntctlc_plugin.cpp @@ -35,6 +35,8 @@ BSLS_IDENT_RCSID(ntctlc_plugin_cpp, "$Id$ $CSID$") #include #include +#include + #if NTC_BUILD_WITH_LZ4 #define LZ4_STATIC_LINKING_ONLY #define LZ4F_STATIC_LINKING_ONLY @@ -106,6 +108,84 @@ BSLS_IDENT_RCSID(ntctlc_plugin_cpp, "$Id$ $CSID$") namespace BloombergLP { namespace ntctlc { +/// @internal @brief +/// Provide utilities to limit the number of bytes produced by a single inflate +/// or deflate operation. +/// +/// @par Thread Safety +/// This struct is thread safe. +/// +/// @ingroup module_ntctlc +struct CompressionLimitUtil { + /// The maximum number of bytes a single inflate or deflate operation may + /// produce when the maximum is not explicitly configured. + static const bsl::size_t k_DEFAULT_MAX_SIZE; + + /// Return the specified configured 'maxSize', if defined, otherwise + /// return the default maximum size. + static bsl::size_t maxSize(const bdlb::NullableValue& maxSize); + + /// Return the number of bytes that may still be produced by an operation + /// limited to the specified 'maxSize' that has already produced the + /// specified 'numBytesProduced' bytes, of which the specified + /// 'numBytesPending' bytes have not yet been appended to the specified + /// 'result'. The result is also limited such that the length of 'result' + /// never exceeds INT_MAX. + static bsl::size_t budget(bsl::size_t maxSize, + bsl::size_t numBytesProduced, + const bdlbb::Blob& result, + bsl::size_t numBytesPending); + + /// Log that the specified 'operation' produces more than the specified + /// 'maxSize' number of bytes. Return the error. + static ntsa::Error exceeded(const char* operation, bsl::size_t maxSize); +}; + +const bsl::size_t CompressionLimitUtil::k_DEFAULT_MAX_SIZE = + static_cast(INT_MAX); + +bsl::size_t CompressionLimitUtil::maxSize( + const bdlb::NullableValue& maxSize) +{ + return maxSize.value_or(k_DEFAULT_MAX_SIZE); +} + +bsl::size_t CompressionLimitUtil::budget(bsl::size_t maxSize, + bsl::size_t numBytesProduced, + const bdlbb::Blob& result, + bsl::size_t numBytesPending) +{ + bsl::size_t remaining = 0; + if (numBytesProduced < maxSize) { + remaining = maxSize - numBytesProduced; + } + + BSLS_ASSERT(result.length() >= 0); + + const bsl::size_t length = + static_cast(result.length()) + numBytesPending; + + bsl::size_t ceiling = 0; + if (length < static_cast(INT_MAX)) { + ceiling = static_cast(INT_MAX) - length; + } + + return remaining < ceiling ? remaining : ceiling; +} + +ntsa::Error CompressionLimitUtil::exceeded(const char* operation, + bsl::size_t maxSize) +{ + NTCI_LOG_CONTEXT(); + + NTCI_LOG_STREAM_ERROR << "Failed to " << operation + << ": the operation produces more than the maximum " + << "of " << maxSize << " bytes" + << NTCI_LOG_STREAM_END; + + return ntsa::Error(ntsa::Error::e_LIMIT); +} + #if NTC_BUILD_WITH_LZ4 /// @internal @brief @@ -144,6 +224,8 @@ class Lz4 : public ntci::Compression LZ4F_decompressOptions_t d_inflaterOptions; LZ4F_preferences_t d_preferences; int d_level; + bsl::size_t d_maxDeflateSize; + bsl::size_t d_maxInflateSize; bsl::shared_ptr d_dataPool_sp; ntca::CompressionConfig d_config; bslma::Allocator* d_allocator_p; @@ -201,6 +283,11 @@ class Lz4 : public ntci::Compression /// Destroy the deflater. Return the error. ntsa::Error deflateDestroy(); + /// Fail the current deflate operation with the specified 'error'. + /// Discard any deflated bytes not yet appended to the result of the + /// operation. Return the 'error'. + ntsa::Error deflateFail(ntsa::Error error); + /// Create the inflater. Return the error. ntsa::Error inflateCreate(); @@ -246,6 +333,12 @@ class Lz4 : public ntci::Compression /// Destroy the deflater. Return the error. ntsa::Error inflateDestroy(); + /// Fail the current inflate operation with the specified 'error'. + /// Discard any inflated bytes not yet appended to the result of the + /// operation, discard any unprocessed input, and reset the inflater to + /// prepare for a new frame. Return the 'error'. + ntsa::Error inflateFail(ntsa::Error error); + /// Allocate the specified 'size' number of bytes. The specified 'opaque' /// field points to the allocator object. static void* allocate(void* opaque, bsl::size_t size); @@ -284,6 +377,12 @@ class Lz4 : public ntci::Compression /// @ingroup module_ntctlc class Zstd : public ntci::Compression { + /// The base-2 logarithm of the maximum window size used by the deflater + /// and accepted by the inflater. This bounds the memory the inflater + /// allocates for a frame, regardless of the window size the frame + /// declares. + static const int k_WINDOW_LOG_MAX = 23; + ZSTD_CCtx* d_deflaterContext_p; ZSTD_inBuffer d_deflaterInput; ZSTD_outBuffer d_deflaterOutput; @@ -295,6 +394,8 @@ class Zstd : public ntci::Compression bdlbb::BlobBuffer d_inflaterBuffer; bsl::size_t d_inflaterBufferSize; int d_level; + bsl::size_t d_maxDeflateSize; + bsl::size_t d_maxInflateSize; bsl::shared_ptr d_dataPool_sp; ntca::CompressionConfig d_config; bslma::Allocator* d_allocator_p; @@ -341,10 +442,12 @@ class Zstd : public ntci::Compression void deflateCommit(bdlbb::Blob* result); /// Feed the input bytes available to the deflate algorithm and write - /// any output bytes to the output buffer. Return the error. + /// at most the specified 'maxBytesWritten' output bytes to the output + /// buffer. Return the error. bsl::size_t deflateCycle(bsl::size_t* numBytesRead, bsl::size_t* numBytesWritten, - ZSTD_EndDirective mode); + ZSTD_EndDirective mode, + bsl::size_t maxBytesWritten); /// Reset the deflater to prepare for a new frame. Return the error. ntsa::Error deflateReset(); @@ -352,6 +455,11 @@ class Zstd : public ntci::Compression /// Destroy the deflater. Return the error. ntsa::Error deflateDestroy(); + /// Fail the current deflate operation with the specified 'error'. + /// Discard any deflated bytes not yet appended to the result of the + /// operation. Return the 'error'. + ntsa::Error deflateFail(ntsa::Error error); + /// Create the inflater. Return the error. ntsa::Error inflateCreate(); @@ -387,9 +495,11 @@ class Zstd : public ntci::Compression void inflateCommit(bdlbb::Blob* result); /// Feed the input bytes available to the inflate algorithm and write - /// any output bytes to the output buffer. Return the error. + /// at most the specified 'maxBytesWritten' output bytes to the output + /// buffer. Return the error. bsl::size_t inflateCycle(bsl::size_t* numBytesRead, - bsl::size_t* numBytesWritten); + bsl::size_t* numBytesWritten, + bsl::size_t maxBytesWritten); /// Reset the inflater to prepare for a new frame. Return the error. ntsa::Error inflateReset(); @@ -397,6 +507,12 @@ class Zstd : public ntci::Compression /// Destroy the deflater. Return the error. ntsa::Error inflateDestroy(); + /// Fail the current inflate operation with the specified 'error'. + /// Discard any inflated bytes not yet appended to the result of the + /// operation, discard any unprocessed input, and reset the inflater to + /// prepare for a new frame. Return the 'error'. + ntsa::Error inflateFail(ntsa::Error error); + /// Allocate the specified 'size' number of bytes. The specified 'opaque' /// field points to the allocator object. static void* allocate(void* opaque, bsl::size_t size); @@ -452,6 +568,8 @@ class Zlib : public ntci::Compression bsl::size_t d_inflaterBufferSize; bsl::uint64_t d_inflaterGeneration; int d_level; + bsl::size_t d_maxDeflateSize; + bsl::size_t d_maxInflateSize; bsl::shared_ptr d_dataPool_sp; ntca::CompressionConfig d_config; bslma::Allocator* d_allocator_p; @@ -498,10 +616,12 @@ class Zlib : public ntci::Compression void deflateCommit(bdlbb::Blob* result); /// Feed the input bytes available to the deflate algorithm and write - /// any output bytes to the output buffer. Return the error. + /// at most the specified 'maxBytesWritten' output bytes to the output + /// buffer. Return the error. int deflateCycle(bsl::size_t* numBytesRead, bsl::size_t* numBytesWritten, - int mode); + int mode, + bsl::size_t maxBytesWritten); /// Reset the deflater to prepare for a new frame. Return the error. ntsa::Error deflateReset(); @@ -509,6 +629,11 @@ class Zlib : public ntci::Compression /// Destroy the deflater. Return the error. ntsa::Error deflateDestroy(); + /// Fail the current deflate operation with the specified 'error'. + /// Discard any deflated bytes not yet appended to the result of the + /// operation. Return the 'error'. + ntsa::Error deflateFail(ntsa::Error error); + /// Create the inflater. Return the error. ntsa::Error inflateCreate(); @@ -544,10 +669,12 @@ class Zlib : public ntci::Compression void inflateCommit(bdlbb::Blob* result); /// Feed the input bytes available to the inflate algorithm and write - /// any output bytes to the output buffer. Return the error. + /// at most the specified 'maxBytesWritten' output bytes to the output + /// buffer. Return the error. int inflateCycle(bsl::size_t* numBytesRead, bsl::size_t* numBytesWritten, - int mode); + int mode, + bsl::size_t maxBytesWritten); /// Reset the inflater to prepare for a new frame. Return the error. ntsa::Error inflateReset(); @@ -555,6 +682,12 @@ class Zlib : public ntci::Compression /// Destroy the deflater. Return the error. ntsa::Error inflateDestroy(); + /// Fail the current inflate operation with the specified 'error'. + /// Discard any inflated bytes not yet appended to the result of the + /// operation, discard any unprocessed input, and reset the inflater to + /// prepare for a new frame. Return the 'error'. + ntsa::Error inflateFail(ntsa::Error error); + /// Allocate the specified 'number' of elements of the specified 'size' in /// bytes. The specified 'opaque' field points to the allocator object. static void* allocate(void* opaque, @@ -613,6 +746,8 @@ class Gzip : public ntci::Compression char d_inflaterEntityComment[128]; bsl::uint64_t d_inflaterGeneration; int d_level; + bsl::size_t d_maxDeflateSize; + bsl::size_t d_maxInflateSize; bsl::shared_ptr d_dataPool_sp; ntca::CompressionConfig d_config; bslma::Allocator* d_allocator_p; @@ -659,10 +794,12 @@ class Gzip : public ntci::Compression void deflateCommit(bdlbb::Blob* result); /// Feed the input bytes available to the deflate algorithm and write - /// any output bytes to the output buffer. Return the error. + /// at most the specified 'maxBytesWritten' output bytes to the output + /// buffer. Return the error. int deflateCycle(bsl::size_t* numBytesRead, bsl::size_t* numBytesWritten, - int mode); + int mode, + bsl::size_t maxBytesWritten); /// Reset the deflater to prepare for a new frame. Return the error. ntsa::Error deflateReset(); @@ -670,6 +807,11 @@ class Gzip : public ntci::Compression /// Destroy the deflater. Return the error. ntsa::Error deflateDestroy(); + /// Fail the current deflate operation with the specified 'error'. + /// Discard any deflated bytes not yet appended to the result of the + /// operation. Return the 'error'. + ntsa::Error deflateFail(ntsa::Error error); + /// Create the inflater. Return the error. ntsa::Error inflateCreate(); @@ -705,10 +847,12 @@ class Gzip : public ntci::Compression void inflateCommit(bdlbb::Blob* result); /// Feed the input bytes available to the inflate algorithm and write - /// any output bytes to the output buffer. Return the error. + /// at most the specified 'maxBytesWritten' output bytes to the output + /// buffer. Return the error. int inflateCycle(bsl::size_t* numBytesRead, bsl::size_t* numBytesWritten, - int mode); + int mode, + bsl::size_t maxBytesWritten); /// Reset the inflater to prepare for a new frame. Return the error. ntsa::Error inflateReset(); @@ -716,6 +860,12 @@ class Gzip : public ntci::Compression /// Destroy the deflater. Return the error. ntsa::Error inflateDestroy(); + /// Fail the current inflate operation with the specified 'error'. + /// Discard any inflated bytes not yet appended to the result of the + /// operation, discard any unprocessed input, and reset the inflater to + /// prepare for a new frame. Return the 'error'. + ntsa::Error inflateFail(ntsa::Error error); + /// Allocate the specified 'number' of elements of the specified 'size' in /// bytes. The specified 'opaque' field points to the allocator object. static void* allocate(void* opaque, @@ -935,12 +1085,23 @@ ntsa::Error Lz4::deflateBegin(ntca::DeflateContext* context, if (LZ4F_isError(errorCode)) { NTCI_LOG_ERROR("Failed to begin compression frame: %s", LZ4F_getErrorName(errorCode)); - return ntsa::Error(ntsa::Error::e_INVALID); + return this->deflateFail(ntsa::Error(ntsa::Error::e_INVALID)); } const bsl::size_t numBytesRead = 0; const bsl::size_t numBytesWritten = static_cast(errorCode); + const bsl::size_t budget = + CompressionLimitUtil::budget(d_maxDeflateSize, + context->bytesWritten(), + *result, + d_deflaterBufferSize); + + if (numBytesWritten > budget) { + return this->deflateFail( + CompressionLimitUtil::exceeded("deflate", d_maxDeflateSize)); + } + NTCTLC_PLUGIN_LOG_DEFLATED_CHAR_BUFFER(header, numBytesWritten); ntcs::BlobUtil::append(result, header, numBytesWritten); @@ -1035,13 +1196,13 @@ ntsa::Error Lz4::deflateNext(ntca::DeflateContext* context, "expected at least %d, found %d", (int)(destinationCapacityRequired), (int)(d_deflaterArenaCapacity)); - return ntsa::Error(ntsa::Error::e_INVALID); + return this->deflateFail(ntsa::Error(ntsa::Error::e_INVALID)); } else { NTCI_LOG_ERROR( "Failed to update compression frame: %s", LZ4F_getErrorName(errorCode)); - return ntsa::Error(ntsa::Error::e_INVALID); + return this->deflateFail(ntsa::Error(ntsa::Error::e_INVALID)); } } @@ -1052,6 +1213,18 @@ ntsa::Error Lz4::deflateNext(ntca::DeflateContext* context, BSLS_ASSERT_OPT(numBytesRead > 0); BSLS_ASSERT_OPT(numBytesWritten > 0); + const bsl::size_t budget = CompressionLimitUtil::budget( + d_maxDeflateSize, + context->bytesWritten() + totalBytesWritten, + *result, + d_deflaterBufferSize); + + if (numBytesWritten > budget) { + return this->deflateFail(CompressionLimitUtil::exceeded( + "deflate", + d_maxDeflateSize)); + } + NTCTLC_PLUGIN_LOG_DEFLATED_CHAR_BUFFER( d_deflaterArena, numBytesWritten); @@ -1087,7 +1260,7 @@ ntsa::Error Lz4::deflateNext(ntca::DeflateContext* context, (int)(sourceSize), (int)(destinationCapacity), (int)(LZ4F_compressBound(sourceSize, &d_preferences))); - return ntsa::Error(ntsa::Error::e_INVALID); + return this->deflateFail(ntsa::Error(ntsa::Error::e_INVALID)); } const bsl::size_t numBytesRead = sourceSize; @@ -1097,6 +1270,17 @@ ntsa::Error Lz4::deflateNext(ntca::DeflateContext* context, BSLS_ASSERT_OPT(numBytesRead > 0); BSLS_ASSERT_OPT(numBytesWritten > 0); + const bsl::size_t budget = CompressionLimitUtil::budget( + d_maxDeflateSize, + context->bytesWritten() + totalBytesWritten, + *result, + d_deflaterBufferSize); + + if (numBytesWritten > budget) { + return this->deflateFail( + CompressionLimitUtil::exceeded("deflate", d_maxDeflateSize)); + } + totalBytesRead += numBytesRead; totalBytesWritten += numBytesWritten; @@ -1133,12 +1317,23 @@ ntsa::Error Lz4::deflateEnd(ntca::DeflateContext* context, if (LZ4F_isError(errorCode)) { NTCI_LOG_ERROR("Failed to end compression frame: %s", LZ4F_getErrorName(errorCode)); - return ntsa::Error(ntsa::Error::e_INVALID); + return this->deflateFail(ntsa::Error(ntsa::Error::e_INVALID)); } const bsl::size_t numBytesRead = 0; const bsl::size_t numBytesWritten = static_cast(errorCode); + const bsl::size_t budget = + CompressionLimitUtil::budget(d_maxDeflateSize, + context->bytesWritten(), + *result, + d_deflaterBufferSize); + + if (numBytesWritten > budget) { + return this->deflateFail( + CompressionLimitUtil::exceeded("deflate", d_maxDeflateSize)); + } + NTCTLC_PLUGIN_LOG_DEFLATED_CHAR_BUFFER(footer, numBytesWritten); ntcs::BlobUtil::append(result, footer, numBytesWritten); @@ -1213,6 +1408,14 @@ ntsa::Error Lz4::deflateDestroy() return ntsa::Error(); } +ntsa::Error Lz4::deflateFail(ntsa::Error error) +{ + d_deflaterBuffer.reset(); + d_deflaterBufferSize = 0; + + return error; +} + NTCCFG_INLINE ntsa::Error Lz4::inflateCreate() { @@ -1306,6 +1509,16 @@ ntsa::Error Lz4::inflateNext(ntca::InflateContext* context, static_cast( d_inflaterBuffer.size() - d_inflaterBufferSize); + const bsl::size_t budget = CompressionLimitUtil::budget( + d_maxInflateSize, + context->bytesWritten() + totalBytesWritten, + *result, + d_inflaterBufferSize); + + if (destinationSize > budget + 1) { + destinationSize = budget + 1; + } + bsl::size_t sourceSize = static_cast(sourceEnd - sourceCurrent); @@ -1319,7 +1532,12 @@ ntsa::Error Lz4::inflateNext(ntca::InflateContext* context, if (LZ4F_isError(errorCode)) { NTCI_LOG_ERROR("Failed to inflate: %s", LZ4F_getErrorName(errorCode)); - return ntsa::Error(ntsa::Error::e_INVALID); + return this->inflateFail(ntsa::Error(ntsa::Error::e_INVALID)); + } + + if (destinationSize > budget) { + return this->inflateFail( + CompressionLimitUtil::exceeded("inflate", d_maxInflateSize)); } if (destinationSize > 0) { @@ -1416,6 +1634,17 @@ ntsa::Error Lz4::inflateDestroy() return ntsa::Error(); } +ntsa::Error Lz4::inflateFail(ntsa::Error error) +{ + d_inflaterBuffer.reset(); + d_inflaterBufferSize = 0; + + ntsa::Error resetError = this->inflateReset(); + NTCCFG_WARNING_UNUSED(resetError); + + return error; +} + void* Lz4::allocate(void* opaque, bsl::size_t size) { bslma::Allocator* allocator = reinterpret_cast(opaque); @@ -1442,6 +1671,10 @@ Lz4::Lz4(const ntca::CompressionConfig& configuration, , d_inflaterBuffer() , d_inflaterBufferSize(0) , d_level(1) +, d_maxDeflateSize( + CompressionLimitUtil::maxSize(configuration.maxDeflateSize())) +, d_maxInflateSize( + CompressionLimitUtil::maxSize(configuration.maxInflateSize())) , d_dataPool_sp(dataPool) , d_config(configuration) , d_allocator_p(bslma::Default::allocator(basicAllocator)) @@ -1550,6 +1783,24 @@ ntsa::Error Zstd::deflateCreate() return Zstd::translateError(rc, "set checksum flag"); } + // Limit the window size to that accepted by the inflater, so that every + // frame produced by the deflater may be inflated, but do not increase + // the window size beyond the default for the compression level. + + const ZSTD_compressionParameters compressionParameters = + ZSTD_getCParams(d_level, ZSTD_CONTENTSIZE_UNKNOWN, 0); + + if (compressionParameters.windowLog > + static_cast(k_WINDOW_LOG_MAX)) + { + rc = ZSTD_CCtx_setParameter(d_deflaterContext_p, + ZSTD_c_windowLog, + k_WINDOW_LOG_MAX); + if (ZSTD_isError(rc)) { + return Zstd::translateError(rc, "set window log"); + } + } + bsl::memset(&d_deflaterInput, 0, sizeof d_deflaterInput); bsl::memset(&d_deflaterOutput, 0, sizeof d_deflaterOutput); @@ -1604,18 +1855,30 @@ ntsa::Error Zstd::deflateNext(ntca::DeflateContext* context, this->deflateOverflow(result); } + const bsl::size_t budget = CompressionLimitUtil::budget( + d_maxDeflateSize, + context->bytesWritten() + totalBytesWritten, + *result, + d_deflaterBufferSize); + bsl::size_t numBytesRead = 0; bsl::size_t numBytesWritten = 0; rc = this->deflateCycle(&numBytesRead, &numBytesWritten, - ZSTD_e_continue); + ZSTD_e_continue, + budget + 1); totalBytesRead += numBytesRead; totalBytesWritten += numBytesWritten; if (ZSTD_isError(rc)) { - return this->translateError(rc, "deflate"); + return this->deflateFail(this->translateError(rc, "deflate")); + } + + if (numBytesWritten > budget) { + return this->deflateFail( + CompressionLimitUtil::exceeded("deflate", d_maxDeflateSize)); } } @@ -1641,26 +1904,38 @@ ntsa::Error Zstd::deflateEnd(ntca::DeflateContext* context, this->deflateOverflow(result); } + const bsl::size_t budget = CompressionLimitUtil::budget( + d_maxDeflateSize, + context->bytesWritten() + totalBytesWritten, + *result, + d_deflaterBufferSize); + bsl::size_t numBytesRead = 0; bsl::size_t numBytesWritten = 0; - rc = this->deflateCycle(&numBytesRead, &numBytesWritten, ZSTD_e_end); + rc = this->deflateCycle(&numBytesRead, + &numBytesWritten, + ZSTD_e_end, + budget + 1); totalBytesRead += numBytesRead; totalBytesWritten += numBytesWritten; - if (rc > 0) { - continue; + if (ZSTD_isError(rc)) { + return this->deflateFail(this->translateError(rc, "deflate")); + } + + if (numBytesWritten > budget) { + return this->deflateFail( + CompressionLimitUtil::exceeded("deflate", d_maxDeflateSize)); } - else if (rc == 0) { + + if (rc == 0) { if (d_deflaterBufferSize != 0) { this->deflateCommit(result); } break; } - else { - return this->translateError(rc, "deflate"); - } } context->setCompressionType(ntca::CompressionType::e_ZSTD); @@ -1712,13 +1987,21 @@ void Zstd::deflateCommit(bdlbb::Blob* result) NTCCFG_INLINE bsl::size_t Zstd::deflateCycle(bsl::size_t* numBytesRead, bsl::size_t* numBytesWritten, - ZSTD_EndDirective mode) + ZSTD_EndDirective mode, + bsl::size_t maxBytesWritten) { bsl::size_t rc = 0; *numBytesRead = 0; *numBytesWritten = 0; + bsl::size_t outputHidden = 0; + if (d_deflaterOutput.size - d_deflaterOutput.pos > maxBytesWritten) { + outputHidden = + d_deflaterOutput.size - d_deflaterOutput.pos - maxBytesWritten; + d_deflaterOutput.size -= outputHidden; + } + bsl::size_t posIn0 = d_deflaterInput.pos; bsl::size_t posOut0 = d_deflaterOutput.pos; @@ -1730,6 +2013,8 @@ bsl::size_t Zstd::deflateCycle(bsl::size_t* numBytesRead, bsl::size_t posIn1 = d_deflaterInput.pos; bsl::size_t posOut1 = d_deflaterOutput.pos; + d_deflaterOutput.size += outputHidden; + BSLS_ASSERT(posIn1 >= posIn0); BSLS_ASSERT(posOut1 >= posOut0); @@ -1772,6 +2057,17 @@ ntsa::Error Zstd::deflateDestroy() return ntsa::Error(); } +ntsa::Error Zstd::deflateFail(ntsa::Error error) +{ + d_deflaterBuffer.reset(); + d_deflaterBufferSize = 0; + + bsl::memset(&d_deflaterInput, 0, sizeof d_deflaterInput); + bsl::memset(&d_deflaterOutput, 0, sizeof d_deflaterOutput); + + return error; +} + NTCCFG_INLINE ntsa::Error Zstd::inflateCreate() { @@ -1798,6 +2094,16 @@ ntsa::Error Zstd::inflateCreate() #endif + // Reject frames whose declared window size exceeds the maximum, which + // bounds the memory allocated to inflate each frame. + + bsl::size_t rc = ZSTD_DCtx_setParameter(d_inflaterContext_p, + ZSTD_d_windowLogMax, + k_WINDOW_LOG_MAX); + if (ZSTD_isError(rc)) { + return Zstd::translateError(rc, "set maximum window log"); + } + bsl::memset(&d_inflaterInput, 0, sizeof d_inflaterInput); bsl::memset(&d_inflaterOutput, 0, sizeof d_inflaterOutput); @@ -1838,31 +2144,38 @@ ntsa::Error Zstd::inflateNext(ntca::InflateContext* context, this->inflateOverflow(result); } + const bsl::size_t budget = CompressionLimitUtil::budget( + d_maxInflateSize, + context->bytesWritten() + totalBytesWritten, + *result, + d_inflaterBufferSize); + bsl::size_t numBytesRead = 0; bsl::size_t numBytesWritten = 0; - rc = this->inflateCycle(&numBytesRead, &numBytesWritten); + rc = this->inflateCycle(&numBytesRead, &numBytesWritten, budget + 1); totalBytesRead += numBytesRead; totalBytesWritten += numBytesWritten; - if (rc > 0) { - continue; + if (ZSTD_isError(rc)) { + return this->inflateFail(this->translateError(rc, "inflate")); + } + + if (numBytesWritten > budget) { + return this->inflateFail( + CompressionLimitUtil::exceeded("inflate", d_maxInflateSize)); } - else if (rc == 0) { + + if (rc == 0) { if (d_inflaterBufferSize != 0) { this->inflateCommit(result); } error = this->inflateReset(); if (error) { - return error; + return this->inflateFail(error); } - - continue; - } - else { - return this->translateError(rc, "inflate"); } } @@ -1880,9 +2193,12 @@ ntsa::Error Zstd::inflateEnd(ntca::InflateContext* context, bdlbb::Blob* result, const ntca::InflateOptions& options) { - NTCCFG_WARNING_UNUSED(result); NTCCFG_WARNING_UNUSED(options); + if (d_inflaterBufferSize != 0) { + this->inflateCommit(result); + } + context->setCompressionType(ntca::CompressionType::e_ZSTD); return ntsa::Error(); @@ -1929,13 +2245,21 @@ void Zstd::inflateCommit(bdlbb::Blob* result) NTCCFG_INLINE bsl::size_t Zstd::inflateCycle(bsl::size_t* numBytesRead, - bsl::size_t* numBytesWritten) + bsl::size_t* numBytesWritten, + bsl::size_t maxBytesWritten) { bsl::size_t rc = 0; *numBytesRead = 0; *numBytesWritten = 0; + bsl::size_t outputHidden = 0; + if (d_inflaterOutput.size - d_inflaterOutput.pos > maxBytesWritten) { + outputHidden = + d_inflaterOutput.size - d_inflaterOutput.pos - maxBytesWritten; + d_inflaterOutput.size -= outputHidden; + } + bsl::size_t posIn0 = d_inflaterInput.pos; bsl::size_t posOut0 = d_inflaterOutput.pos; @@ -1946,6 +2270,8 @@ bsl::size_t Zstd::inflateCycle(bsl::size_t* numBytesRead, bsl::size_t posIn1 = d_inflaterInput.pos; bsl::size_t posOut1 = d_inflaterOutput.pos; + d_inflaterOutput.size += outputHidden; + BSLS_ASSERT(posIn1 >= posIn0); BSLS_ASSERT(posOut1 >= posOut0); @@ -1988,6 +2314,20 @@ ntsa::Error Zstd::inflateDestroy() return ntsa::Error(); } +ntsa::Error Zstd::inflateFail(ntsa::Error error) +{ + d_inflaterBuffer.reset(); + d_inflaterBufferSize = 0; + + bsl::memset(&d_inflaterInput, 0, sizeof d_inflaterInput); + bsl::memset(&d_inflaterOutput, 0, sizeof d_inflaterOutput); + + ntsa::Error resetError = this->inflateReset(); + NTCCFG_WARNING_UNUSED(resetError); + + return error; +} + const char* Zstd::describeError(bsl::size_t error) { return ZSTD_getErrorName(error); @@ -2055,6 +2395,10 @@ Zstd::Zstd(const ntca::CompressionConfig& configuration, , d_inflaterBuffer() , d_inflaterBufferSize(0) , d_level(0) +, d_maxDeflateSize( + CompressionLimitUtil::maxSize(configuration.maxDeflateSize())) +, d_maxInflateSize( + CompressionLimitUtil::maxSize(configuration.maxInflateSize())) , d_dataPool_sp(dataPool) , d_config(configuration) , d_allocator_p(bslma::Default::allocator(basicAllocator)) @@ -2196,24 +2540,55 @@ ntsa::Error Zlib::deflateNext(ntca::DeflateContext* context, bsl::size_t totalBytesWritten = 0; bsl::size_t totalBytesRead = 0; - d_deflaterStream.next_in = const_cast(data); - d_deflaterStream.avail_in = static_cast(size); + const bsl::uint8_t* source = data; + bsl::size_t sourceRemaining = size; + + while (true) { + if (d_deflaterStream.avail_in == 0) { + if (sourceRemaining == 0) { + break; + } + + const bsl::size_t sourceSize = + sourceRemaining < static_cast(UINT_MAX) + ? sourceRemaining + : static_cast(UINT_MAX); + + d_deflaterStream.next_in = const_cast(source); + d_deflaterStream.avail_in = static_cast(sourceSize); + + source += sourceSize; + sourceRemaining -= sourceSize; + } - while (d_deflaterStream.avail_in != 0) { if (d_deflaterStream.avail_out == 0) { this->deflateOverflow(result); } + const bsl::size_t budget = CompressionLimitUtil::budget( + d_maxDeflateSize, + context->bytesWritten() + totalBytesWritten, + *result, + d_deflaterBufferSize); + bsl::size_t numBytesRead = 0; bsl::size_t numBytesWritten = 0; - rc = this->deflateCycle(&numBytesRead, &numBytesWritten, Z_NO_FLUSH); + rc = this->deflateCycle(&numBytesRead, + &numBytesWritten, + Z_NO_FLUSH, + budget + 1); totalBytesRead += numBytesRead; totalBytesWritten += numBytesWritten; + if (numBytesWritten > budget) { + return this->deflateFail( + CompressionLimitUtil::exceeded("deflate", d_maxDeflateSize)); + } + if (rc != Z_OK && rc != Z_BUF_ERROR) { - return this->translateError(rc, "deflate"); + return this->deflateFail(this->translateError(rc, "deflate")); } } @@ -2241,14 +2616,28 @@ ntsa::Error Zlib::deflateEnd(ntca::DeflateContext* context, this->deflateOverflow(result); } + const bsl::size_t budget = CompressionLimitUtil::budget( + d_maxDeflateSize, + context->bytesWritten() + totalBytesWritten, + *result, + d_deflaterBufferSize); + bsl::size_t numBytesRead = 0; bsl::size_t numBytesWritten = 0; - rc = this->deflateCycle(&numBytesRead, &numBytesWritten, Z_FINISH); + rc = this->deflateCycle(&numBytesRead, + &numBytesWritten, + Z_FINISH, + budget + 1); totalBytesRead += numBytesRead; totalBytesWritten += numBytesWritten; + if (numBytesWritten > budget) { + return this->deflateFail( + CompressionLimitUtil::exceeded("deflate", d_maxDeflateSize)); + } + if (rc == Z_OK || rc == Z_BUF_ERROR) { continue; } @@ -2259,7 +2648,7 @@ ntsa::Error Zlib::deflateEnd(ntca::DeflateContext* context, break; } else { - return this->translateError(rc, "deflate"); + return this->deflateFail(this->translateError(rc, "deflate")); } } @@ -2323,13 +2712,21 @@ void Zlib::deflateCommit(bdlbb::Blob* result) NTCCFG_INLINE int Zlib::deflateCycle(bsl::size_t* numBytesRead, bsl::size_t* numBytesWritten, - int mode) + int mode, + bsl::size_t maxBytesWritten) { int rc = 0; *numBytesRead = 0; *numBytesWritten = 0; + uInt availOutHidden = 0; + if (d_deflaterStream.avail_out > maxBytesWritten) { + availOutHidden = d_deflaterStream.avail_out - + static_cast(maxBytesWritten); + d_deflaterStream.avail_out = static_cast(maxBytesWritten); + } + uInt availIn0 = d_deflaterStream.avail_in; uInt availOut0 = d_deflaterStream.avail_out; @@ -2338,6 +2735,8 @@ int Zlib::deflateCycle(bsl::size_t* numBytesRead, uInt availIn1 = d_deflaterStream.avail_in; uInt availOut1 = d_deflaterStream.avail_out; + d_deflaterStream.avail_out += availOutHidden; + BSLS_ASSERT(availIn0 >= availIn1); BSLS_ASSERT(availOut0 >= availOut1); @@ -2396,6 +2795,19 @@ ntsa::Error Zlib::deflateDestroy() return ntsa::Error(); } +ntsa::Error Zlib::deflateFail(ntsa::Error error) +{ + d_deflaterBuffer.reset(); + d_deflaterBufferSize = 0; + + d_deflaterStream.next_in = 0; + d_deflaterStream.avail_in = 0; + d_deflaterStream.next_out = 0; + d_deflaterStream.avail_out = 0; + + return error; +} + NTCCFG_INLINE ntsa::Error Zlib::inflateCreate() { @@ -2450,22 +2862,53 @@ ntsa::Error Zlib::inflateNext(ntca::InflateContext* context, BSLS_ASSERT(d_inflaterStream.next_in == 0); BSLS_ASSERT(d_inflaterStream.avail_in == 0); - d_inflaterStream.next_in = const_cast(data); - d_inflaterStream.avail_in = static_cast(size); + const bsl::uint8_t* source = data; + bsl::size_t sourceRemaining = size; + + while (true) { + if (d_inflaterStream.avail_in == 0) { + if (sourceRemaining == 0) { + break; + } + + const bsl::size_t sourceSize = + sourceRemaining < static_cast(UINT_MAX) + ? sourceRemaining + : static_cast(UINT_MAX); + + d_inflaterStream.next_in = const_cast(source); + d_inflaterStream.avail_in = static_cast(sourceSize); + + source += sourceSize; + sourceRemaining -= sourceSize; + } - while (d_inflaterStream.avail_in != 0) { if (d_inflaterStream.avail_out == 0) { this->inflateOverflow(result); } + const bsl::size_t budget = CompressionLimitUtil::budget( + d_maxInflateSize, + context->bytesWritten() + totalBytesWritten, + *result, + d_inflaterBufferSize); + bsl::size_t numBytesRead = 0; bsl::size_t numBytesWritten = 0; - rc = this->inflateCycle(&numBytesRead, &numBytesWritten, Z_NO_FLUSH); + rc = this->inflateCycle(&numBytesRead, + &numBytesWritten, + Z_NO_FLUSH, + budget + 1); totalBytesRead += numBytesRead; totalBytesWritten += numBytesWritten; + if (numBytesWritten > budget) { + return this->inflateFail( + CompressionLimitUtil::exceeded("inflate", d_maxInflateSize)); + } + if (rc == Z_OK || rc == Z_BUF_ERROR) { continue; } @@ -2478,13 +2921,13 @@ ntsa::Error Zlib::inflateNext(ntca::InflateContext* context, error = this->inflateReset(); if (error) { - return error; + return this->inflateFail(error); } continue; } else { - return this->translateError(rc, "inflate"); + return this->inflateFail(this->translateError(rc, "inflate")); } } @@ -2519,14 +2962,28 @@ ntsa::Error Zlib::inflateEnd(ntca::InflateContext* context, this->inflateOverflow(result); } + const bsl::size_t budget = CompressionLimitUtil::budget( + d_maxInflateSize, + context->bytesWritten() + totalBytesWritten, + *result, + d_inflaterBufferSize); + bsl::size_t numBytesRead = 0; bsl::size_t numBytesWritten = 0; - rc = this->inflateCycle(&numBytesRead, &numBytesWritten, Z_SYNC_FLUSH); + rc = this->inflateCycle(&numBytesRead, + &numBytesWritten, + Z_SYNC_FLUSH, + budget + 1); totalBytesRead += numBytesRead; totalBytesWritten += numBytesWritten; + if (numBytesWritten > budget) { + return this->inflateFail( + CompressionLimitUtil::exceeded("inflate", d_maxInflateSize)); + } + if (rc == Z_OK || rc == Z_BUF_ERROR) { if (numBytesRead == 0 && numBytesWritten == 0) { break; @@ -2544,16 +3001,20 @@ ntsa::Error Zlib::inflateEnd(ntca::InflateContext* context, error = this->inflateReset(); if (error) { - return error; + return this->inflateFail(error); } continue; } else { - return this->translateError(rc, "inflate"); + return this->inflateFail(this->translateError(rc, "inflate")); } } + if (d_inflaterBufferSize != 0) { + this->inflateCommit(result); + } + d_inflaterStream.next_in = 0; d_inflaterStream.avail_in = 0; @@ -2605,13 +3066,21 @@ void Zlib::inflateCommit(bdlbb::Blob* result) NTCCFG_INLINE int Zlib::inflateCycle(bsl::size_t* numBytesRead, bsl::size_t* numBytesWritten, - int mode) + int mode, + bsl::size_t maxBytesWritten) { int rc = 0; *numBytesRead = 0; *numBytesWritten = 0; + uInt availOutHidden = 0; + if (d_inflaterStream.avail_out > maxBytesWritten) { + availOutHidden = d_inflaterStream.avail_out - + static_cast(maxBytesWritten); + d_inflaterStream.avail_out = static_cast(maxBytesWritten); + } + uInt availIn0 = d_inflaterStream.avail_in; uInt availOut0 = d_inflaterStream.avail_out; @@ -2620,6 +3089,8 @@ int Zlib::inflateCycle(bsl::size_t* numBytesRead, uInt availIn1 = d_inflaterStream.avail_in; uInt availOut1 = d_inflaterStream.avail_out; + d_inflaterStream.avail_out += availOutHidden; + BSLS_ASSERT(availIn0 >= availIn1); BSLS_ASSERT(availOut0 >= availOut1); @@ -2678,6 +3149,22 @@ ntsa::Error Zlib::inflateDestroy() return ntsa::Error(); } +ntsa::Error Zlib::inflateFail(ntsa::Error error) +{ + d_inflaterBuffer.reset(); + d_inflaterBufferSize = 0; + + d_inflaterStream.next_in = 0; + d_inflaterStream.avail_in = 0; + d_inflaterStream.next_out = 0; + d_inflaterStream.avail_out = 0; + + ntsa::Error resetError = this->inflateReset(); + NTCCFG_WARNING_UNUSED(resetError); + + return error; +} + void* Zlib::allocate(void* opaque, unsigned int number, unsigned int size) { bslma::Allocator* allocator = reinterpret_cast(opaque); @@ -2792,6 +3279,10 @@ Zlib::Zlib(const ntca::CompressionConfig& configuration, , d_inflaterBufferSize(0) , d_inflaterGeneration(0) , d_level(Z_DEFAULT_COMPRESSION) +, d_maxDeflateSize( + CompressionLimitUtil::maxSize(configuration.maxDeflateSize())) +, d_maxInflateSize( + CompressionLimitUtil::maxSize(configuration.maxInflateSize())) , d_dataPool_sp(dataPool) , d_config(configuration) , d_allocator_p(bslma::Default::allocator(basicAllocator)) @@ -2950,24 +3441,55 @@ ntsa::Error Gzip::deflateNext(ntca::DeflateContext* context, bsl::size_t totalBytesWritten = 0; bsl::size_t totalBytesRead = 0; - d_deflaterStream.next_in = const_cast(data); - d_deflaterStream.avail_in = static_cast(size); + const bsl::uint8_t* source = data; + bsl::size_t sourceRemaining = size; + + while (true) { + if (d_deflaterStream.avail_in == 0) { + if (sourceRemaining == 0) { + break; + } + + const bsl::size_t sourceSize = + sourceRemaining < static_cast(UINT_MAX) + ? sourceRemaining + : static_cast(UINT_MAX); + + d_deflaterStream.next_in = const_cast(source); + d_deflaterStream.avail_in = static_cast(sourceSize); + + source += sourceSize; + sourceRemaining -= sourceSize; + } - while (d_deflaterStream.avail_in != 0) { if (d_deflaterStream.avail_out == 0) { this->deflateOverflow(result); } + const bsl::size_t budget = CompressionLimitUtil::budget( + d_maxDeflateSize, + context->bytesWritten() + totalBytesWritten, + *result, + d_deflaterBufferSize); + bsl::size_t numBytesRead = 0; bsl::size_t numBytesWritten = 0; - rc = this->deflateCycle(&numBytesRead, &numBytesWritten, Z_NO_FLUSH); + rc = this->deflateCycle(&numBytesRead, + &numBytesWritten, + Z_NO_FLUSH, + budget + 1); totalBytesRead += numBytesRead; totalBytesWritten += numBytesWritten; + if (numBytesWritten > budget) { + return this->deflateFail( + CompressionLimitUtil::exceeded("deflate", d_maxDeflateSize)); + } + if (rc != Z_OK && rc != Z_BUF_ERROR) { - return this->translateError(rc, "deflate"); + return this->deflateFail(this->translateError(rc, "deflate")); } } @@ -2995,14 +3517,28 @@ ntsa::Error Gzip::deflateEnd(ntca::DeflateContext* context, this->deflateOverflow(result); } + const bsl::size_t budget = CompressionLimitUtil::budget( + d_maxDeflateSize, + context->bytesWritten() + totalBytesWritten, + *result, + d_deflaterBufferSize); + bsl::size_t numBytesRead = 0; bsl::size_t numBytesWritten = 0; - rc = this->deflateCycle(&numBytesRead, &numBytesWritten, Z_FINISH); + rc = this->deflateCycle(&numBytesRead, + &numBytesWritten, + Z_FINISH, + budget + 1); totalBytesRead += numBytesRead; totalBytesWritten += numBytesWritten; + if (numBytesWritten > budget) { + return this->deflateFail( + CompressionLimitUtil::exceeded("deflate", d_maxDeflateSize)); + } + if (rc == Z_OK || rc == Z_BUF_ERROR) { continue; } @@ -3013,7 +3549,7 @@ ntsa::Error Gzip::deflateEnd(ntca::DeflateContext* context, break; } else { - return this->translateError(rc, "deflate"); + return this->deflateFail(this->translateError(rc, "deflate")); } } @@ -3077,13 +3613,21 @@ void Gzip::deflateCommit(bdlbb::Blob* result) NTCCFG_INLINE int Gzip::deflateCycle(bsl::size_t* numBytesRead, bsl::size_t* numBytesWritten, - int mode) + int mode, + bsl::size_t maxBytesWritten) { int rc = 0; *numBytesRead = 0; *numBytesWritten = 0; + uInt availOutHidden = 0; + if (d_deflaterStream.avail_out > maxBytesWritten) { + availOutHidden = d_deflaterStream.avail_out - + static_cast(maxBytesWritten); + d_deflaterStream.avail_out = static_cast(maxBytesWritten); + } + uInt availIn0 = d_deflaterStream.avail_in; uInt availOut0 = d_deflaterStream.avail_out; @@ -3092,6 +3636,8 @@ int Gzip::deflateCycle(bsl::size_t* numBytesRead, uInt availIn1 = d_deflaterStream.avail_in; uInt availOut1 = d_deflaterStream.avail_out; + d_deflaterStream.avail_out += availOutHidden; + BSLS_ASSERT(availIn0 >= availIn1); BSLS_ASSERT(availOut0 >= availOut1); @@ -3177,6 +3723,19 @@ ntsa::Error Gzip::deflateDestroy() return ntsa::Error(); } +ntsa::Error Gzip::deflateFail(ntsa::Error error) +{ + d_deflaterBuffer.reset(); + d_deflaterBufferSize = 0; + + d_deflaterStream.next_in = 0; + d_deflaterStream.avail_in = 0; + d_deflaterStream.next_out = 0; + d_deflaterStream.avail_out = 0; + + return error; +} + NTCCFG_INLINE ntsa::Error Gzip::inflateCreate() { @@ -3234,22 +3793,53 @@ ntsa::Error Gzip::inflateNext(ntca::InflateContext* context, BSLS_ASSERT(d_inflaterStream.next_in == 0); BSLS_ASSERT(d_inflaterStream.avail_in == 0); - d_inflaterStream.next_in = const_cast(data); - d_inflaterStream.avail_in = static_cast(size); + const bsl::uint8_t* source = data; + bsl::size_t sourceRemaining = size; + + while (true) { + if (d_inflaterStream.avail_in == 0) { + if (sourceRemaining == 0) { + break; + } + + const bsl::size_t sourceSize = + sourceRemaining < static_cast(UINT_MAX) + ? sourceRemaining + : static_cast(UINT_MAX); + + d_inflaterStream.next_in = const_cast(source); + d_inflaterStream.avail_in = static_cast(sourceSize); + + source += sourceSize; + sourceRemaining -= sourceSize; + } - while (d_inflaterStream.avail_in != 0) { if (d_inflaterStream.avail_out == 0) { this->inflateOverflow(result); } + const bsl::size_t budget = CompressionLimitUtil::budget( + d_maxInflateSize, + context->bytesWritten() + totalBytesWritten, + *result, + d_inflaterBufferSize); + bsl::size_t numBytesRead = 0; bsl::size_t numBytesWritten = 0; - rc = this->inflateCycle(&numBytesRead, &numBytesWritten, Z_NO_FLUSH); + rc = this->inflateCycle(&numBytesRead, + &numBytesWritten, + Z_NO_FLUSH, + budget + 1); totalBytesRead += numBytesRead; totalBytesWritten += numBytesWritten; + if (numBytesWritten > budget) { + return this->inflateFail( + CompressionLimitUtil::exceeded("inflate", d_maxInflateSize)); + } + if (rc == Z_OK || rc == Z_BUF_ERROR) { continue; } @@ -3263,13 +3853,13 @@ ntsa::Error Gzip::inflateNext(ntca::InflateContext* context, error = this->inflateReset(); if (error) { - return error; + return this->inflateFail(error); } continue; } else { - return this->translateError(rc, "inflate"); + return this->inflateFail(this->translateError(rc, "inflate")); } } @@ -3304,14 +3894,28 @@ ntsa::Error Gzip::inflateEnd(ntca::InflateContext* context, this->inflateOverflow(result); } + const bsl::size_t budget = CompressionLimitUtil::budget( + d_maxInflateSize, + context->bytesWritten() + totalBytesWritten, + *result, + d_inflaterBufferSize); + bsl::size_t numBytesRead = 0; bsl::size_t numBytesWritten = 0; - rc = this->inflateCycle(&numBytesRead, &numBytesWritten, Z_SYNC_FLUSH); + rc = this->inflateCycle(&numBytesRead, + &numBytesWritten, + Z_SYNC_FLUSH, + budget + 1); totalBytesRead += numBytesRead; totalBytesWritten += numBytesWritten; + if (numBytesWritten > budget) { + return this->inflateFail( + CompressionLimitUtil::exceeded("inflate", d_maxInflateSize)); + } + if (rc == Z_OK || rc == Z_BUF_ERROR) { if (numBytesRead == 0 && numBytesWritten == 0) { break; @@ -3330,16 +3934,20 @@ ntsa::Error Gzip::inflateEnd(ntca::InflateContext* context, error = this->inflateReset(); if (error) { - return error; + return this->inflateFail(error); } continue; } else { - return this->translateError(rc, "inflate"); + return this->inflateFail(this->translateError(rc, "inflate")); } } + if (d_inflaterBufferSize != 0) { + this->inflateCommit(result); + } + d_inflaterStream.next_in = 0; d_inflaterStream.avail_in = 0; @@ -3391,13 +3999,21 @@ void Gzip::inflateCommit(bdlbb::Blob* result) NTCCFG_INLINE int Gzip::inflateCycle(bsl::size_t* numBytesRead, bsl::size_t* numBytesWritten, - int mode) + int mode, + bsl::size_t maxBytesWritten) { int rc = 0; *numBytesRead = 0; *numBytesWritten = 0; + uInt availOutHidden = 0; + if (d_inflaterStream.avail_out > maxBytesWritten) { + availOutHidden = d_inflaterStream.avail_out - + static_cast(maxBytesWritten); + d_inflaterStream.avail_out = static_cast(maxBytesWritten); + } + uInt availIn0 = d_inflaterStream.avail_in; uInt availOut0 = d_inflaterStream.avail_out; @@ -3406,6 +4022,8 @@ int Gzip::inflateCycle(bsl::size_t* numBytesRead, uInt availIn1 = d_inflaterStream.avail_in; uInt availOut1 = d_inflaterStream.avail_out; + d_inflaterStream.avail_out += availOutHidden; + BSLS_ASSERT(availIn0 >= availIn1); BSLS_ASSERT(availOut0 >= availOut1); @@ -3481,6 +4099,22 @@ ntsa::Error Gzip::inflateDestroy() return ntsa::Error(); } +ntsa::Error Gzip::inflateFail(ntsa::Error error) +{ + d_inflaterBuffer.reset(); + d_inflaterBufferSize = 0; + + d_inflaterStream.next_in = 0; + d_inflaterStream.avail_in = 0; + d_inflaterStream.next_out = 0; + d_inflaterStream.avail_out = 0; + + ntsa::Error resetError = this->inflateReset(); + NTCCFG_WARNING_UNUSED(resetError); + + return error; +} + void* Gzip::allocate(void* opaque, unsigned int number, unsigned int size) { bslma::Allocator* allocator = reinterpret_cast(opaque); @@ -3595,6 +4229,10 @@ Gzip::Gzip(const ntca::CompressionConfig& configuration, , d_inflaterBufferSize(0) , d_inflaterGeneration(0) , d_level(Z_DEFAULT_COMPRESSION) +, d_maxDeflateSize( + CompressionLimitUtil::maxSize(configuration.maxDeflateSize())) +, d_maxInflateSize( + CompressionLimitUtil::maxSize(configuration.maxInflateSize())) , d_dataPool_sp(dataPool) , d_config(configuration) , d_allocator_p(bslma::Default::allocator(basicAllocator)) diff --git a/groups/ntc/ntctlc/ntctlc_plugin.t.cpp b/groups/ntc/ntctlc/ntctlc_plugin.t.cpp index 25a6ca9c..d76a1806 100644 --- a/groups/ntc/ntctlc/ntctlc_plugin.t.cpp +++ b/groups/ntc/ntctlc/ntctlc_plugin.t.cpp @@ -21,6 +21,7 @@ BSLS_IDENT_RCSID(ntctlc_plugin_t_cpp, "$Id$ $CSID$") #include #include #include +#include #include using namespace BloombergLP; @@ -62,11 +63,60 @@ class PluginTest // TODO. static void verifyUsage(); + // Verify each compression type honors the configured maximum number of + // bytes inflated and deflated per operation. + static void verifyLimits(); + + // Verify each compression type rejects malformed input, and resets after + // the failure such that subsequent, well-formed input is inflated. + static void verifyMalformed(); + + // Verify the zstd inflater rejects frames whose declared window size + // exceeds the maximum, and that the zstd deflater never produces such + // frames. + static void verifyZstdWindow(); + private: /// Verify the integrity of inflating and deflating a data stream /// according to the specified 'parameters'. static void verifyParameters(const Parameters& parameters); + /// Verify the specified compression 'type' honors the configured maximum + /// number of bytes inflated and deflated per operation. + static void verifyLimitsForType(ntca::CompressionType::Value type); + + /// Verify the specified compression 'type' rejects malformed input, and + /// resets after the failure. + static void verifyMalformedForType(ntca::CompressionType::Value type); + + /// Load into the specified 'result' the compression types supported by + /// this build. + static void loadTypes(bsl::vector* result); + + /// Return a new data pool. + static bsl::shared_ptr createDataPool(); + + /// Return a new compression mechanism created according to the specified + /// 'configuration' that allocates data containers from the specified + /// 'dataPool'. + static bsl::shared_ptr createCompression( + const ntca::CompressionConfig& configuration, + const bsl::shared_ptr& dataPool); + + /// Deflate the specified 'data' using the specified 'compression' and + /// append the result to the specified 'result'. Return the error. + static ntsa::Error deflate( + const bsl::shared_ptr& compression, + bdlbb::Blob* result, + const bdlbb::Blob& data); + + /// Inflate the specified 'data' using the specified 'compression' and + /// append the result to the specified 'result'. Return the error. + static ntsa::Error inflate( + const bsl::shared_ptr& compression, + bdlbb::Blob* result, + const bdlbb::Blob& data); + /// Declare the log category for this class. BALL_LOG_SET_CLASS_CATEGORY("NTC.COMPRESSION"); }; @@ -851,5 +901,612 @@ NTSCFG_TEST_FUNCTION(ntctlc::PluginTest::verifyUsage) #endif } +void PluginTest::loadTypes(bsl::vector* result) +{ +#if NTC_BUILD_WITH_LZ4 + result->push_back(ntca::CompressionType::e_LZ4); +#endif + +#if NTC_BUILD_WITH_ZSTD + result->push_back(ntca::CompressionType::e_ZSTD); +#endif + +#if NTC_BUILD_WITH_ZLIB + result->push_back(ntca::CompressionType::e_ZLIB); + result->push_back(ntca::CompressionType::e_GZIP); +#endif +} + +bsl::shared_ptr PluginTest::createDataPool() +{ + bsl::shared_ptr dataPool; + dataPool.createInplace(NTSCFG_TEST_ALLOCATOR, + 4096, + 4096, + NTSCFG_TEST_ALLOCATOR); + + return dataPool; +} + +bsl::shared_ptr PluginTest::createCompression( + const ntca::CompressionConfig& configuration, + const bsl::shared_ptr& dataPool) +{ + bsl::shared_ptr driver; + ntctlc::Plugin::load(&driver); + + bsl::shared_ptr compression; + ntsa::Error error = driver->createCompression(&compression, + configuration, + dataPool, + NTSCFG_TEST_ALLOCATOR); + NTSCFG_TEST_OK(error); + + return compression; +} + +ntsa::Error PluginTest::deflate( + const bsl::shared_ptr& compression, + bdlbb::Blob* result, + const bdlbb::Blob& data) +{ + ntca::DeflateOptions deflateOptions; + ntca::DeflateContext deflateContext; + + return compression->deflate(&deflateContext, + result, + data, + deflateOptions); +} + +ntsa::Error PluginTest::inflate( + const bsl::shared_ptr& compression, + bdlbb::Blob* result, + const bdlbb::Blob& data) +{ + ntca::InflateOptions inflateOptions; + ntca::InflateContext inflateContext; + + return compression->inflate(&inflateContext, + result, + data, + inflateOptions); +} + +void PluginTest::verifyLimitsForType(ntca::CompressionType::Value type) +{ + ntsa::Error error; + + BALL_LOG_INFO << "Testing limits for " << type << BALL_LOG_END; + + bsl::shared_ptr dataPool = PluginTest::createDataPool(); + + ntca::CompressionConfig baseConfig; + baseConfig.setType(type); + baseConfig.setGoal(ntca::CompressionGoal::e_BALANCED); + + bsl::shared_ptr unlimitedCompression = + PluginTest::createCompression(baseConfig, dataPool); + + // Generate the original data and its deflated form. + + bsl::shared_ptr original = dataPool->createIncomingBlob(); + ntscfg::TestDataUtil::generateData( + original.get(), + 64 * 1024, + 0, + ntscfg::TestDataUtil::k_DATASET_CLIENT_COMPRESSABLE); + + const bsl::size_t k_ORIGINAL_SIZE = + static_cast(original->length()); + + bsl::shared_ptr deflated = dataPool->createOutgoingBlob(); + error = PluginTest::deflate(unlimitedCompression, deflated.get(), *original); + NTSCFG_TEST_OK(error); + + const bsl::size_t k_DEFLATED_SIZE = + static_cast(deflated->length()); + + // Generate a small message and its deflated form. + + const char k_SMALL[] = "abbcccddddeeeffg"; + + bsl::shared_ptr small = dataPool->createIncomingBlob(); + bdlbb::BlobUtil::append(small.get(), k_SMALL, sizeof k_SMALL - 1); + + const bsl::size_t k_SMALL_SIZE = static_cast(small->length()); + + bsl::shared_ptr smallDeflated = + dataPool->createOutgoingBlob(); + error = PluginTest::deflate(unlimitedCompression, + smallDeflated.get(), + *small); + NTSCFG_TEST_OK(error); + + const bsl::size_t k_SMALL_DEFLATED_SIZE = + static_cast(smallDeflated->length()); + + // Concern: an inflate operation succeeds if and only if it produces no + // more than the maximum inflate size, never appends more than the + // maximum inflate size, and the inflater resets after a failure. + + { + const bsl::size_t k_LIMIT[] = {0, + 1, + k_ORIGINAL_SIZE / 2, + k_ORIGINAL_SIZE - 1, + k_ORIGINAL_SIZE, + k_ORIGINAL_SIZE + 1}; + + for (bsl::size_t i = 0; i < sizeof k_LIMIT / sizeof k_LIMIT[0]; ++i) + { + const bsl::size_t limit = k_LIMIT[i]; + + ntca::CompressionConfig config = baseConfig; + config.setMaxInflateSize(limit); + + bsl::shared_ptr compression = + PluginTest::createCompression(config, dataPool); + + bsl::shared_ptr inflated = + dataPool->createIncomingBlob(); + + error = PluginTest::inflate(compression, inflated.get(), *deflated); + + if (limit >= k_ORIGINAL_SIZE) { + NTSCFG_TEST_OK(error); + NTSCFG_TEST_EQ(bdlbb::BlobUtil::compare(*inflated, *original), + 0); + } + else { + NTSCFG_TEST_ERROR(error, ntsa::Error::e_LIMIT); + NTSCFG_TEST_LE(static_cast(inflated->length()), + limit); + + if (limit >= k_SMALL_SIZE) { + bsl::shared_ptr smallInflated = + dataPool->createIncomingBlob(); + + error = PluginTest::inflate(compression, + smallInflated.get(), + *smallDeflated); + NTSCFG_TEST_OK(error); + + NTSCFG_TEST_EQ( + bdlbb::BlobUtil::compare(*smallInflated, *small), + 0); + } + } + } + } + + // Concern: the maximum inflate size applies to the entire operation, even + // when the operation is fed its input across many calls. Note that + // inflating a blob composed of many buffers feeds each buffer to the + // inflater in a separate call. + + { + bdlbb::SimpleBlobBufferFactory singleByteBufferFactory( + 1, + NTSCFG_TEST_ALLOCATOR); + + bsl::vector bytes(k_DEFLATED_SIZE); + bdlbb::BlobUtil::copy(&bytes[0], + *deflated, + 0, + static_cast(k_DEFLATED_SIZE)); + + bdlbb::Blob fragmented(&singleByteBufferFactory, + NTSCFG_TEST_ALLOCATOR); + bdlbb::BlobUtil::append(&fragmented, + &bytes[0], + static_cast(bytes.size())); + + NTSCFG_TEST_EQ(static_cast(fragmented.numDataBuffers()), + k_DEFLATED_SIZE); + + const bsl::size_t k_LIMIT[] = {k_ORIGINAL_SIZE - 1, k_ORIGINAL_SIZE}; + + for (bsl::size_t i = 0; i < sizeof k_LIMIT / sizeof k_LIMIT[0]; ++i) + { + const bsl::size_t limit = k_LIMIT[i]; + + ntca::CompressionConfig config = baseConfig; + config.setMaxInflateSize(limit); + + bsl::shared_ptr compression = + PluginTest::createCompression(config, dataPool); + + bsl::shared_ptr inflated = + dataPool->createIncomingBlob(); + + error = PluginTest::inflate(compression, inflated.get(), fragmented); + + if (limit >= k_ORIGINAL_SIZE) { + NTSCFG_TEST_OK(error); + NTSCFG_TEST_EQ(bdlbb::BlobUtil::compare(*inflated, *original), + 0); + } + else { + NTSCFG_TEST_ERROR(error, ntsa::Error::e_LIMIT); + NTSCFG_TEST_LE(static_cast(inflated->length()), + limit); + } + } + } + + // Concern: the maximum inflate size applies to the entire operation when + // the operation is fed its input as an array of buffers, and the failure + // is reported to the caller. + + { + bsl::vector bytes(k_DEFLATED_SIZE); + bdlbb::BlobUtil::copy(&bytes[0], + *deflated, + 0, + static_cast(k_DEFLATED_SIZE)); + + const bsl::size_t k_LIMIT[] = {k_ORIGINAL_SIZE - 1, k_ORIGINAL_SIZE}; + + for (bsl::size_t i = 0; i < sizeof k_LIMIT / sizeof k_LIMIT[0]; ++i) + { + const bsl::size_t limit = k_LIMIT[i]; + + ntca::CompressionConfig config = baseConfig; + config.setMaxInflateSize(limit); + + bsl::shared_ptr compression = + PluginTest::createCompression(config, dataPool); + + ntsa::Data data(NTSCFG_TEST_ALLOCATOR); + ntsa::ConstBufferArray& bufferArray = data.makeConstBufferArray(); + for (bsl::size_t j = 0; j < bytes.size(); ++j) { + bufferArray.append(&bytes[j], 1); + } + + bsl::shared_ptr inflated = + dataPool->createIncomingBlob(); + + ntca::InflateOptions inflateOptions; + ntca::InflateContext inflateContext; + + error = compression->inflate(&inflateContext, + inflated.get(), + data, + inflateOptions); + + if (limit >= k_ORIGINAL_SIZE) { + NTSCFG_TEST_OK(error); + NTSCFG_TEST_EQ(bdlbb::BlobUtil::compare(*inflated, *original), + 0); + } + else { + NTSCFG_TEST_ERROR(error, ntsa::Error::e_LIMIT); + NTSCFG_TEST_LE(static_cast(inflated->length()), + limit); + } + } + } + + // Concern: the maximum inflate size applies to each operation, not + // cumulatively across operations. + + { + ntca::CompressionConfig config = baseConfig; + config.setMaxInflateSize(k_ORIGINAL_SIZE); + + bsl::shared_ptr compression = + PluginTest::createCompression(config, dataPool); + + for (bsl::size_t i = 0; i < 3; ++i) { + bsl::shared_ptr inflated = + dataPool->createIncomingBlob(); + + error = PluginTest::inflate(compression, inflated.get(), *deflated); + NTSCFG_TEST_OK(error); + + NTSCFG_TEST_EQ(bdlbb::BlobUtil::compare(*inflated, *original), 0); + } + } + + // Concern: a deflate operation succeeds if and only if it produces no + // more than the maximum deflate size, never appends more than the + // maximum deflate size, and leaves no residue that is appended to the + // result of a subsequent operation. + + { + const bsl::size_t k_LIMIT[] = {0, + 1, + k_DEFLATED_SIZE / 2, + k_DEFLATED_SIZE - 1, + k_DEFLATED_SIZE, + k_DEFLATED_SIZE + 1}; + + for (bsl::size_t i = 0; i < sizeof k_LIMIT / sizeof k_LIMIT[0]; ++i) + { + const bsl::size_t limit = k_LIMIT[i]; + + ntca::CompressionConfig config = baseConfig; + config.setMaxDeflateSize(limit); + + bsl::shared_ptr compression = + PluginTest::createCompression(config, dataPool); + + bsl::shared_ptr output = + dataPool->createOutgoingBlob(); + + error = PluginTest::deflate(compression, output.get(), *original); + + if (limit >= k_DEFLATED_SIZE) { + NTSCFG_TEST_OK(error); + NTSCFG_TEST_EQ(static_cast(output->length()), + k_DEFLATED_SIZE); + + bsl::shared_ptr inflated = + dataPool->createIncomingBlob(); + + error = PluginTest::inflate(unlimitedCompression, + inflated.get(), + *output); + NTSCFG_TEST_OK(error); + + NTSCFG_TEST_EQ(bdlbb::BlobUtil::compare(*inflated, *original), + 0); + } + else { + NTSCFG_TEST_ERROR(error, ntsa::Error::e_LIMIT); + NTSCFG_TEST_LE(static_cast(output->length()), + limit); + + if (limit >= k_SMALL_DEFLATED_SIZE) { + bsl::shared_ptr smallOutput = + dataPool->createOutgoingBlob(); + + error = PluginTest::deflate(compression, + smallOutput.get(), + *small); + NTSCFG_TEST_OK(error); + + NTSCFG_TEST_EQ( + static_cast(smallOutput->length()), + k_SMALL_DEFLATED_SIZE); + + bsl::shared_ptr smallInflated = + dataPool->createIncomingBlob(); + + error = PluginTest::inflate(unlimitedCompression, + smallInflated.get(), + *smallOutput); + NTSCFG_TEST_OK(error); + + NTSCFG_TEST_EQ( + bdlbb::BlobUtil::compare(*smallInflated, *small), + 0); + } + } + } + } + + // Concern: inflating a highly-compressed payload fails once the maximum + // inflate size is reached, without inflating the entire payload. + + { + const bsl::size_t k_BOMB_SIZE = 16 * 1024 * 1024; + const bsl::size_t k_BOMB_LIMIT = 1024 * 1024; + + bsl::vector zeros(64 * 1024, 0); + + bsl::shared_ptr bomb = dataPool->createIncomingBlob(); + for (bsl::size_t i = 0; i < k_BOMB_SIZE / zeros.size(); ++i) { + bdlbb::BlobUtil::append(bomb.get(), + &zeros[0], + static_cast(zeros.size())); + } + + bsl::shared_ptr bombDeflated = + dataPool->createOutgoingBlob(); + error = PluginTest::deflate(unlimitedCompression, + bombDeflated.get(), + *bomb); + NTSCFG_TEST_OK(error); + + ntca::CompressionConfig config = baseConfig; + config.setMaxInflateSize(k_BOMB_LIMIT); + + bsl::shared_ptr compression = + PluginTest::createCompression(config, dataPool); + + bsl::shared_ptr inflated = dataPool->createIncomingBlob(); + + error = PluginTest::inflate(compression, inflated.get(), *bombDeflated); + NTSCFG_TEST_ERROR(error, ntsa::Error::e_LIMIT); + + NTSCFG_TEST_LE(static_cast(inflated->length()), + k_BOMB_LIMIT); + } +} + +void PluginTest::verifyMalformedForType(ntca::CompressionType::Value type) +{ + ntsa::Error error; + + BALL_LOG_INFO << "Testing malformed input for " << type << BALL_LOG_END; + + bsl::shared_ptr dataPool = PluginTest::createDataPool(); + + ntca::CompressionConfig config; + config.setType(type); + config.setGoal(ntca::CompressionGoal::e_BALANCED); + + bsl::shared_ptr compression = + PluginTest::createCompression(config, dataPool); + + const char k_SMALL[] = "abbcccddddeeeffg"; + + bsl::shared_ptr small = dataPool->createIncomingBlob(); + bdlbb::BlobUtil::append(small.get(), k_SMALL, sizeof k_SMALL - 1); + + bsl::shared_ptr smallDeflated = + dataPool->createOutgoingBlob(); + error = PluginTest::deflate(compression, smallDeflated.get(), *small); + NTSCFG_TEST_OK(error); + + const char k_GARBAGE[] = "This is not a compressed frame!"; + + for (bsl::size_t i = 0; i < 2; ++i) { + // Concern: malformed input is rejected. + + bsl::shared_ptr garbage = dataPool->createIncomingBlob(); + bdlbb::BlobUtil::append(garbage.get(), k_GARBAGE, sizeof k_GARBAGE - 1); + + bsl::shared_ptr inflated = dataPool->createIncomingBlob(); + + error = PluginTest::inflate(compression, inflated.get(), *garbage); + NTSCFG_TEST_ERROR(error, ntsa::Error::e_INVALID); + + // Concern: the inflater resets after the failure, so a subsequent, + // well-formed frame is inflated, and no residue from the failed + // operation is appended to its result. + + bsl::shared_ptr smallInflated = + dataPool->createIncomingBlob(); + + error = PluginTest::inflate(compression, + smallInflated.get(), + *smallDeflated); + NTSCFG_TEST_OK(error); + + NTSCFG_TEST_EQ(bdlbb::BlobUtil::compare(*smallInflated, *small), 0); + } +} + +NTSCFG_TEST_FUNCTION(ntctlc::PluginTest::verifyLimits) +{ + bsl::vector typeVector; + PluginTest::loadTypes(&typeVector); + + for (bsl::size_t i = 0; i < typeVector.size(); ++i) { + PluginTest::verifyLimitsForType(typeVector[i]); + } +} + +NTSCFG_TEST_FUNCTION(ntctlc::PluginTest::verifyMalformed) +{ + bsl::vector typeVector; + PluginTest::loadTypes(&typeVector); + + for (bsl::size_t i = 0; i < typeVector.size(); ++i) { + PluginTest::verifyMalformedForType(typeVector[i]); + } +} + +NTSCFG_TEST_FUNCTION(ntctlc::PluginTest::verifyZstdWindow) +{ +#if NTC_BUILD_WITH_ZSTD + + ntsa::Error error; + + bsl::shared_ptr dataPool = PluginTest::createDataPool(); + + ntca::CompressionConfig config; + config.setType(ntca::CompressionType::e_ZSTD); + config.setGoal(ntca::CompressionGoal::e_BALANCED); + + bsl::shared_ptr compression = + PluginTest::createCompression(config, dataPool); + + // A frame having no content size, declaring a window of the specified + // size, containing a single, last, raw block of the single byte 'x'. + + const unsigned char k_FRAME_WINDOW_2_24[] = + {0x28, 0xB5, 0x2F, 0xFD, 0x00, 0x70, 0x09, 0x00, 0x00, 'x'}; + + const unsigned char k_FRAME_WINDOW_2_23[] = + {0x28, 0xB5, 0x2F, 0xFD, 0x00, 0x68, 0x09, 0x00, 0x00, 'x'}; + + // Concern: a frame declaring a window larger than the maximum is + // rejected. + + { + bsl::shared_ptr frame = dataPool->createIncomingBlob(); + bdlbb::BlobUtil::append(frame.get(), + reinterpret_cast( + k_FRAME_WINDOW_2_24), + sizeof k_FRAME_WINDOW_2_24); + + bsl::shared_ptr inflated = dataPool->createIncomingBlob(); + + error = PluginTest::inflate(compression, inflated.get(), *frame); + NTSCFG_TEST_ERROR(error, ntsa::Error::e_INVALID); + + NTSCFG_TEST_EQ(inflated->length(), 0); + } + + // Concern: a frame declaring a window equal to the maximum is accepted. + + { + bsl::shared_ptr frame = dataPool->createIncomingBlob(); + bdlbb::BlobUtil::append(frame.get(), + reinterpret_cast( + k_FRAME_WINDOW_2_23), + sizeof k_FRAME_WINDOW_2_23); + + bsl::shared_ptr inflated = dataPool->createIncomingBlob(); + + error = PluginTest::inflate(compression, inflated.get(), *frame); + NTSCFG_TEST_OK(error); + + bsl::shared_ptr expected = dataPool->createIncomingBlob(); + bdlbb::BlobUtil::append(expected.get(), "x", 1); + + NTSCFG_TEST_EQ(bdlbb::BlobUtil::compare(*inflated, *expected), 0); + } + + // Concern: frames produced by the deflater at every compression goal, + // including those whose default window exceeds the maximum, may be + // inflated. + + { + bsl::shared_ptr original = dataPool->createIncomingBlob(); + ntscfg::TestDataUtil::generateData( + original.get(), + 64 * 1024, + 0, + ntscfg::TestDataUtil::k_DATASET_CLIENT_COMPRESSABLE); + + const ntca::CompressionGoal::Value k_GOAL[] = { + ntca::CompressionGoal::e_BEST_SIZE, + ntca::CompressionGoal::e_BETTER_SIZE, + ntca::CompressionGoal::e_BALANCED, + ntca::CompressionGoal::e_BETTER_SPEED, + ntca::CompressionGoal::e_BEST_SPEED}; + + for (bsl::size_t i = 0; i < sizeof k_GOAL / sizeof k_GOAL[0]; ++i) { + ntca::CompressionConfig deflaterConfig; + deflaterConfig.setType(ntca::CompressionType::e_ZSTD); + deflaterConfig.setGoal(k_GOAL[i]); + + bsl::shared_ptr deflater = + PluginTest::createCompression(deflaterConfig, dataPool); + + bsl::shared_ptr deflated = + dataPool->createOutgoingBlob(); + + error = PluginTest::deflate(deflater, deflated.get(), *original); + NTSCFG_TEST_OK(error); + + bsl::shared_ptr inflated = + dataPool->createIncomingBlob(); + + error = PluginTest::inflate(compression, inflated.get(), *deflated); + NTSCFG_TEST_OK(error); + + NTSCFG_TEST_EQ(bdlbb::BlobUtil::compare(*inflated, *original), 0); + } + } + +#endif +} + } // close namespace ntctlc } // close namespace BloombergLP From 3ba3280ada053393e18e8ca2147682d3cc2e4b42 Mon Sep 17 00:00:00 2001 From: Matt Millett Date: Fri, 2 Oct 2026 09:04:41 -0400 Subject: [PATCH 2/3] Remove artificial Zstd compression window caps and restore compatibility with all compression levels --- groups/ntc/ntctlc/ntctlc_plugin.cpp | 34 -------- groups/ntc/ntctlc/ntctlc_plugin.t.cpp | 113 -------------------------- 2 files changed, 147 deletions(-) diff --git a/groups/ntc/ntctlc/ntctlc_plugin.cpp b/groups/ntc/ntctlc/ntctlc_plugin.cpp index a68337da..843bfca2 100644 --- a/groups/ntc/ntctlc/ntctlc_plugin.cpp +++ b/groups/ntc/ntctlc/ntctlc_plugin.cpp @@ -377,12 +377,6 @@ class Lz4 : public ntci::Compression /// @ingroup module_ntctlc class Zstd : public ntci::Compression { - /// The base-2 logarithm of the maximum window size used by the deflater - /// and accepted by the inflater. This bounds the memory the inflater - /// allocates for a frame, regardless of the window size the frame - /// declares. - static const int k_WINDOW_LOG_MAX = 23; - ZSTD_CCtx* d_deflaterContext_p; ZSTD_inBuffer d_deflaterInput; ZSTD_outBuffer d_deflaterOutput; @@ -1783,24 +1777,6 @@ ntsa::Error Zstd::deflateCreate() return Zstd::translateError(rc, "set checksum flag"); } - // Limit the window size to that accepted by the inflater, so that every - // frame produced by the deflater may be inflated, but do not increase - // the window size beyond the default for the compression level. - - const ZSTD_compressionParameters compressionParameters = - ZSTD_getCParams(d_level, ZSTD_CONTENTSIZE_UNKNOWN, 0); - - if (compressionParameters.windowLog > - static_cast(k_WINDOW_LOG_MAX)) - { - rc = ZSTD_CCtx_setParameter(d_deflaterContext_p, - ZSTD_c_windowLog, - k_WINDOW_LOG_MAX); - if (ZSTD_isError(rc)) { - return Zstd::translateError(rc, "set window log"); - } - } - bsl::memset(&d_deflaterInput, 0, sizeof d_deflaterInput); bsl::memset(&d_deflaterOutput, 0, sizeof d_deflaterOutput); @@ -2094,16 +2070,6 @@ ntsa::Error Zstd::inflateCreate() #endif - // Reject frames whose declared window size exceeds the maximum, which - // bounds the memory allocated to inflate each frame. - - bsl::size_t rc = ZSTD_DCtx_setParameter(d_inflaterContext_p, - ZSTD_d_windowLogMax, - k_WINDOW_LOG_MAX); - if (ZSTD_isError(rc)) { - return Zstd::translateError(rc, "set maximum window log"); - } - bsl::memset(&d_inflaterInput, 0, sizeof d_inflaterInput); bsl::memset(&d_inflaterOutput, 0, sizeof d_inflaterOutput); diff --git a/groups/ntc/ntctlc/ntctlc_plugin.t.cpp b/groups/ntc/ntctlc/ntctlc_plugin.t.cpp index d76a1806..1563aff7 100644 --- a/groups/ntc/ntctlc/ntctlc_plugin.t.cpp +++ b/groups/ntc/ntctlc/ntctlc_plugin.t.cpp @@ -71,11 +71,6 @@ class PluginTest // the failure such that subsequent, well-formed input is inflated. static void verifyMalformed(); - // Verify the zstd inflater rejects frames whose declared window size - // exceeds the maximum, and that the zstd deflater never produces such - // frames. - static void verifyZstdWindow(); - private: /// Verify the integrity of inflating and deflating a data stream /// according to the specified 'parameters'. @@ -1400,113 +1395,5 @@ NTSCFG_TEST_FUNCTION(ntctlc::PluginTest::verifyMalformed) } } -NTSCFG_TEST_FUNCTION(ntctlc::PluginTest::verifyZstdWindow) -{ -#if NTC_BUILD_WITH_ZSTD - - ntsa::Error error; - - bsl::shared_ptr dataPool = PluginTest::createDataPool(); - - ntca::CompressionConfig config; - config.setType(ntca::CompressionType::e_ZSTD); - config.setGoal(ntca::CompressionGoal::e_BALANCED); - - bsl::shared_ptr compression = - PluginTest::createCompression(config, dataPool); - - // A frame having no content size, declaring a window of the specified - // size, containing a single, last, raw block of the single byte 'x'. - - const unsigned char k_FRAME_WINDOW_2_24[] = - {0x28, 0xB5, 0x2F, 0xFD, 0x00, 0x70, 0x09, 0x00, 0x00, 'x'}; - - const unsigned char k_FRAME_WINDOW_2_23[] = - {0x28, 0xB5, 0x2F, 0xFD, 0x00, 0x68, 0x09, 0x00, 0x00, 'x'}; - - // Concern: a frame declaring a window larger than the maximum is - // rejected. - - { - bsl::shared_ptr frame = dataPool->createIncomingBlob(); - bdlbb::BlobUtil::append(frame.get(), - reinterpret_cast( - k_FRAME_WINDOW_2_24), - sizeof k_FRAME_WINDOW_2_24); - - bsl::shared_ptr inflated = dataPool->createIncomingBlob(); - - error = PluginTest::inflate(compression, inflated.get(), *frame); - NTSCFG_TEST_ERROR(error, ntsa::Error::e_INVALID); - - NTSCFG_TEST_EQ(inflated->length(), 0); - } - - // Concern: a frame declaring a window equal to the maximum is accepted. - - { - bsl::shared_ptr frame = dataPool->createIncomingBlob(); - bdlbb::BlobUtil::append(frame.get(), - reinterpret_cast( - k_FRAME_WINDOW_2_23), - sizeof k_FRAME_WINDOW_2_23); - - bsl::shared_ptr inflated = dataPool->createIncomingBlob(); - - error = PluginTest::inflate(compression, inflated.get(), *frame); - NTSCFG_TEST_OK(error); - - bsl::shared_ptr expected = dataPool->createIncomingBlob(); - bdlbb::BlobUtil::append(expected.get(), "x", 1); - - NTSCFG_TEST_EQ(bdlbb::BlobUtil::compare(*inflated, *expected), 0); - } - - // Concern: frames produced by the deflater at every compression goal, - // including those whose default window exceeds the maximum, may be - // inflated. - - { - bsl::shared_ptr original = dataPool->createIncomingBlob(); - ntscfg::TestDataUtil::generateData( - original.get(), - 64 * 1024, - 0, - ntscfg::TestDataUtil::k_DATASET_CLIENT_COMPRESSABLE); - - const ntca::CompressionGoal::Value k_GOAL[] = { - ntca::CompressionGoal::e_BEST_SIZE, - ntca::CompressionGoal::e_BETTER_SIZE, - ntca::CompressionGoal::e_BALANCED, - ntca::CompressionGoal::e_BETTER_SPEED, - ntca::CompressionGoal::e_BEST_SPEED}; - - for (bsl::size_t i = 0; i < sizeof k_GOAL / sizeof k_GOAL[0]; ++i) { - ntca::CompressionConfig deflaterConfig; - deflaterConfig.setType(ntca::CompressionType::e_ZSTD); - deflaterConfig.setGoal(k_GOAL[i]); - - bsl::shared_ptr deflater = - PluginTest::createCompression(deflaterConfig, dataPool); - - bsl::shared_ptr deflated = - dataPool->createOutgoingBlob(); - - error = PluginTest::deflate(deflater, deflated.get(), *original); - NTSCFG_TEST_OK(error); - - bsl::shared_ptr inflated = - dataPool->createIncomingBlob(); - - error = PluginTest::inflate(compression, inflated.get(), *deflated); - NTSCFG_TEST_OK(error); - - NTSCFG_TEST_EQ(bdlbb::BlobUtil::compare(*inflated, *original), 0); - } - } - -#endif -} - } // close namespace ntctlc } // close namespace BloombergLP From 84d1c7347f5763d885eca170d8e25a09c1232d29 Mon Sep 17 00:00:00 2001 From: Matt Millett Date: Fri, 2 Oct 2026 09:33:27 -0400 Subject: [PATCH 3/3] Test compression hardening over all compression algorithms --- groups/ntc/ntctlc/ntctlc_plugin.t.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/groups/ntc/ntctlc/ntctlc_plugin.t.cpp b/groups/ntc/ntctlc/ntctlc_plugin.t.cpp index 1563aff7..7571581e 100644 --- a/groups/ntc/ntctlc/ntctlc_plugin.t.cpp +++ b/groups/ntc/ntctlc/ntctlc_plugin.t.cpp @@ -33,7 +33,7 @@ namespace ntctlc { // #define NTCTLC_PLUGIN_TEST_VARIATION 1339 // Uncomment to test only a specific compression type. -#define NTCTLC_PLUGIN_TEST_COMPRESSION_TYPE ntca::CompressionType::e_LZ4 +// #define NTCTLC_PLUGIN_TEST_COMPRESSION_TYPE ntca::CompressionType::e_LZ4 // Provide tests for 'ntctlc::Plugin'. class PluginTest