Skip to content

Fix canonical integers and truncation errors #319

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
mrabine merged 4 commits into from
Nov 14, 2026
Merged

Fix canonical integers and truncation errors #319

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
8268e3f
Fix canonical integers and truncation errors
mrabine Nov 14, 2026
0913c29
Write CBOR floats in half precision when exact
mrabine Nov 14, 2026
0066f36
Report write errors on the output stream
mrabine Nov 14, 2026
86c45e9
Bound CBOR writer buffer for the optimizer
mrabine Nov 14, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
87 changes: 82 additions & 5 deletions data/include/join/cbor.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -154,6 +154,7 @@ namespace join
*/
explicit BasicCborWriter (std::ostream& document)
: _out (document.rdbuf ())
, _document (&document)
{
}

Expand Down Expand Up @@ -281,8 +282,17 @@ namespace join
float f = static_cast<float> (value);
if (std::bit_cast<uint64_t> (static_cast<double> (f)) == std::bit_cast<uint64_t> (value))
{
append (0xfa);
pack (f);
uint16_t half;
if (toHalf (f, half))
{
append (0xf9);
pack (half);
}
else
{
append (0xfa);
pack (f);
}
}
else
{
Expand Down Expand Up @@ -547,7 +557,10 @@ namespace join
{
if (_size)
{
_out->sputn (_buffer, _size);
if (JOIN_UNLIKELY (_out->sputn (_buffer, _size) != static_cast<std::streamsize> (_size)))
{
_document->setstate (std::ios_base::badbit);
}
_size = 0;
}
}
Expand Down Expand Up @@ -582,6 +595,10 @@ namespace join
*/
inline void append (const char* data, size_t size)
{
if (_size > sizeof (_buffer))
{
__builtin_unreachable ();
}
if (JOIN_UNLIKELY (size == 0))
{
return;
Expand All @@ -591,7 +608,10 @@ namespace join
flush ();
if (size >= sizeof (_buffer))
{
_out->sputn (data, size);
if (JOIN_UNLIKELY (_out->sputn (data, size) != static_cast<std::streamsize> (size)))
{
_document->setstate (std::ios_base::badbit);
}
return;
}
}
Expand All @@ -609,6 +629,60 @@ namespace join
append (reinterpret_cast<const char*> (&swap (value)), sizeof (value));
}

/**
* @brief convert a float to half precision when no information is lost.
* @param value float value to convert.
* @param half half precision bits.
* @return true if the value is exactly representable in half precision, false otherwise.
*/
static bool toHalf (float value, uint16_t& half) noexcept
{
const uint32_t bits = std::bit_cast<uint32_t> (value);
const uint16_t sign = static_cast<uint16_t> ((bits >> 16) & 0x8000);
const uint32_t exponent = (bits >> 23) & 0xff;
const uint32_t mantissa = bits & 0x7fffff;

if (exponent == 0xff)
{
if ((mantissa != 0) && (mantissa != 0x400000))
{
return false;
}
half = sign | 0x7c00 | static_cast<uint16_t> (mantissa >> 13);
return true;
}

if ((exponent == 0) && (mantissa == 0))
{
half = sign;
return true;
}

if ((exponent >= 113) && (exponent <= 142))
{
if ((mantissa & 0x1fff) != 0)
{
return false;
}
half = sign | static_cast<uint16_t> (((exponent - 112) << 10) | (mantissa >> 13));
return true;
}

if ((exponent >= 103) && (exponent <= 112))
{
const uint32_t significand = 0x800000 | mantissa;
const uint32_t shift = 126 - exponent;
if ((significand & ((1u << shift) - 1)) != 0)
{
return false;
}
half = sign | static_cast<uint16_t> (significand >> shift);
return true;
}

return false;
}

/**
* @brief write a data item head with the shortest argument encoding.
* @param major major type, already shifted.
Expand Down Expand Up @@ -656,6 +730,9 @@ namespace join
/// underlying output stream.
std::streambuf* _out;

/// output stream receiving the error state.
std::ostream* _document;

/// output buffer.
char _buffer[1024] = {};

Expand Down Expand Up @@ -1126,7 +1203,7 @@ namespace join
return -1;
}

if (JOIN_UNLIKELY (len > std::numeric_limits<uint32_t>::max ()))
if (JOIN_UNLIKELY (len > std::numeric_limits<uint32_t>::max () - output.size ()))
{
join::lastError = make_error_code (SaxErrc::InvalidValue);
return -1;
Expand Down
72 changes: 63 additions & 9 deletions data/include/join/json.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -209,6 +209,7 @@ namespace join
*/
explicit BasicJsonWriter (std::ostream& document, size_t indentation = 0)
: _out (document.rdbuf ())
, _document (&document)
, _indentation ((WriteMode == JsonWriteMode::Canonical) ? 0 : indentation)
{
}
Expand Down Expand Up @@ -636,7 +637,10 @@ namespace join
{
if (_size)
{
_out->sputn (_buffer, _size);
if (JOIN_UNLIKELY (_out->sputn (_buffer, _size) != static_cast<std::streamsize> (_size)))
{
_document->setstate (std::ios_base::badbit);
}
_size = 0;
}
}
Expand Down Expand Up @@ -773,6 +777,10 @@ namespace join
{
char buf[25];
char* end = dconv::dtoa (buf, value);
if ((end - buf > 2) && (end[-2] == '.') && (end[-1] == '0'))
{
end -= 2;
}
for (char* pos = buf; pos < end; ++pos)
{
append (*pos);
Expand Down Expand Up @@ -880,7 +888,10 @@ namespace join
flush ();
if (size >= sizeof (_buffer))
{
_out->sputn (data, size);
if (JOIN_UNLIKELY (_out->sputn (data, size) != static_cast<std::streamsize> (size)))
{
_document->setstate (std::ios_base::badbit);
}
return;
}
}
Expand Down Expand Up @@ -992,6 +1003,9 @@ namespace join
/// underlying output stream.
std::streambuf* _out;

/// output stream receiving the error state.
std::ostream* _document;

/// output buffer.
char _buffer[1024] = {};

Expand Down Expand Up @@ -1284,6 +1298,9 @@ namespace join
case 'f':
document.get ();
return readFalse (document, handler);
case std::char_traits<char>::eof ():
join::lastError = make_error_code (SaxErrc::Truncated);
return -1;
default:
return readNumber (document, handler);
}
Expand Down Expand Up @@ -1581,7 +1598,14 @@ namespace join

if (ch < 0x20)
{
join::lastError = make_error_code (JsonErrc::IllegalCharacter);
if (ch == std::char_traits<char>::eof ())
{
join::lastError = make_error_code (SaxErrc::Truncated);
}
else
{
join::lastError = make_error_code (JsonErrc::IllegalCharacter);
}
return -1;
}
}
Expand Down Expand Up @@ -1654,7 +1678,14 @@ namespace join
break;
}

join::lastError = make_error_code (JsonErrc::MissingComma);
if (ch == std::char_traits<char>::eof ())
{
join::lastError = make_error_code (SaxErrc::Truncated);
}
else
{
join::lastError = make_error_code (JsonErrc::MissingComma);
}
return -1;
}

Expand Down Expand Up @@ -1694,9 +1725,17 @@ namespace join

for (;;)
{
if (JOIN_UNLIKELY (document.get () != '"'))
int quote = document.get ();
if (JOIN_UNLIKELY (quote != '"'))
{
join::lastError = make_error_code (JsonErrc::MissingQuote);
if (quote == std::char_traits<char>::eof ())
{
join::lastError = make_error_code (SaxErrc::Truncated);
}
else
{
join::lastError = make_error_code (JsonErrc::MissingQuote);
}
return -1;
}

Expand All @@ -1713,9 +1752,17 @@ namespace join
}
}

if (JOIN_UNLIKELY (document.get () != ':'))
int colon = document.get ();
if (JOIN_UNLIKELY (colon != ':'))
{
join::lastError = make_error_code (JsonErrc::MissingColon);
if (colon == std::char_traits<char>::eof ())
{
join::lastError = make_error_code (SaxErrc::Truncated);
}
else
{
join::lastError = make_error_code (JsonErrc::MissingColon);
}
return -1;
}

Expand Down Expand Up @@ -1754,7 +1801,14 @@ namespace join
break;
}

join::lastError = make_error_code (JsonErrc::MissingComma);
if (ch == std::char_traits<char>::eof ())
{
join::lastError = make_error_code (SaxErrc::Truncated);
}
else
{
join::lastError = make_error_code (JsonErrc::MissingComma);
}
return -1;
}

Expand Down
14 changes: 12 additions & 2 deletions data/include/join/pack.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -74,6 +74,7 @@ namespace join
*/
explicit BasicPackWriter (std::ostream& document)
: _out (document.rdbuf ())
, _document (&document)
{
}

Expand Down Expand Up @@ -649,7 +650,10 @@ namespace join
{
if (_size)
{
_out->sputn (_buffer, _size);
if (JOIN_UNLIKELY (_out->sputn (_buffer, _size) != static_cast<std::streamsize> (_size)))
{
_document->setstate (std::ios_base::badbit);
}
_size = 0;
}
}
Expand Down Expand Up @@ -693,7 +697,10 @@ namespace join
flush ();
if (size >= sizeof (_buffer))
{
_out->sputn (data, size);
if (JOIN_UNLIKELY (_out->sputn (data, size) != static_cast<std::streamsize> (size)))
{
_document->setstate (std::ios_base::badbit);
}
return;
}
}
Expand Down Expand Up @@ -725,6 +732,9 @@ namespace join
/// underlying output stream.
std::streambuf* _out;

/// output stream receiving the error state.
std::ostream* _document;

/// output buffer.
char _buffer[1024] = {};

Expand Down
1 change: 1 addition & 0 deletions data/tests/cbor_reader_test.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -498,6 +498,7 @@ TEST (CborReader, fail)
ASSERT_EQ (error (hex ("5f5cff")), SaxErrc::InvalidValue);
ASSERT_EQ (error (hex ("7f7bffffffffffffffff")), SaxErrc::InvalidValue);
ASSERT_EQ (error (hex ("5f5b0000000100000000")), SaxErrc::InvalidValue);
ASSERT_EQ (error (hex ("5f41005affffffff")), SaxErrc::InvalidValue);

ASSERT_EQ (error (hex ("bf6161ff")), SaxErrc::InvalidKey);
ASSERT_EQ (error (hex ("0000")), SaxErrc::ExtraData);
Expand Down
Loading
Loading