Skip to content

Add in-place data construction for Encoder #1

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
Li-Xiang-Ideal wants to merge 2 commits into
base: main
Choose a base branch
from
Open

Add in-place data construction for Encoder #1

Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
2496d7b
Add in-place data construction for Encoder
Li-Xiang-Ideal Jan 30, 2026
e6d6a3d
Add typed and transformed array writes
Li-Xiang-Ideal Jul 5, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
163 changes: 159 additions & 4 deletions wxf_parser.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,9 @@
#include <span>
#include <string>
#include <string_view>
#include <type_traits>
#include <unordered_map>
#include <utility>

namespace WXF_PARSER {

Expand Down Expand Up @@ -179,6 +181,11 @@ namespace WXF_PARSER {
// move from existing buffer
Encoder(std::vector<uint8_t>&& buf) : buffer(std::move(buf)) {}

const size_t size() const { return buffer.size(); }
void reserve(const size_t new_size) { buffer.reserve(new_size); }
void resize(const size_t new_size) { buffer.resize(new_size); }
void resize(const size_t new_size, const uint8_t val) { buffer.resize(new_size, val); }

// push ustr directly
Encoder& push_ustr(const std::vector<uint8_t>& str) { buffer.insert(buffer.end(), str.begin(), str.end()); return *this; }
Encoder& push_ustr(const std::string_view str) {
Expand All @@ -193,6 +200,36 @@ namespace WXF_PARSER {
buffer.insert(buffer.end(), (uint8_t*)start, (uint8_t*)end); return *this;
}

// generate data as ustr in the back
template<typename T, typename F>
Encoder& generate_back_ustr(const size_t len, F&& func) {
static_assert(std::is_trivially_copyable_v<T>, "T should be trivially copyable");
static_assert(std::is_invocable_r_v<T, F&, size_t>, "func should be callable with size_t and return T");
size_t old_size = buffer.size();
buffer.resize(old_size + len * sizeof(T));
uint8_t* data_ptr = buffer.data() + old_size;
for (size_t i = 0; i < len; i++) {
T value = func(i);
std::memcpy(data_ptr + i * sizeof(T), &value, sizeof(T));
}
return *this;
}

// generate transformed data as ustr in the back
template<typename T1, typename T2, typename F>
Encoder& transform_back_ustr(const T2* src_ptr, const size_t len, F&& func) {
static_assert(std::is_trivially_copyable_v<T1>, "T1 should be trivially copyable");
static_assert(std::is_invocable_r_v<T1, F&, const T2&>, "func should be callable with T2 and return T1");
size_t old_size = buffer.size();
buffer.resize(old_size + len * sizeof(T1));
uint8_t* data_ptr = buffer.data() + old_size;
for (size_t i = 0; i < len; i++) {
T1 value = func(src_ptr[i]);
std::memcpy(data_ptr + i * sizeof(T1), &value, sizeof(T1));
}
return *this;
}

Encoder& push_integer(const int64_t val) {
auto num_type = minimal_signed_bits(val);
switch (num_type) {
Expand Down Expand Up @@ -265,8 +302,59 @@ namespace WXF_PARSER {
return all_len;
}

template<typename T>
Encoder& push_array(const std::vector<size_t>& dimension_array, const std::span<T> data, WXF_HEAD type, uint8_t num_type) {
template<typename T, typename F = std::identity>
requires std::is_invocable_v<F&, const T&>
Encoder& push_array_data(const std::span<T> data, uint8_t num_type, F&& func = std::identity{}) {
// backup current size
size_t old_size = buffer.size();
using value_t = std::remove_cvref_t<std::invoke_result_t<F&, const T&>>;

if constexpr (std::is_integral_v<value_t>) {
const bool consistent_sign = (num_type >> 2) == (std::is_unsigned_v<value_t> ? (16 >> 2) : 0);
if (size_of_arr_num_type(num_type) == sizeof(value_t) && consistent_sign) {
if constexpr (std::is_same_v<std::remove_cvref_t<F>, std::identity>) {
return push_ustr(data.data(), data.size());
}
else {
return transform_back_ustr<value_t>(data.data(), data.size(), func);
}
}

#define APPEND_CASTED_ARRAY_DATA(TYPE) do { \
transform_back_ustr<TYPE>(data.data(), data.size(), \
[&](const T& value) { return static_cast<TYPE>(func(value)); }); \
} while (0)

switch (num_type) {
case 0: APPEND_CASTED_ARRAY_DATA(int8_t); break;
case 1: APPEND_CASTED_ARRAY_DATA(int16_t); break;
case 2: APPEND_CASTED_ARRAY_DATA(int32_t); break;
case 3: APPEND_CASTED_ARRAY_DATA(int64_t); break;
case 16: APPEND_CASTED_ARRAY_DATA(uint8_t); break;
case 17: APPEND_CASTED_ARRAY_DATA(uint16_t); break;
case 18: APPEND_CASTED_ARRAY_DATA(uint32_t); break;
case 19: APPEND_CASTED_ARRAY_DATA(uint64_t); break;
default: std::cerr << "Encoder::push_array_data: unsupported integer array num_type "
<< static_cast<int>(num_type) << "." << std::endl;
buffer.resize(old_size);
break;
}
return *this;
#undef APPEND_CASTED_ARRAY_DATA
}
else {
if constexpr (std::is_same_v<std::remove_cvref_t<F>, std::identity>) {
return push_ustr(data.data(), data.size());
}
else {
return transform_back_ustr<value_t>(data.data(), data.size(), func);
}
}
}

template<typename T, typename F = std::identity>
requires std::is_invocable_v<F&, const T&>
Encoder& push_array(const std::vector<size_t>& dimension_array, const std::span<T> data, WXF_HEAD type, uint8_t num_type, F&& func = std::identity{}) {
// backup current size
size_t old_size = buffer.size();

Expand All @@ -281,7 +369,74 @@ namespace WXF_PARSER {
}

// push data
return push_ustr(data.data(), data.size());
if constexpr (std::is_integral_v<T>) {
const bool supported_num_type = num_type <= 3 || (num_type >= 16 && num_type <= 19);
if (!supported_num_type) {
std::cerr << "Encoder::push_array: unsupported integer array num_type "
<< static_cast<int>(num_type) << "." << std::endl;
buffer.resize(old_size);
return *this;
}
}
return push_array_data(data, num_type, std::forward<F>(func));
}

template<typename F>
requires std::is_invocable_v<F&, size_t>
Encoder& push_generated_array_data(const size_t len, uint8_t num_type, F&& func) {
// backup current size
size_t old_size = buffer.size();
using value_t = std::remove_cvref_t<std::invoke_result_t<F&, size_t>>;

if constexpr (std::is_integral_v<value_t>) {
#define GENERATE_ARRAY_DATA(TYPE) do { \
generate_back_ustr<TYPE>(len, [&](size_t i) { \
return static_cast<TYPE>(func(i)); \
});} while (0)

switch (num_type) {
case 0: GENERATE_ARRAY_DATA(int8_t); break;
case 1: GENERATE_ARRAY_DATA(int16_t); break;
case 2: GENERATE_ARRAY_DATA(int32_t); break;
case 3: GENERATE_ARRAY_DATA(int64_t); break;
case 16: GENERATE_ARRAY_DATA(uint8_t); break;
case 17: GENERATE_ARRAY_DATA(uint16_t); break;
case 18: GENERATE_ARRAY_DATA(uint32_t); break;
case 19: GENERATE_ARRAY_DATA(uint64_t); break;
default: std::cerr << "Encoder::push_generated_array_data: unsupported integer array num_type "
<< static_cast<int>(num_type) << "." << std::endl;
buffer.resize(old_size);
break;
}
return *this;
#undef GENERATE_ARRAY_DATA
}
else {
return generate_back_ustr<value_t>(len, func);
}
}

template<typename F>
requires std::is_invocable_v<F&, size_t>
Encoder& push_generated_array(const std::vector<size_t>& dimension_array, WXF_HEAD type, uint8_t num_type, F&& func) {
// backup current size
size_t old_size = buffer.size();
using value_t = std::remove_cvref_t<std::invoke_result_t<F&, size_t>>;

// [array_type, num_type, rank, dimensions..., data...]
auto all_len = push_array_info(dimension_array, type, num_type);

// push data
if constexpr (std::is_integral_v<value_t>) {
const bool supported_num_type = num_type <= 3 || (num_type >= 16 && num_type <= 19);
if (!supported_num_type) {
std::cerr << "Encoder::push_generated_array: unsupported integer array num_type "
<< static_cast<int>(num_type) << "." << std::endl;
buffer.resize(old_size);
return *this;
}
}
return push_generated_array_data(all_len, num_type, std::forward<F>(func));
}

template<typename T>
Expand Down Expand Up @@ -1314,4 +1469,4 @@ namespace WXF_PARSER {
fullform_to_wxf(encoder, FullForm::parse_FullForm(ff_template), map);
return encoder;
}
} // namespace WXF_PARSER
} // namespace WXF_PARSER