Skip to content

aws: compression: validate valid columnar formats #12304

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
edsiper merged 2 commits into from
Sep 3, 2026
+54 −0
Merged

aws: compression: validate valid columnar formats #12304

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
c19bdb4
aws: compression: Validate supported columnar formats
cosmo0920 Aug 18, 2026
bc709fa
tests: internal: Add a test case for invalid iformat for columnar
cosmo0920 Aug 18, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
36 changes: 36 additions & 0 deletions src/aws/compression/arrow/compress.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,34 @@ static GArrowCompressionType compression_type_to_garrow(int compression_type)
}
}

static int validate_columnar_compression(int columnar_format,
int compression_type)
{
if (columnar_format == FLB_AWS_COMPRESS_FORMAT_PARQUET) {
switch (compression_type) {
case FLB_AWS_COMPRESS_NONE:
case FLB_AWS_COMPRESS_SNAPPY:
case FLB_AWS_COMPRESS_GZIP:
case FLB_AWS_COMPRESS_ZSTD:
return 0;
default:
return -1;
}
}

if (columnar_format == FLB_AWS_COMPRESS_FORMAT_ARROW) {
switch (compression_type) {
case FLB_AWS_COMPRESS_NONE:
case FLB_AWS_COMPRESS_ZSTD:
return 0;
default:
return -1;
}
}

return -1;
}

static int choose_block_size(size_t size)
{
int block_size = 8 * 1024 * 1024;
Expand Down Expand Up @@ -251,6 +279,14 @@ int flb_aws_compression_compress_columnar(int columnar_format,
gsize len;
uint8_t *buf;

if (validate_columnar_compression(columnar_format,
compression_type) != 0) {
flb_error("[aws][compress] unsupported compression type %d "
"for columnar format %d",
compression_type, columnar_format);
return -1;
}

table = parse_json((uint8_t *) json, size);
if (table == NULL) {
flb_error("[aws][compress] Failed to parse JSON into Arrow Table");
Expand Down
18 changes: 18 additions & 0 deletions tests/internal/aws_compress.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -419,6 +419,22 @@ void test_parquet_format_uncompressed()
flb_free(out_buf);
}

void test_parquet_format_invalid_compression()
{
int ret;
void *out_buf = NULL;
size_t out_size = 0;
char *json = "{\"key\":\"value\",\"num\":42}\n";
size_t json_len = strlen(json);

ret = flb_aws_compression_compress_columnar(FLB_AWS_COMPRESS_FORMAT_PARQUET,
json, json_len,
&out_buf, &out_size, -1);
TEST_CHECK(ret == -1);
TEST_CHECK(out_buf == NULL);
TEST_CHECK(out_size == 0);
}

void test_parquet_compression_reduces_size()
{
int ret;
Expand Down Expand Up @@ -545,6 +561,8 @@ TEST_LIST = {
{ "test_parquet_format_zstd", test_parquet_format_zstd },
{ "test_parquet_format_gzip", test_parquet_format_gzip },
{ "test_parquet_format_uncompressed", test_parquet_format_uncompressed },
{ "test_parquet_format_invalid_compression",
test_parquet_format_invalid_compression },
{ "test_parquet_compression_reduces_size",
test_parquet_compression_reduces_size },
#endif
Expand Down
Loading