diff --git a/cpp/tests/io/comp/comp_test.cpp b/cpp/tests/io/comp/comp_test.cpp index d80ad305023f..882f0bf9fa9e 100644 --- a/cpp/tests/io/comp/comp_test.cpp +++ b/cpp/tests/io/comp/comp_test.cpp @@ -22,6 +22,8 @@ #include +#include +#include #include using cudf::device_span; @@ -420,16 +422,23 @@ void roundtrip_test(cudf::io::compression_type compression) // Keep adding to the test data expected.insert(expected.end(), num_string.begin(), num_string.end()); } - if (cudf::io::detail::compress_max_allowed_chunk_size(compression) - .value_or(std::numeric_limits::max()) < expected.size()) { - // Skip if the data is too large for the compressor - return; - } + } + + // Exercise representative small, medium, and large inputs. The largest input preserves the + // previous test's maximum coverage without repeating the same round trip at every size in + // between. + auto const test_sizes = std::array{size_t{1 << 10}, size_t{1 << 20}, expected.size()}; + auto const max_input_size = cudf::io::detail::compress_max_allowed_chunk_size(compression) + .value_or(std::numeric_limits::max()); + for (auto const test_size : test_sizes) { + if (test_size > max_input_size) { continue; } + + auto const test_input = cudf::host_span{expected.data(), test_size}; auto d_comp = rmm::device_uvector( - cudf::io::detail::max_compressed_size(compression, expected.size()), stream, mr); + cudf::io::detail::max_compressed_size(compression, test_input.size()), stream, mr); { - auto const d_orig = cudf::detail::make_device_uvector_async(expected, stream, mr); + auto const d_orig = cudf::detail::make_device_uvector_async(test_input, stream, mr); auto hd_srcs = cudf::detail::hostdevice_vector>(1, stream); hd_srcs[0] = d_orig; hd_srcs.host_to_device_async(stream); @@ -448,7 +457,7 @@ void roundtrip_test(cudf::io::compression_type compression) d_comp.resize(hd_stats[0].bytes_written, stream); } - auto d_got = rmm::device_uvector(expected.size(), stream); + auto d_got = rmm::device_uvector(test_input.size(), stream); { auto hd_srcs = cudf::detail::hostdevice_vector>(1, stream); hd_srcs[0] = d_comp; @@ -463,14 +472,14 @@ void roundtrip_test(cudf::io::compression_type compression) hd_stats.host_to_device_async(stream); cudf::io::detail::decompress( - compression, hd_srcs, hd_dsts, hd_stats, expected.size(), expected.size(), stream); + compression, hd_srcs, hd_dsts, hd_stats, test_input.size(), test_input.size(), stream); hd_stats.device_to_host(stream); ASSERT_EQ(hd_stats[0].status, codec_status::SUCCESS); } auto const got = cudf::detail::make_std_vector(d_got, stream); - EXPECT_EQ(expected, got); + EXPECT_TRUE(std::equal(test_input.begin(), test_input.end(), got.begin(), got.end())); } } diff --git a/cpp/tests/io/cudftable_test.cpp b/cpp/tests/io/cudftable_test.cpp index 18afff9d41a0..4eb87fd6e18f 100644 --- a/cpp/tests/io/cudftable_test.cpp +++ b/cpp/tests/io/cudftable_test.cpp @@ -554,7 +554,9 @@ TEST_F(CudftableTest, LongStringColumns) TEST_F(CudftableTest, ManyColumns) { - constexpr int num_cols = 12'345; + // Exercise large packed metadata without creating enough tiny device allocations to + // dominate the test when it runs concurrently with other C++ test binaries. + constexpr int num_cols = 1'234; std::vector columns; for (int i = 0; i < num_cols; ++i) { cudf::test::fixed_width_column_wrapper col({i % 10, (i + 1) % 10, (i + 2) % 10}); diff --git a/cpp/tests/io/orc_chunked_reader_test.cu b/cpp/tests/io/orc_chunked_reader_test.cu index ee8c6c634870..2bb79e2d7a1b 100644 --- a/cpp/tests/io/orc_chunked_reader_test.cu +++ b/cpp/tests/io/orc_chunked_reader_test.cu @@ -1128,7 +1128,8 @@ void input_limit_test_read(int test_location, cudf::table_view const& input, output_limit output_limit_bytes, input_limit input_limit_bytes, - int const* expected_chunk_counts) + int const* expected_chunk_counts, + bool require_multiple_chunks = false) { CUDF_EXPECTS(test_files.size() == input_limit_expected_file_count, "Unexpected count of test filenames."); @@ -1142,6 +1143,7 @@ void input_limit_test_read(int test_location, // EXPECT_EQ(expected_chunk_counts[idx], num_chunks); // TODO: equal CUDF_TEST_EXPECT_TABLES_EQUIVALENT(*result, input); + if (require_multiple_chunks) { EXPECT_GT(num_chunks, 1); } } } @@ -1242,7 +1244,7 @@ TEST_F(OrcChunkedReaderInputLimitTest, ListType) // this test runs over 3 hours when racecheck is used if (getenv("LIBCUDF_RACECHECK_ENABLED")) { GTEST_SKIP(); } - int constexpr num_rows = 50'000'000; + int constexpr num_rows = 25'000'000; int constexpr list_size = 4; auto const stream = cudf::get_default_stream(); @@ -1274,13 +1276,18 @@ TEST_F(OrcChunkedReaderInputLimitTest, ListType) // Although we set `stripe_size_rows` to be very large, the writer only write // 250k rows (top level) per stripe due to having nested type. - // Thus, we have 200 stripes in total. + // Thus, we have 100 stripes in total. input_limit_test_write(test_files, input, cudf::io::default_stripe_size_rows); { int constexpr expected[] = {3, 40, 3}; - input_limit_test_read( - __LINE__, test_files, input, output_limit{0UL}, input_limit{5 * 1024 * 1024UL}, expected); + input_limit_test_read(__LINE__, + test_files, + input, + output_limit{0UL}, + input_limit{5 * 1024 * 1024UL}, + expected, + true); } { @@ -1299,7 +1306,7 @@ TEST_F(OrcChunkedReaderInputLimitTest, MixedColumnsHavingList) // this test runs over 3 hours when racecheck is used if (getenv("LIBCUDF_RACECHECK_ENABLED")) { GTEST_SKIP(); } - int constexpr num_rows = 50'000'000; + int constexpr num_rows = 25'000'000; int constexpr list_size = 4; int constexpr str_size = 3; @@ -1360,13 +1367,18 @@ TEST_F(OrcChunkedReaderInputLimitTest, MixedColumnsHavingList) // Although we set `stripe_size_rows` to be very large, the writer only write // 250k rows (top level) per stripe due to having nested type. - // Thus, we have 200 stripes in total. + // Thus, we have 100 stripes in total. input_limit_test_write(test_files, input, cudf::io::default_stripe_size_rows); { int constexpr expected[] = {13, 8, 6}; - input_limit_test_read( - __LINE__, test_files, input, output_limit{0UL}, input_limit{128 * 1024 * 1024UL}, expected); + input_limit_test_read(__LINE__, + test_files, + input, + output_limit{0UL}, + input_limit{128 * 1024 * 1024UL}, + expected, + true); } { @@ -1461,7 +1473,7 @@ TEST_F(OrcChunkedReaderInputLimitTest, SizeTypeRowsOverflow) int64_t constexpr num_rows = 500'000'000l; int constexpr rows_per_stripe = 1'000'000; - int constexpr num_reps = 10; + int constexpr num_reps = 5; int64_t constexpr total_rows = num_rows * num_reps; static_assert(total_rows > std::numeric_limits::max()); diff --git a/cpp/tests/io/text/data_chunk_source_test.cpp b/cpp/tests/io/text/data_chunk_source_test.cpp index 561be9b4d1bf..ddc796563bae 100644 --- a/cpp/tests/io/text/data_chunk_source_test.cpp +++ b/cpp/tests/io/text/data_chunk_source_test.cpp @@ -157,6 +157,19 @@ enum class compression { ENABLED, DISABLED }; enum class eof { ADD_EOF_BLOCK, NO_EOF_BLOCK }; +// 40 MiB exercises two full 16 MiB BGZIP reader loads plus a partial third load. +constexpr int bgzip_input_doublings = 22; + +std::string make_bgzip_test_input() +{ + std::string input{"bananarama"}; + input.reserve(input.size() << bgzip_input_doublings); + for (int i = 0; i < bgzip_input_doublings; ++i) { + input += input; + } + return input; +} + uint64_t virtual_offset(std::size_t block_offset, std::size_t local_offset) { return (block_offset << 16) | local_offset; @@ -225,11 +238,7 @@ using DataChunkDecompressionTest = DecompressionTest; TEST_P(DataChunkDecompressionTest, BgzipSource) { auto const filename = temp_env->get_temp_filepath("bgzip_source"); - std::string input{"bananarama"}; - input.reserve(input.size() << 25); - for (int i = 0; i < 24; i++) { - input = input + input; - } + auto const input = make_bgzip_test_input(); { std::ofstream output_stream{filename}; std::default_random_engine rng{}; @@ -244,11 +253,7 @@ TEST_P(DataChunkDecompressionTest, BgzipSource) TEST_F(DataChunkSourceTest, BgzipSourceVirtualOffsets) { auto const filename = temp_env->get_temp_filepath("bgzip_source_offsets"); - std::string input{"bananarama"}; - input.reserve(input.size() << 25); - for (int i = 0; i < 24; i++) { - input = input + input; - } + auto input = make_bgzip_test_input(); std::string const padding_garbage(10000, 'g'); std::string const data_garbage{"GARBAGE"}; std::string const begininput{"begin of bananarama"}; @@ -332,11 +337,7 @@ TEST_F(DataChunkSourceTest, BgzipSourceVirtualOffsetsSingleChunk) TEST_F(DataChunkSourceTest, BgzipCompressedSourceVirtualOffsets) { auto const filename = temp_env->get_temp_filepath("bgzip_source_compressed_offsets"); - std::string input{"bananarama"}; - input.reserve(input.size() << 25); - for (int i = 0; i < 24; i++) { - input = input + input; - } + auto input = make_bgzip_test_input(); std::string const padding_garbage(10000, 'g'); std::string const data_garbage{"GARBAGE"}; std::string const begininput{"begin of bananarama"};