diff --git a/src/amd_detail/backend/fastpath.cpp b/src/amd_detail/backend/fastpath.cpp index 1238b778..adcd1f4b 100644 --- a/src/amd_detail/backend/fastpath.cpp +++ b/src/amd_detail/backend/fastpath.cpp @@ -134,19 +134,22 @@ Fastpath::score(shared_ptr file, shared_ptr buffer, size_t size, accept_io &= 0 <= file_offset; accept_io &= 0 <= buffer_offset; + uint64_t mem_align_mask{4096 - 1}; + uint64_t offset_align_mask{4096 - 1}; + #if defined(STATX_DIOALIGN) const struct statx &stx{file->getStatx()}; accept_io &= !!(stx.stx_mask & STATX_DIOALIGN); accept_io &= stx.stx_dio_offset_align && stx.stx_dio_mem_align; - accept_io &= !(size & (stx.stx_dio_offset_align - 1)); - accept_io &= !(file_offset & (stx.stx_dio_offset_align - 1)); - accept_io &= !(buffer_offset & (stx.stx_dio_mem_align - 1)); -#else - accept_io &= !(size & 0xFFF); - accept_io &= !(file_offset & 0xFFF); - accept_io &= !(buffer_offset & 0xFFF); + mem_align_mask = stx.stx_dio_mem_align - 1; + offset_align_mask = stx.stx_dio_offset_align - 1; #endif + accept_io &= !(size & offset_align_mask); + accept_io &= !(file_offset & static_cast(offset_align_mask)); + auto buffer_address{reinterpret_cast(buffer->getBuffer())}; + accept_io &= !((buffer_address + buffer_offset) & static_cast(mem_align_mask)); + return accept_io ? 100 : -1; } diff --git a/test/amd_detail/fastpath.cpp b/test/amd_detail/fastpath.cpp index 3e4564b0..e4c69079 100644 --- a/test/amd_detail/fastpath.cpp +++ b/test/amd_detail/fastpath.cpp @@ -54,8 +54,8 @@ operator==(const hipAmdFileHandle_t &lhs, const hipAmdFileHandle_t &rhs) // Provide default values for variables used in fastpath tests struct FastpathTestBase { const size_t DEFAULT_IO_SIZE{1024 * 1024}; - void *const DEFAULT_BUFFER_ADDR{reinterpret_cast(0x600DB10C)}; - const off_t DEFAULT_BUFFER_OFFSET{512}; + void *const DEFAULT_BUFFER_ADDR{reinterpret_cast(0xABAD'CAFE'0000'0000)}; + const off_t DEFAULT_BUFFER_OFFSET{DEFAULT_MEM_ALIGN}; const size_t DEFAULT_BUFFER_LENGTH{DEFAULT_IO_SIZE + static_cast(DEFAULT_BUFFER_OFFSET)}; const hipMemoryType DEFAULT_BUFFER_TYPE{hipMemoryTypeDevice}; const optional DEFAULT_UNBUFFERED_FD{7}; @@ -79,11 +79,26 @@ struct FastpathTestBase { struct FastpathTest : public FastpathTestBase, public Test {}; +TEST_F(FastpathTest, TestDefaults) +{ + ASSERT_FALSE((DEFAULT_MEM_ALIGN & (DEFAULT_MEM_ALIGN - 1))); + ASSERT_TRUE(DEFAULT_MEM_ALIGN > 1); + ASSERT_FALSE((DEFAULT_OFFSET_ALIGN & (DEFAULT_OFFSET_ALIGN - 1))); + ASSERT_TRUE(DEFAULT_OFFSET_ALIGN > 1); + ASSERT_FALSE((reinterpret_cast(DEFAULT_BUFFER_ADDR) & (DEFAULT_MEM_ALIGN - 1))); + ASSERT_FALSE((DEFAULT_BUFFER_OFFSET & (DEFAULT_MEM_ALIGN - 1))); + ASSERT_FALSE((DEFAULT_IO_SIZE & (DEFAULT_OFFSET_ALIGN - 1))); + ASSERT_FALSE((DEFAULT_FILE_OFFSET & (DEFAULT_OFFSET_ALIGN - 1))); +} + TEST_F(FastpathTest, UnbufferedFdAvailable) { EXPECT_CALL(*mfile, getUnbufferedFd).WillOnce(Return(DEFAULT_UNBUFFERED_FD)); EXPECT_CALL(*mbuffer, getType).WillOnce(Return(DEFAULT_BUFFER_TYPE)); +#if defined(STATX_DIOALIGN) EXPECT_CALL(*mfile, getStatx).WillOnce(ReturnRef(DEFAULT_STATX)); +#endif + EXPECT_CALL(*mbuffer, getBuffer).WillOnce(Return(DEFAULT_BUFFER_ADDR)); ASSERT_EQ(Fastpath().score(mfile, mbuffer, DEFAULT_IO_SIZE, DEFAULT_FILE_OFFSET, DEFAULT_BUFFER_OFFSET), SCORE_ACCEPT); @@ -93,7 +108,10 @@ TEST_F(FastpathTest, UnbufferedFdNotAvailable) { EXPECT_CALL(*mfile, getUnbufferedFd).WillOnce(Return(nullopt)); EXPECT_CALL(*mbuffer, getType).WillOnce(Return(DEFAULT_BUFFER_TYPE)); +#if defined(STATX_DIOALIGN) EXPECT_CALL(*mfile, getStatx).WillOnce(ReturnRef(DEFAULT_STATX)); +#endif + EXPECT_CALL(*mbuffer, getBuffer).WillOnce(Return(DEFAULT_BUFFER_ADDR)); ASSERT_EQ(Fastpath().score(mfile, mbuffer, DEFAULT_IO_SIZE, DEFAULT_FILE_OFFSET, DEFAULT_BUFFER_OFFSET), SCORE_REJECT); @@ -103,7 +121,10 @@ TEST_F(FastpathTest, ScoreRejectsNegativeAlignedFileOffset) { EXPECT_CALL(*mfile, getUnbufferedFd).WillOnce(Return(DEFAULT_UNBUFFERED_FD)); EXPECT_CALL(*mbuffer, getType).WillOnce(Return(DEFAULT_BUFFER_TYPE)); +#if defined(STATX_DIOALIGN) EXPECT_CALL(*mfile, getStatx).WillOnce(ReturnRef(DEFAULT_STATX)); +#endif + EXPECT_CALL(*mbuffer, getBuffer).WillOnce(Return(DEFAULT_BUFFER_ADDR)); ASSERT_EQ(Fastpath().score(mfile, mbuffer, DEFAULT_IO_SIZE, -static_cast(DEFAULT_OFFSET_ALIGN), DEFAULT_BUFFER_OFFSET), @@ -114,9 +135,30 @@ TEST_F(FastpathTest, ScoreRejectsNegativeAlignedBufferOffset) { EXPECT_CALL(*mfile, getUnbufferedFd).WillOnce(Return(DEFAULT_UNBUFFERED_FD)); EXPECT_CALL(*mbuffer, getType).WillOnce(Return(DEFAULT_BUFFER_TYPE)); +#if defined(STATX_DIOALIGN) EXPECT_CALL(*mfile, getStatx).WillOnce(ReturnRef(DEFAULT_STATX)); +#endif + EXPECT_CALL(*mbuffer, getBuffer).WillOnce(Return(DEFAULT_BUFFER_ADDR)); - ASSERT_EQ(Fastpath().score(mfile, mbuffer, DEFAULT_IO_SIZE, DEFAULT_BUFFER_OFFSET, + ASSERT_EQ(Fastpath().score(mfile, mbuffer, DEFAULT_IO_SIZE, DEFAULT_FILE_OFFSET, + -static_cast(DEFAULT_MEM_ALIGN)), + SCORE_REJECT); +} + +TEST_F(FastpathTest, ScoreRejectsBufferAddressPlusBufferOffsetIsUnaligned) +{ + EXPECT_CALL(*mfile, getUnbufferedFd).WillOnce(Return(DEFAULT_UNBUFFERED_FD)); + EXPECT_CALL(*mbuffer, getType).WillOnce(Return(DEFAULT_BUFFER_TYPE)); +#if defined(STATX_DIOALIGN) + EXPECT_CALL(*mfile, getStatx).WillOnce(ReturnRef(DEFAULT_STATX)); +#endif + // The DEFAULT_BUFFER_ADDR is DEFAULT_MEM_ALIGN aligned. Ensure that this + // test's buffer is not DEFAULT_MEM_ALIGN aligned. + EXPECT_CALL(*mbuffer, getBuffer) + .WillOnce(Return(reinterpret_cast(reinterpret_cast(DEFAULT_BUFFER_ADDR) + + (DEFAULT_MEM_ALIGN >> 1)))); + + ASSERT_EQ(Fastpath().score(mfile, mbuffer, DEFAULT_IO_SIZE, DEFAULT_FILE_OFFSET, -static_cast(DEFAULT_MEM_ALIGN)), SCORE_REJECT); } @@ -127,7 +169,10 @@ TEST_P(FastpathSupportedHipMemoryParam, Score) { EXPECT_CALL(*mfile, getUnbufferedFd).WillOnce(Return(DEFAULT_UNBUFFERED_FD)); EXPECT_CALL(*mbuffer, getType).WillOnce(Return(GetParam())); +#if defined(STATX_DIOALIGN) EXPECT_CALL(*mfile, getStatx).WillOnce(ReturnRef(DEFAULT_STATX)); +#endif + EXPECT_CALL(*mbuffer, getBuffer).WillOnce(Return(DEFAULT_BUFFER_ADDR)); ASSERT_EQ(Fastpath().score(mfile, mbuffer, DEFAULT_IO_SIZE, DEFAULT_FILE_OFFSET, DEFAULT_BUFFER_OFFSET), SCORE_ACCEPT); @@ -141,7 +186,10 @@ TEST_P(FastpathUnsupportedHipMemoryParam, Score) { EXPECT_CALL(*mfile, getUnbufferedFd).WillOnce(Return(DEFAULT_UNBUFFERED_FD)); EXPECT_CALL(*mbuffer, getType).WillOnce(Return(GetParam())); +#if defined(STATX_DIOALIGN) EXPECT_CALL(*mfile, getStatx).WillOnce(ReturnRef(DEFAULT_STATX)); +#endif + EXPECT_CALL(*mbuffer, getBuffer).WillOnce(Return(DEFAULT_BUFFER_ADDR)); ASSERT_EQ(Fastpath().score(mfile, mbuffer, DEFAULT_IO_SIZE, DEFAULT_FILE_OFFSET, DEFAULT_BUFFER_OFFSET), SCORE_REJECT); @@ -156,14 +204,18 @@ TEST_P(FastpathAlignedIoSizesParam, Score) { EXPECT_CALL(*mfile, getUnbufferedFd).WillOnce(Return(DEFAULT_UNBUFFERED_FD)); EXPECT_CALL(*mbuffer, getType).WillOnce(Return(DEFAULT_BUFFER_TYPE)); +#if defined(STATX_DIOALIGN) EXPECT_CALL(*mfile, getStatx).WillOnce(ReturnRef(DEFAULT_STATX)); +#endif + EXPECT_CALL(*mbuffer, getBuffer).WillOnce(Return(DEFAULT_BUFFER_ADDR)); ASSERT_EQ(Fastpath().score(mfile, mbuffer, GetParam(), DEFAULT_FILE_OFFSET, DEFAULT_BUFFER_OFFSET), SCORE_ACCEPT); } INSTANTIATE_TEST_SUITE_P(FastpathTest, FastpathAlignedIoSizesParam, - Values(-DEFAULT_OFFSET_ALIGN, 0, DEFAULT_OFFSET_ALIGN)); + Values(0, DEFAULT_OFFSET_ALIGN, DEFAULT_OFFSET_ALIGN << 1, + DEFAULT_OFFSET_ALIGN << 2)); struct FastpathUnalignedIoSizesParam : public FastpathTestBase, public TestWithParam {}; @@ -171,14 +223,18 @@ TEST_P(FastpathUnalignedIoSizesParam, Score) { EXPECT_CALL(*mfile, getUnbufferedFd).WillOnce(Return(DEFAULT_UNBUFFERED_FD)); EXPECT_CALL(*mbuffer, getType).WillOnce(Return(DEFAULT_BUFFER_TYPE)); +#if defined(STATX_DIOALIGN) EXPECT_CALL(*mfile, getStatx).WillOnce(ReturnRef(DEFAULT_STATX)); +#endif + EXPECT_CALL(*mbuffer, getBuffer).WillOnce(Return(DEFAULT_BUFFER_ADDR)); ASSERT_EQ(Fastpath().score(mfile, mbuffer, GetParam(), DEFAULT_FILE_OFFSET, DEFAULT_BUFFER_OFFSET), SCORE_REJECT); } INSTANTIATE_TEST_SUITE_P(FastpathTest, FastpathUnalignedIoSizesParam, - Values(-DEFAULT_OFFSET_ALIGN + 1, 1, DEFAULT_OFFSET_ALIGN - 1)); + Values(1, DEFAULT_OFFSET_ALIGN >> 1, DEFAULT_OFFSET_ALIGN - 1, + DEFAULT_OFFSET_ALIGN + 1)); struct FastpathAlignedFileOffsetsParam : public FastpathTestBase, public TestWithParam {}; @@ -186,14 +242,20 @@ TEST_P(FastpathAlignedFileOffsetsParam, Score) { EXPECT_CALL(*mfile, getUnbufferedFd).WillOnce(Return(DEFAULT_UNBUFFERED_FD)); EXPECT_CALL(*mbuffer, getType).WillOnce(Return(DEFAULT_BUFFER_TYPE)); +#if defined(STATX_DIOALIGN) EXPECT_CALL(*mfile, getStatx).WillOnce(ReturnRef(DEFAULT_STATX)); +#endif + EXPECT_CALL(*mbuffer, getBuffer).WillOnce(Return(DEFAULT_BUFFER_ADDR)); ASSERT_EQ(Fastpath().score(mfile, mbuffer, DEFAULT_IO_SIZE, GetParam(), DEFAULT_BUFFER_OFFSET), - SCORE_ACCEPT); + GetParam() >= 0 ? SCORE_ACCEPT : SCORE_REJECT); } INSTANTIATE_TEST_SUITE_P(FastpathTest, FastpathAlignedFileOffsetsParam, - Values(-DEFAULT_OFFSET_ALIGN, 0, DEFAULT_OFFSET_ALIGN)); + Values(-static_cast(DEFAULT_OFFSET_ALIGN << 2), + -static_cast(DEFAULT_OFFSET_ALIGN << 1), + -static_cast(DEFAULT_OFFSET_ALIGN), 0, DEFAULT_OFFSET_ALIGN, + DEFAULT_OFFSET_ALIGN << 1, DEFAULT_OFFSET_ALIGN << 2)); struct FastpathUnalignedFileOffsetsParam : public FastpathTestBase, public TestWithParam {}; @@ -201,14 +263,22 @@ TEST_P(FastpathUnalignedFileOffsetsParam, Score) { EXPECT_CALL(*mfile, getUnbufferedFd).WillOnce(Return(DEFAULT_UNBUFFERED_FD)); EXPECT_CALL(*mbuffer, getType).WillOnce(Return(DEFAULT_BUFFER_TYPE)); +#if defined(STATX_DIOALIGN) EXPECT_CALL(*mfile, getStatx).WillOnce(ReturnRef(DEFAULT_STATX)); +#endif + EXPECT_CALL(*mbuffer, getBuffer).WillOnce(Return(DEFAULT_BUFFER_ADDR)); ASSERT_EQ(Fastpath().score(mfile, mbuffer, DEFAULT_IO_SIZE, GetParam(), DEFAULT_BUFFER_OFFSET), SCORE_REJECT); } INSTANTIATE_TEST_SUITE_P(FastpathTest, FastpathUnalignedFileOffsetsParam, - Values(-DEFAULT_OFFSET_ALIGN + 1, 1, DEFAULT_OFFSET_ALIGN - 1)); + Values(-static_cast(DEFAULT_OFFSET_ALIGN << 2) - 1, + -static_cast(DEFAULT_OFFSET_ALIGN << 1) + 1, + -static_cast(DEFAULT_OFFSET_ALIGN) - 1, + -static_cast(DEFAULT_OFFSET_ALIGN >> 1), 1, DEFAULT_OFFSET_ALIGN >> 1, + DEFAULT_OFFSET_ALIGN - 1, (DEFAULT_OFFSET_ALIGN << 1) + 1, + (DEFAULT_OFFSET_ALIGN << 2) - 1)); struct FastpathAlignedBufferOffsetsParam : public FastpathTestBase, public TestWithParam {}; @@ -216,14 +286,20 @@ TEST_P(FastpathAlignedBufferOffsetsParam, Score) { EXPECT_CALL(*mfile, getUnbufferedFd).WillOnce(Return(DEFAULT_UNBUFFERED_FD)); EXPECT_CALL(*mbuffer, getType).WillOnce(Return(DEFAULT_BUFFER_TYPE)); +#if defined(STATX_DIOALIGN) EXPECT_CALL(*mfile, getStatx).WillOnce(ReturnRef(DEFAULT_STATX)); +#endif + EXPECT_CALL(*mbuffer, getBuffer).WillOnce(Return(DEFAULT_BUFFER_ADDR)); - ASSERT_EQ(Fastpath().score(mfile, mbuffer, DEFAULT_IO_SIZE, DEFAULT_BUFFER_OFFSET, GetParam()), - SCORE_ACCEPT); + ASSERT_EQ(Fastpath().score(mfile, mbuffer, DEFAULT_IO_SIZE, DEFAULT_FILE_OFFSET, GetParam()), + GetParam() >= 0 ? SCORE_ACCEPT : SCORE_REJECT); } INSTANTIATE_TEST_SUITE_P(FastpathTest, FastpathAlignedBufferOffsetsParam, - Values(-DEFAULT_MEM_ALIGN, 0, DEFAULT_MEM_ALIGN)); + Values(-static_cast(DEFAULT_MEM_ALIGN << 2), + -static_cast(DEFAULT_MEM_ALIGN << 1), + -static_cast(DEFAULT_MEM_ALIGN), 0, DEFAULT_MEM_ALIGN, + DEFAULT_MEM_ALIGN << 1, DEFAULT_MEM_ALIGN << 2)); /// @brief Tests negative and unaligned buffer offsets struct FastpathUnalignedBufferOffsetsParam : public FastpathTestBase, public TestWithParam {}; @@ -232,14 +308,22 @@ TEST_P(FastpathUnalignedBufferOffsetsParam, Score) { EXPECT_CALL(*mfile, getUnbufferedFd).WillOnce(Return(DEFAULT_UNBUFFERED_FD)); EXPECT_CALL(*mbuffer, getType).WillOnce(Return(DEFAULT_BUFFER_TYPE)); +#if defined(STATX_DIOALIGN) EXPECT_CALL(*mfile, getStatx).WillOnce(ReturnRef(DEFAULT_STATX)); +#endif + EXPECT_CALL(*mbuffer, getBuffer).WillOnce(Return(DEFAULT_BUFFER_ADDR)); ASSERT_EQ(Fastpath().score(mfile, mbuffer, DEFAULT_IO_SIZE, DEFAULT_FILE_OFFSET, GetParam()), SCORE_REJECT); } INSTANTIATE_TEST_SUITE_P(FastpathTest, FastpathUnalignedBufferOffsetsParam, - Values(-DEFAULT_MEM_ALIGN + 1, 1, DEFAULT_MEM_ALIGN - 1)); + Values(-static_cast(DEFAULT_MEM_ALIGN << 2) - 1, + -static_cast(DEFAULT_MEM_ALIGN << 1) + 1, + -static_cast(DEFAULT_MEM_ALIGN) - 1, + -static_cast(DEFAULT_MEM_ALIGN >> 1), 1, DEFAULT_MEM_ALIGN >> 1, + DEFAULT_MEM_ALIGN - 1, (DEFAULT_MEM_ALIGN << 1) + 1, + (DEFAULT_MEM_ALIGN << 2) - 1)); struct FastpathIoParam : public FastpathTestBase, public TestWithParam {