Skip to content

hipfile/fastpath: Ensure (buffer_address + buffer_offset) is aligned #175

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
kurtmcmillan wants to merge 12 commits into
base: develop
Choose a base branch
from
+106 −19
Open

hipfile/fastpath: Ensure (buffer_address + buffer_offset) is aligned #175

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
12 commits
Select commit Hold shift + click to select a range
858112e
hipFile/test: Ensure default values are setup properly for score tests
kurtmcmillan Jan 27, 2026
c729861
hipfile/test: Ensure fastpath test pass when STATX_DIOALIGN is not de…
kurtmcmillan Jan 27, 2026
d936fbe
hipfile/test: Use valid param values for FastpathAlignedIoSizesParam
kurtmcmillan Jan 27, 2026
dcc49cc
hipfile/test: Use valid param values for FastpathUnalignedIoSizesParam
kurtmcmillan Jan 27, 2026
180e018
hipfile/test: Use valid param values for FastpathAlignedFileOffsetsParam
kurtmcmillan Jan 27, 2026
b12c3ab
hipfile/test: Use valid param values for FastpathUnalignedFileOffsets…
kurtmcmillan Jan 27, 2026
deee9a9
hipfile/test: Correct file_offset argument in FastpathAlignedBufferOf…
kurtmcmillan Jan 27, 2026
b2bdd38
hipfile/test: Use valid param values for FastpathAlignedBufferOffsets…
kurtmcmillan Jan 27, 2026
7d65b63
hipfile/test: Use valid param values for FastpathUnalignedBufferOffse…
kurtmcmillan Jan 27, 2026
3715413
hipfile/fastpath: Modify checks in Fastpath::score(...)
kurtmcmillan Jan 28, 2026
6bc8b1b
hipfile/fastpath: Ensure (buffer_address + buffer_offset) is aligned
kurtmcmillan Jan 27, 2026
af88d42
hipfile/test: Use correct argument in fastpath test
kurtmcmillan Jan 28, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
17 changes: 10 additions & 7 deletions src/amd_detail/backend/fastpath.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -134,19 +134,22 @@ Fastpath::score(shared_ptr<IFile> file, shared_ptr<IBuffer> buffer, size_t size,
accept_io &= 0 <= file_offset;
accept_io &= 0 <= buffer_offset;

uint64_t mem_align_mask{4096 - 1};
uint64_t offset_align_mask{4096 - 1};

#if defined(STATX_DIOALIGN)
const struct statx &stx{file->getStatx()};
accept_io &= !!(stx.stx_mask & STATX_DIOALIGN);
accept_io &= stx.stx_dio_offset_align && stx.stx_dio_mem_align;
accept_io &= !(size & (stx.stx_dio_offset_align - 1));
accept_io &= !(file_offset & (stx.stx_dio_offset_align - 1));
accept_io &= !(buffer_offset & (stx.stx_dio_mem_align - 1));
#else
accept_io &= !(size & 0xFFF);
accept_io &= !(file_offset & 0xFFF);
accept_io &= !(buffer_offset & 0xFFF);
mem_align_mask = stx.stx_dio_mem_align - 1;
offset_align_mask = stx.stx_dio_offset_align - 1;
#endif

accept_io &= !(size & offset_align_mask);
accept_io &= !(file_offset & static_cast<int64_t>(offset_align_mask));
auto buffer_address{reinterpret_cast<intptr_t>(buffer->getBuffer())};
accept_io &= !((buffer_address + buffer_offset) & static_cast<int64_t>(mem_align_mask));

return accept_io ? 100 : -1;
}

Expand Down
108 changes: 96 additions & 12 deletions test/amd_detail/fastpath.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -54,8 +54,8 @@ operator==(const hipAmdFileHandle_t &lhs, const hipAmdFileHandle_t &rhs)
// Provide default values for variables used in fastpath tests
struct FastpathTestBase {
const size_t DEFAULT_IO_SIZE{1024 * 1024};
void *const DEFAULT_BUFFER_ADDR{reinterpret_cast<void *>(0x600DB10C)};
const off_t DEFAULT_BUFFER_OFFSET{512};
void *const DEFAULT_BUFFER_ADDR{reinterpret_cast<void *>(0xABAD'CAFE'0000'0000)};
const off_t DEFAULT_BUFFER_OFFSET{DEFAULT_MEM_ALIGN};
const size_t DEFAULT_BUFFER_LENGTH{DEFAULT_IO_SIZE + static_cast<size_t>(DEFAULT_BUFFER_OFFSET)};
const hipMemoryType DEFAULT_BUFFER_TYPE{hipMemoryTypeDevice};
const optional<int> DEFAULT_UNBUFFERED_FD{7};
Expand All @@ -79,11 +79,26 @@ struct FastpathTestBase {

struct FastpathTest : public FastpathTestBase, public Test {};

TEST_F(FastpathTest, TestDefaults)
{
ASSERT_FALSE((DEFAULT_MEM_ALIGN & (DEFAULT_MEM_ALIGN - 1)));
ASSERT_TRUE(DEFAULT_MEM_ALIGN > 1);
ASSERT_FALSE((DEFAULT_OFFSET_ALIGN & (DEFAULT_OFFSET_ALIGN - 1)));
ASSERT_TRUE(DEFAULT_OFFSET_ALIGN > 1);
ASSERT_FALSE((reinterpret_cast<uintptr_t>(DEFAULT_BUFFER_ADDR) & (DEFAULT_MEM_ALIGN - 1)));
ASSERT_FALSE((DEFAULT_BUFFER_OFFSET & (DEFAULT_MEM_ALIGN - 1)));
ASSERT_FALSE((DEFAULT_IO_SIZE & (DEFAULT_OFFSET_ALIGN - 1)));
ASSERT_FALSE((DEFAULT_FILE_OFFSET & (DEFAULT_OFFSET_ALIGN - 1)));
}

TEST_F(FastpathTest, UnbufferedFdAvailable)
{
EXPECT_CALL(*mfile, getUnbufferedFd).WillOnce(Return(DEFAULT_UNBUFFERED_FD));
EXPECT_CALL(*mbuffer, getType).WillOnce(Return(DEFAULT_BUFFER_TYPE));
#if defined(STATX_DIOALIGN)
EXPECT_CALL(*mfile, getStatx).WillOnce(ReturnRef(DEFAULT_STATX));
#endif
EXPECT_CALL(*mbuffer, getBuffer).WillOnce(Return(DEFAULT_BUFFER_ADDR));

ASSERT_EQ(Fastpath().score(mfile, mbuffer, DEFAULT_IO_SIZE, DEFAULT_FILE_OFFSET, DEFAULT_BUFFER_OFFSET),
SCORE_ACCEPT);
Expand All @@ -93,7 +108,10 @@ TEST_F(FastpathTest, UnbufferedFdNotAvailable)
{
EXPECT_CALL(*mfile, getUnbufferedFd).WillOnce(Return(nullopt));
EXPECT_CALL(*mbuffer, getType).WillOnce(Return(DEFAULT_BUFFER_TYPE));
#if defined(STATX_DIOALIGN)
EXPECT_CALL(*mfile, getStatx).WillOnce(ReturnRef(DEFAULT_STATX));
#endif
EXPECT_CALL(*mbuffer, getBuffer).WillOnce(Return(DEFAULT_BUFFER_ADDR));

ASSERT_EQ(Fastpath().score(mfile, mbuffer, DEFAULT_IO_SIZE, DEFAULT_FILE_OFFSET, DEFAULT_BUFFER_OFFSET),
SCORE_REJECT);
Expand All @@ -103,7 +121,10 @@ TEST_F(FastpathTest, ScoreRejectsNegativeAlignedFileOffset)
{
EXPECT_CALL(*mfile, getUnbufferedFd).WillOnce(Return(DEFAULT_UNBUFFERED_FD));
EXPECT_CALL(*mbuffer, getType).WillOnce(Return(DEFAULT_BUFFER_TYPE));
#if defined(STATX_DIOALIGN)
EXPECT_CALL(*mfile, getStatx).WillOnce(ReturnRef(DEFAULT_STATX));
#endif
EXPECT_CALL(*mbuffer, getBuffer).WillOnce(Return(DEFAULT_BUFFER_ADDR));

ASSERT_EQ(Fastpath().score(mfile, mbuffer, DEFAULT_IO_SIZE, -static_cast<hoff_t>(DEFAULT_OFFSET_ALIGN),
DEFAULT_BUFFER_OFFSET),
Expand All @@ -114,9 +135,30 @@ TEST_F(FastpathTest, ScoreRejectsNegativeAlignedBufferOffset)
{
EXPECT_CALL(*mfile, getUnbufferedFd).WillOnce(Return(DEFAULT_UNBUFFERED_FD));
EXPECT_CALL(*mbuffer, getType).WillOnce(Return(DEFAULT_BUFFER_TYPE));
#if defined(STATX_DIOALIGN)
EXPECT_CALL(*mfile, getStatx).WillOnce(ReturnRef(DEFAULT_STATX));
#endif
EXPECT_CALL(*mbuffer, getBuffer).WillOnce(Return(DEFAULT_BUFFER_ADDR));

ASSERT_EQ(Fastpath().score(mfile, mbuffer, DEFAULT_IO_SIZE, DEFAULT_BUFFER_OFFSET,
ASSERT_EQ(Fastpath().score(mfile, mbuffer, DEFAULT_IO_SIZE, DEFAULT_FILE_OFFSET,
-static_cast<hoff_t>(DEFAULT_MEM_ALIGN)),
SCORE_REJECT);
}

TEST_F(FastpathTest, ScoreRejectsBufferAddressPlusBufferOffsetIsUnaligned)
{
EXPECT_CALL(*mfile, getUnbufferedFd).WillOnce(Return(DEFAULT_UNBUFFERED_FD));
EXPECT_CALL(*mbuffer, getType).WillOnce(Return(DEFAULT_BUFFER_TYPE));
#if defined(STATX_DIOALIGN)
EXPECT_CALL(*mfile, getStatx).WillOnce(ReturnRef(DEFAULT_STATX));
#endif
// The DEFAULT_BUFFER_ADDR is DEFAULT_MEM_ALIGN aligned. Ensure that this
// test's buffer is not DEFAULT_MEM_ALIGN aligned.
EXPECT_CALL(*mbuffer, getBuffer)
.WillOnce(Return(reinterpret_cast<void *>(reinterpret_cast<uintptr_t>(DEFAULT_BUFFER_ADDR) +
(DEFAULT_MEM_ALIGN >> 1))));

ASSERT_EQ(Fastpath().score(mfile, mbuffer, DEFAULT_IO_SIZE, DEFAULT_FILE_OFFSET,
-static_cast<hoff_t>(DEFAULT_MEM_ALIGN)),
SCORE_REJECT);
}
Expand All @@ -127,7 +169,10 @@ TEST_P(FastpathSupportedHipMemoryParam, Score)
{
EXPECT_CALL(*mfile, getUnbufferedFd).WillOnce(Return(DEFAULT_UNBUFFERED_FD));
EXPECT_CALL(*mbuffer, getType).WillOnce(Return(GetParam()));
#if defined(STATX_DIOALIGN)
EXPECT_CALL(*mfile, getStatx).WillOnce(ReturnRef(DEFAULT_STATX));
#endif
EXPECT_CALL(*mbuffer, getBuffer).WillOnce(Return(DEFAULT_BUFFER_ADDR));

ASSERT_EQ(Fastpath().score(mfile, mbuffer, DEFAULT_IO_SIZE, DEFAULT_FILE_OFFSET, DEFAULT_BUFFER_OFFSET),
SCORE_ACCEPT);
Expand All @@ -141,7 +186,10 @@ TEST_P(FastpathUnsupportedHipMemoryParam, Score)
{
EXPECT_CALL(*mfile, getUnbufferedFd).WillOnce(Return(DEFAULT_UNBUFFERED_FD));
EXPECT_CALL(*mbuffer, getType).WillOnce(Return(GetParam()));
#if defined(STATX_DIOALIGN)
EXPECT_CALL(*mfile, getStatx).WillOnce(ReturnRef(DEFAULT_STATX));
#endif
EXPECT_CALL(*mbuffer, getBuffer).WillOnce(Return(DEFAULT_BUFFER_ADDR));

ASSERT_EQ(Fastpath().score(mfile, mbuffer, DEFAULT_IO_SIZE, DEFAULT_FILE_OFFSET, DEFAULT_BUFFER_OFFSET),
SCORE_REJECT);
Expand All @@ -156,74 +204,102 @@ TEST_P(FastpathAlignedIoSizesParam, Score)
{
EXPECT_CALL(*mfile, getUnbufferedFd).WillOnce(Return(DEFAULT_UNBUFFERED_FD));
EXPECT_CALL(*mbuffer, getType).WillOnce(Return(DEFAULT_BUFFER_TYPE));
#if defined(STATX_DIOALIGN)
EXPECT_CALL(*mfile, getStatx).WillOnce(ReturnRef(DEFAULT_STATX));
#endif
EXPECT_CALL(*mbuffer, getBuffer).WillOnce(Return(DEFAULT_BUFFER_ADDR));

ASSERT_EQ(Fastpath().score(mfile, mbuffer, GetParam(), DEFAULT_FILE_OFFSET, DEFAULT_BUFFER_OFFSET),
SCORE_ACCEPT);
}

INSTANTIATE_TEST_SUITE_P(FastpathTest, FastpathAlignedIoSizesParam,
Values(-DEFAULT_OFFSET_ALIGN, 0, DEFAULT_OFFSET_ALIGN));
Values(0, DEFAULT_OFFSET_ALIGN, DEFAULT_OFFSET_ALIGN << 1,
DEFAULT_OFFSET_ALIGN << 2));

struct FastpathUnalignedIoSizesParam : public FastpathTestBase, public TestWithParam<size_t> {};

TEST_P(FastpathUnalignedIoSizesParam, Score)
{
EXPECT_CALL(*mfile, getUnbufferedFd).WillOnce(Return(DEFAULT_UNBUFFERED_FD));
EXPECT_CALL(*mbuffer, getType).WillOnce(Return(DEFAULT_BUFFER_TYPE));
#if defined(STATX_DIOALIGN)
EXPECT_CALL(*mfile, getStatx).WillOnce(ReturnRef(DEFAULT_STATX));
#endif
EXPECT_CALL(*mbuffer, getBuffer).WillOnce(Return(DEFAULT_BUFFER_ADDR));

ASSERT_EQ(Fastpath().score(mfile, mbuffer, GetParam(), DEFAULT_FILE_OFFSET, DEFAULT_BUFFER_OFFSET),
SCORE_REJECT);
}

INSTANTIATE_TEST_SUITE_P(FastpathTest, FastpathUnalignedIoSizesParam,
Values(-DEFAULT_OFFSET_ALIGN + 1, 1, DEFAULT_OFFSET_ALIGN - 1));
Values(1, DEFAULT_OFFSET_ALIGN >> 1, DEFAULT_OFFSET_ALIGN - 1,
DEFAULT_OFFSET_ALIGN + 1));

struct FastpathAlignedFileOffsetsParam : public FastpathTestBase, public TestWithParam<hoff_t> {};

TEST_P(FastpathAlignedFileOffsetsParam, Score)
{
EXPECT_CALL(*mfile, getUnbufferedFd).WillOnce(Return(DEFAULT_UNBUFFERED_FD));
EXPECT_CALL(*mbuffer, getType).WillOnce(Return(DEFAULT_BUFFER_TYPE));
#if defined(STATX_DIOALIGN)
EXPECT_CALL(*mfile, getStatx).WillOnce(ReturnRef(DEFAULT_STATX));
#endif
EXPECT_CALL(*mbuffer, getBuffer).WillOnce(Return(DEFAULT_BUFFER_ADDR));

ASSERT_EQ(Fastpath().score(mfile, mbuffer, DEFAULT_IO_SIZE, GetParam(), DEFAULT_BUFFER_OFFSET),
SCORE_ACCEPT);
GetParam() >= 0 ? SCORE_ACCEPT : SCORE_REJECT);
}

INSTANTIATE_TEST_SUITE_P(FastpathTest, FastpathAlignedFileOffsetsParam,
Values(-DEFAULT_OFFSET_ALIGN, 0, DEFAULT_OFFSET_ALIGN));
Values(-static_cast<hoff_t>(DEFAULT_OFFSET_ALIGN << 2),
-static_cast<hoff_t>(DEFAULT_OFFSET_ALIGN << 1),
-static_cast<hoff_t>(DEFAULT_OFFSET_ALIGN), 0, DEFAULT_OFFSET_ALIGN,
DEFAULT_OFFSET_ALIGN << 1, DEFAULT_OFFSET_ALIGN << 2));

struct FastpathUnalignedFileOffsetsParam : public FastpathTestBase, public TestWithParam<hoff_t> {};

TEST_P(FastpathUnalignedFileOffsetsParam, Score)
{
EXPECT_CALL(*mfile, getUnbufferedFd).WillOnce(Return(DEFAULT_UNBUFFERED_FD));
EXPECT_CALL(*mbuffer, getType).WillOnce(Return(DEFAULT_BUFFER_TYPE));
#if defined(STATX_DIOALIGN)
EXPECT_CALL(*mfile, getStatx).WillOnce(ReturnRef(DEFAULT_STATX));
#endif
EXPECT_CALL(*mbuffer, getBuffer).WillOnce(Return(DEFAULT_BUFFER_ADDR));

ASSERT_EQ(Fastpath().score(mfile, mbuffer, DEFAULT_IO_SIZE, GetParam(), DEFAULT_BUFFER_OFFSET),
SCORE_REJECT);
}

INSTANTIATE_TEST_SUITE_P(FastpathTest, FastpathUnalignedFileOffsetsParam,
Values(-DEFAULT_OFFSET_ALIGN + 1, 1, DEFAULT_OFFSET_ALIGN - 1));
Values(-static_cast<hoff_t>(DEFAULT_OFFSET_ALIGN << 2) - 1,
-static_cast<hoff_t>(DEFAULT_OFFSET_ALIGN << 1) + 1,
-static_cast<hoff_t>(DEFAULT_OFFSET_ALIGN) - 1,
-static_cast<hoff_t>(DEFAULT_OFFSET_ALIGN >> 1), 1, DEFAULT_OFFSET_ALIGN >> 1,
DEFAULT_OFFSET_ALIGN - 1, (DEFAULT_OFFSET_ALIGN << 1) + 1,
(DEFAULT_OFFSET_ALIGN << 2) - 1));

struct FastpathAlignedBufferOffsetsParam : public FastpathTestBase, public TestWithParam<hoff_t> {};

TEST_P(FastpathAlignedBufferOffsetsParam, Score)
{
EXPECT_CALL(*mfile, getUnbufferedFd).WillOnce(Return(DEFAULT_UNBUFFERED_FD));
EXPECT_CALL(*mbuffer, getType).WillOnce(Return(DEFAULT_BUFFER_TYPE));
#if defined(STATX_DIOALIGN)
EXPECT_CALL(*mfile, getStatx).WillOnce(ReturnRef(DEFAULT_STATX));
#endif
EXPECT_CALL(*mbuffer, getBuffer).WillOnce(Return(DEFAULT_BUFFER_ADDR));

ASSERT_EQ(Fastpath().score(mfile, mbuffer, DEFAULT_IO_SIZE, DEFAULT_BUFFER_OFFSET, GetParam()),
SCORE_ACCEPT);
ASSERT_EQ(Fastpath().score(mfile, mbuffer, DEFAULT_IO_SIZE, DEFAULT_FILE_OFFSET, GetParam()),
GetParam() >= 0 ? SCORE_ACCEPT : SCORE_REJECT);
}

INSTANTIATE_TEST_SUITE_P(FastpathTest, FastpathAlignedBufferOffsetsParam,
Values(-DEFAULT_MEM_ALIGN, 0, DEFAULT_MEM_ALIGN));
Values(-static_cast<hoff_t>(DEFAULT_MEM_ALIGN << 2),
-static_cast<hoff_t>(DEFAULT_MEM_ALIGN << 1),
-static_cast<hoff_t>(DEFAULT_MEM_ALIGN), 0, DEFAULT_MEM_ALIGN,
DEFAULT_MEM_ALIGN << 1, DEFAULT_MEM_ALIGN << 2));

/// @brief Tests negative and unaligned buffer offsets
struct FastpathUnalignedBufferOffsetsParam : public FastpathTestBase, public TestWithParam<hoff_t> {};
Expand All @@ -232,14 +308,22 @@ TEST_P(FastpathUnalignedBufferOffsetsParam, Score)
{
EXPECT_CALL(*mfile, getUnbufferedFd).WillOnce(Return(DEFAULT_UNBUFFERED_FD));
EXPECT_CALL(*mbuffer, getType).WillOnce(Return(DEFAULT_BUFFER_TYPE));
#if defined(STATX_DIOALIGN)
EXPECT_CALL(*mfile, getStatx).WillOnce(ReturnRef(DEFAULT_STATX));
#endif
EXPECT_CALL(*mbuffer, getBuffer).WillOnce(Return(DEFAULT_BUFFER_ADDR));

ASSERT_EQ(Fastpath().score(mfile, mbuffer, DEFAULT_IO_SIZE, DEFAULT_FILE_OFFSET, GetParam()),
SCORE_REJECT);
}

INSTANTIATE_TEST_SUITE_P(FastpathTest, FastpathUnalignedBufferOffsetsParam,
Values(-DEFAULT_MEM_ALIGN + 1, 1, DEFAULT_MEM_ALIGN - 1));
Values(-static_cast<hoff_t>(DEFAULT_MEM_ALIGN << 2) - 1,
-static_cast<hoff_t>(DEFAULT_MEM_ALIGN << 1) + 1,
-static_cast<hoff_t>(DEFAULT_MEM_ALIGN) - 1,
-static_cast<hoff_t>(DEFAULT_MEM_ALIGN >> 1), 1, DEFAULT_MEM_ALIGN >> 1,
DEFAULT_MEM_ALIGN - 1, (DEFAULT_MEM_ALIGN << 1) + 1,
(DEFAULT_MEM_ALIGN << 2) - 1));

struct FastpathIoParam : public FastpathTestBase, public TestWithParam<IoType> {

Expand Down
Loading