Skip to content

[AIE2P] Implement sparse fifo_ld + sparse extract/concat/insert #970

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
matteius wants to merge 6 commits into
base: aie-public
Choose a base branch
from
Open

[AIE2P] Implement sparse fifo_ld + sparse extract/concat/insert #970

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
d7c1bf5
[AIE2P][Headers] Implement sparse extract/concat/insert via field access
Apr 26, 2026
37ba002
[AIE2P] Implement v128int8_sparse fifo_ld_pop L3-L5 backend chain
Apr 26, 2026
ac3270e
Merge branch 'aie-public' into feature/aie2p-sparse-fifo-ld-intrinsics
matteius Apr 28, 2026
59f8347
[AIE2P] Add sparse FIFO load tests
Apr 28, 2026
0dd64b0
Merge branch 'feature/aie2p-sparse-fifo-ld-intrinsics' of https://git…
Apr 28, 2026
b5fe4c0
Merge branch 'aie-public' into feature/aie2p-sparse-fifo-ld-intrinsics
matteius Apr 28, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions clang/include/clang/Basic/BuiltinsAIE2P.def
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -342,6 +342,7 @@ BUILTIN(__builtin_aie2p_fifo_ld_fill, "vv*&V32i&i&", "t")
BUILTIN(__builtin_aie2p_fifo_ld_pop_512_unaligned, "V64cv*&V32i&i&", "t")
BUILTIN(__builtin_aie2p_fifo_ld_pop_544_bfp16, "vv*&V32i&i&V64c&V8c&", "t")
BUILTIN(__builtin_aie2p_fifo_ld_pop_576_bfp16, "vv*&V32i&i&V64c&V8c&", "t")
BUILTIN(__builtin_aie2p_fifo_ld_pop_640_unaligned_sparse, "vv*&V32i&i&V64c&V16c&", "t")
BUILTIN(__builtin_aie2p_fifo_ld_pop_1d_512_unaligned, "V64cv*&V32i&i&i", "t")
BUILTIN(__builtin_aie2p_fifo_ld_pop_1d_544_bfp16, "vv*&V32i&i&iV64c&V8c&", "t")
BUILTIN(__builtin_aie2p_fifo_ld_pop_1d_576_bfp16, "vv*&V32i&i&iV64c&V8c&", "t")
Expand Down
7 changes: 6 additions & 1 deletion clang/lib/CodeGen/CGBuiltin.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23893,6 +23893,8 @@ static llvm::Intrinsic::ID getAIE2PIntrinsicFunction(unsigned BuiltinID) {
return Intrinsic::aie2p_fifo_ld_pop_576_bfp16;
case AIE::BI__builtin_aie2p_fifo_ld_pop_544_bfp16:
return Intrinsic::aie2p_fifo_ld_pop_544_bfp16;
case AIE::BI__builtin_aie2p_fifo_ld_pop_640_unaligned_sparse:
return Intrinsic::aie2p_fifo_ld_pop_640_unaligned_sparse;
case AIE::BI__builtin_aie2p_fifo_ld_pop_1d_512_unaligned:
return Intrinsic::aie2p_fifo_ld_pop_1d_unaligned;
case AIE::BI__builtin_aie2p_fifo_ld_pop_1d_576_bfp16:
Expand Down Expand Up @@ -24680,6 +24682,7 @@ Value *CodeGenFunction::EmitAIEBuiltinExpr(unsigned BuiltinID,
case AIE::BI__builtin_aie2p_fifo_ld_pop_3d_544_bfp16:
case AIE::BI__builtin_aie2p_fifo_ld_pop_544_bfp16:
case AIE::BI__builtin_aie2p_fifo_ld_pop_576_bfp16:
case AIE::BI__builtin_aie2p_fifo_ld_pop_640_unaligned_sparse:
case AIE::BI__builtin_aie2ps_fifo_ld_pop_BFP640:
case AIE::BI__builtin_aie2ps_fifo_ld_pop_BFP768:
case AIE::BI__builtin_aie2ps_fifo_ld_pop_1d_BFP640:
Expand Down Expand Up @@ -24728,6 +24731,7 @@ Value *CodeGenFunction::EmitAIEBuiltinExpr(unsigned BuiltinID,
case AIE::BI__builtin_aie2p_fifo_ld_pop_3d_544_bfp16:
case AIE::BI__builtin_aie2p_fifo_ld_pop_544_bfp16:
case AIE::BI__builtin_aie2p_fifo_ld_pop_576_bfp16:
case AIE::BI__builtin_aie2p_fifo_ld_pop_640_unaligned_sparse:
MXStructCount = 2;
break;
case AIE::BI__builtin_aie2ps_fifo_ld_pop_BFP640:
Expand Down Expand Up @@ -25066,7 +25070,8 @@ Value *CodeGenFunction::EmitAIE2PBuiltinExpr(unsigned BuiltinID,
case AIE::BI__builtin_aie2p_fifo_ld_pop_2d_544_bfp16:
case AIE::BI__builtin_aie2p_fifo_ld_pop_3d_512_unaligned:
case AIE::BI__builtin_aie2p_fifo_ld_pop_3d_576_bfp16:
case AIE::BI__builtin_aie2p_fifo_ld_pop_3d_544_bfp16: {
case AIE::BI__builtin_aie2p_fifo_ld_pop_3d_544_bfp16:
case AIE::BI__builtin_aie2p_fifo_ld_pop_640_unaligned_sparse: {
return this->EmitAIEBuiltinExpr(BuiltinID, E, Arch);
}
default:
Expand Down
12 changes: 5 additions & 7 deletions clang/lib/Headers/aie2p/aie2p_aie_api_compat.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -50,13 +50,11 @@ struct v8cfloat {
};
struct v16cfloat {};

struct v128uint16_sparse {};
struct v256uint8_sparse {};
struct v512uint4_sparse {};

struct v128int16_sparse {};
struct v256int8_sparse {};
struct v512int4_sparse {};
// AIE2P-larger (1280-bit) sparse vector types are now defined in
// aiebase_typedefs.h (Followup H — G-T3.6-003 Tier 2) so that they are
// visible to aie2p_upd_ext.h, which is included before this compat header.
// The previous empty-stub definitions here (`struct v256int8_sparse {};`)
// shadowed those real types and were removed.

struct v256bfp16ebs16_sparse {};
struct v256bfp16ebs8_sparse {};
Expand Down
71 changes: 70 additions & 1 deletion clang/lib/Headers/aie2p/aie2p_ldst.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -488,6 +488,71 @@ FIFO_ST(__aie_dm_resource_cd, restrict)
return fifo_ld_popx(p, s, 31, 31); \
}

// Sparse FIFO load for the 640-bit-class types (data=v64* + sparsity_t mask).
// Wires through the new __builtin_aie2p_fifo_ld_pop_640_unaligned_sparse
// builtin, which lowers to silicon ops VLDA_POP_640_normal_pop /
// VLDB_POP_640_normal_pop (selected via VLD_POP_640_normal_pop_pseudo).
//
// T must be a sparse vector struct (v128int8_sparse, v128uint8_sparse,
// v64int16_sparse, v64uint16_sparse, v256int4_sparse, v256uint4_sparse)
// with the standard {DATA data; sparsity_t mask;} layout from
// aiebase_typedefs.h. T_DATA is the 512-bit dense data field type.
#define FIFO_LD_SPARSE(T, T_DATA, DM_BANK, RESTRICT) \
INTRINSIC(void) \
fifo_ld_reset(T##_unaligned DM_BANK *RESTRICT &p, fifo_state_t &s) { \
s.pos = 0; \
sparse_fifo_t &fifo = s.fifo; \
__builtin_aie2p_fifo_ld_fill((void DM_BANK *RESTRICT &)p, fifo, s.pos); \
} \
\
INTRINSIC(void) \
fifo_ld_fill(T##_unaligned DM_BANK *RESTRICT &p, fifo_state_t &s) { \
int &pos = s.pos; \
sparse_fifo_t &fifo = s.fifo; \
__builtin_aie2p_fifo_ld_fill((void DM_BANK *RESTRICT &)p, fifo, pos); \
} \
\
INTRINSIC(T) \
fifo_ld_pop(T##_unaligned DM_BANK *RESTRICT &p, fifo_state_t &s) { \
int &pos = s.pos; \
sparse_fifo_t &fifo = s.fifo; \
T r; \
__builtin_aie2p_fifo_ld_pop_640_unaligned_sparse( \
(void DM_BANK *RESTRICT &)p, fifo, pos, (v64char &)r.data, \
(v16char &)r.mask); \
return r; \
}

// Wide-pair sparse FIFO load: pop a smaller sparse vector twice and stitch
// into the wider sparse vector via Followup H's set_##T1 + insert helpers.
// T1 is the wide sparse type (e.g. v256int8_sparse, must have lo/hi fields
// of T2 type per aiebase_typedefs.h composite-struct definitions).
// T2 is the narrow sparse type (e.g. v128int8_sparse).
#define FIFO_LD_SPARSE_WIDE(T1, T2, DM_BANK, RESTRICT) \
INTRINSIC(void) \
fifo_ld_reset(T1##_unaligned DM_BANK *RESTRICT &p, fifo_state_t &s) { \
T2##_unaligned DM_BANK *RESTRICT &q = \
(T2##_unaligned DM_BANK * RESTRICT &)p; \
fifo_ld_reset(q, s); \
} \
\
INTRINSIC(void) \
fifo_ld_fill(T1##_unaligned DM_BANK *RESTRICT &p, fifo_state_t &s) { \
T2##_unaligned DM_BANK *RESTRICT &q = \
(T2##_unaligned DM_BANK * RESTRICT &)p; \
fifo_ld_fill(q, s); \
} \
\
INTRINSIC(T1) \
fifo_ld_pop(T1##_unaligned DM_BANK *RESTRICT &p, fifo_state_t &s) { \
T2##_unaligned DM_BANK *RESTRICT &q = \
(T2##_unaligned DM_BANK * RESTRICT &)p; \
T1 v; \
v = set_##T1(0, fifo_ld_pop(q, s)); \
v = insert(v, 1, fifo_ld_pop(q, s)); \
return v; \
}

#define FIFO_LD_BFP16_WIDE(T1, T2, DM_BANK, RESTRICT) \
INTRINSIC(void) \
fifo_ld_reset(T1##_unaligned DM_BANK *RESTRICT &p, fifo_state_t &s) { \
Expand Down Expand Up @@ -571,7 +636,9 @@ FIFO_ST(__aie_dm_resource_cd, restrict)
FIFO_FILLX(v64bfp16ebs8_unaligned, DM_BANK, RESTRICT) \
FIFO_FILLX(v64bfp16ebs16_unaligned, DM_BANK, RESTRICT) \
FIFO_LD_BFP16_WIDE(v128bfp16ebs8, v64bfp16ebs8, DM_BANK, RESTRICT) \
FIFO_LD_BFP16_WIDE(v128bfp16ebs16, v64bfp16ebs16, DM_BANK, RESTRICT)
FIFO_LD_BFP16_WIDE(v128bfp16ebs16, v64bfp16ebs16, DM_BANK, RESTRICT) \
FIFO_LD_SPARSE(v128int8_sparse, v64int8, DM_BANK, RESTRICT) \
FIFO_LD_SPARSE_WIDE(v256int8_sparse, v128int8_sparse, DM_BANK, RESTRICT)

FIFO_LD(, )
FIFO_LD(__aie_dm_resource_a, )
Expand Down Expand Up @@ -602,6 +669,8 @@ FIFO_LD(__aie_dm_resource_cd, restrict)
#undef FIFO_POPX
#undef FIFO_FILLX
#undef FIFO_LD_BFP16_WIDE
#undef FIFO_LD_SPARSE
#undef FIFO_LD_SPARSE_WIDE
#undef FIFO_LD

#endif // AIE2P_LDST_H
Loading