From 6766c43eb6d0183b808384147dc384372647eb3d Mon Sep 17 00:00:00 2001 From: Michael Kenzel Date: Fri, 24 Nov 2023 05:29:41 +0100 Subject: [PATCH 1/6] update dependencies --- build.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/build.py b/build.py index a91c6841..3e8282df 100755 --- a/build.py +++ b/build.py @@ -102,8 +102,8 @@ def __init__(self, dir, buildsystem): self.rv_source_dir = self.source_dir/"rv" def pull(self): - pull_git_dependency(self.source_dir, "https://github.com/llvm/llvm-project.git", branch="llvmorg-14.0.1") - pull_git_dependency(self.rv_source_dir, "https://github.com/cdl-saarland/rv.git", "--recurse-submodules", branch="release/14.x") + pull_git_dependency(self.source_dir, "https://github.com/llvm/llvm-project.git", branch="llvmorg-16.0.6") + pull_git_dependency(self.rv_source_dir, "https://github.com/cdl-saarland/rv.git", "--recurse-submodules", branch="release/16.x") def configure(self, configs, anydsl): for patch in [ @@ -129,6 +129,7 @@ def configure(self, configs, anydsl): LLVM_TOOL_RV_BUILD=True, CLANG_INCLUDE_DOCS=False, CLANG_INCLUDE_TESTS=False, + RV_ENABLE_PLUGIN=False, LLVM_RVPLUG_LINK_INTO_TOOLS=False, LLVM_BUILD_LLVM_DYLIB=False if windows else True, LLVM_LINK_LLVM_DYLIB=False if windows else True @@ -136,7 +137,7 @@ def configure(self, configs, anydsl): def build(self, config): self.buildsystem.build(self.build_dir, config, "clang", "llvm-as", "lld", "LLVMExecutionEngine", "LLVMRuntimeDyld") - self.buildsystem.build(self.build_dir, config, "RV", "LLVMRV", "LLVMMCJIT") + self.buildsystem.build(self.build_dir, config, "RV", "LLVMMCJIT") @component(depends_on=(LLVM,)) class Thorin(Build): @@ -146,7 +147,7 @@ def __init__(self, dir, buildsystem): self.half_source_dir = self.source_dir/"half" def pull(self): - pull_git_dependency(self.source_dir, "https://github.com/AnyDSL/thorin.git", branch="ipdps23") + pull_git_dependency(self.source_dir, "https://github.com/AnyDSL/thorin.git", branch="master") pull_svn_dependency(self.half_source_dir, "https://svn.code.sf.net/p/half/code/tags/release-2.2.0") def configure(self, configs, llvm): @@ -167,7 +168,7 @@ def __init__(self, dir, buildsystem): self.source_dir = dir/"dependencies"/"anydsl"/"artic" def pull(self): - pull_git_dependency(self.source_dir, "https://github.com/AnyDSL/artic.git", branch="ipdps23") + pull_git_dependency(self.source_dir, "https://github.com/AnyDSL/artic.git", branch="master") def configure(self, configs, thorin): self.buildsystem.configure(self.build_dir, configs, self.source_dir, @@ -184,7 +185,7 @@ def __init__(self, dir, buildsystem): self.source_dir = dir/"dependencies"/"anydsl"/"runtime" def pull(self): - pull_git_dependency(self.source_dir, "https://github.com/AnyDSL/runtime.git", branch="ipdps23") + pull_git_dependency(self.source_dir, "https://github.com/AnyDSL/runtime.git", branch="master") def configure(self, configs, anydsl, llvm, thorin, artic): if windows: From 31f764db9d511124cb1b94784711fbf542aeb594 Mon Sep 17 00:00:00 2001 From: Michael Kenzel Date: Fri, 24 Nov 2023 05:30:23 +0100 Subject: [PATCH 2/6] fix warning --- build.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.py b/build.py index 3e8282df..d7f85479 100755 --- a/build.py +++ b/build.py @@ -218,7 +218,7 @@ def __init__(self, dir, buildsystem): def configure(self, configs, boost, runtime): self.buildsystem.configure(self.build_dir, configs, self.source_dir, Boost_NO_SYSTEM_PATHS=True, - BOOST_ROOT=boost.build_dir, + Boost_ROOT=boost.build_dir, AnyDSL_runtime_DIR=runtime.build_dir/"share"/"anydsl"/"cmake", BUILD_TESTING=True, CMAKE_EXPORT_COMPILE_COMMANDS=True, From ffbca77546baf62e8816ec5098d7e75683e29a2d Mon Sep 17 00:00:00 2001 From: Michael Kenzel Date: Fri, 24 Nov 2023 05:59:05 +0100 Subject: [PATCH 3/6] improved memory barriers --- src/mapping_cuda.art | 31 ++++++++++++++++++++++------- src/mapping_nvvm.art | 46 +++++++++++++++++++++++++++++++------------- 2 files changed, 57 insertions(+), 20 deletions(-) diff --git a/src/mapping_cuda.art b/src/mapping_cuda.art index f454a851..9ecb2ee1 100644 --- a/src/mapping_cuda.art +++ b/src/mapping_cuda.art @@ -38,20 +38,37 @@ fn @cuda_pred(b: bool) -> i32 { if b { 1 } else { 0 } } +fn @cuda_memory_barrier(order: memory_order) { + if cuda_device_arch() >= 700 { + (@|| { + match order { + memory_order::acquire => asm("fence.acq_rel.gpu;" :::: "volatile"), + memory_order::release => asm("fence.acq_rel.gpu;" :::: "volatile"), + memory_order::acq_rel => asm("fence.acq_rel.gpu;" :::: "volatile"), + memory_order::seq_cst => asm("fence.sc.gpu;" :::: "volatile"), + _ => () + } + })() + } + else { + cuda_threadfence() + } +} + fn @cuda_legacy_atomic_memory_barrier_pre(order:memory_order) { match order { - memory_order::release => cuda_threadfence(), - memory_order::acq_rel => cuda_threadfence(), - memory_order::seq_cst => cuda_threadfence(), + memory_order::release => cuda_memory_barrier(memory_order::release), + memory_order::acq_rel => cuda_memory_barrier(memory_order::acq_rel), + memory_order::seq_cst => cuda_memory_barrier(memory_order::seq_cst), _ => () } } fn @cuda_legacy_atomic_memory_barrier_post(order:memory_order) { match order { - memory_order::acquire => cuda_threadfence(), - memory_order::acq_rel => cuda_threadfence(), - memory_order::seq_cst => cuda_threadfence(), + memory_order::acquire => cuda_memory_barrier(memory_order::acquire), + memory_order::acq_rel => cuda_memory_barrier(memory_order::acq_rel), + memory_order::seq_cst => cuda_memory_barrier(memory_order::seq_cst), _ => () } } @@ -189,7 +206,7 @@ fn @cuda_thread(idx: fn(i32) -> u32, gid: fn() -> u32, body: fn(thread_context) atomic_inc_global_u32 = cuda_atomic_inc_global_u32, - memory_barrier = @|_| cuda_threadfence(), + memory_barrier = cuda_memory_barrier, timestamp = @|| cuda_globaltimer() as i64, timestamp32 = @|| cuda_globaltimer_lo() as i32, diff --git a/src/mapping_nvvm.art b/src/mapping_nvvm.art index 9155d9ce..4df13ecc 100644 --- a/src/mapping_nvvm.art +++ b/src/mapping_nvvm.art @@ -39,35 +39,55 @@ fn @nvvm_pred(b: bool) -> i32 { if b { 1 } else { 0 } } -fn @nvvm_legacy_atomic_memory_order_wrap(order:memory_order, f: fn() -> ()) { +fn @nvvm_memory_barrier(order: memory_order) { + if nvvm_device_arch() >= 700 { + (@|| { + match order { + memory_order::acquire => asm("fence.acq_rel.gpu;" :::: "volatile"), + memory_order::release => asm("fence.acq_rel.gpu;" :::: "volatile"), + memory_order::acq_rel => asm("fence.acq_rel.gpu;" :::: "volatile"), + memory_order::seq_cst => asm("fence.sc.gpu;" :::: "volatile"), + _ => () + } + })() + } + else { + nvvm_threadfence() + } +} + +fn @nvvm_legacy_atomic_memory_barrier_pre(order:memory_order) { match order { - memory_order::release => nvvm_threadfence(), - memory_order::acq_rel => nvvm_threadfence(), - memory_order::seq_cst => nvvm_threadfence(), + memory_order::release => nvvm_memory_barrier(memory_order::release), + memory_order::acq_rel => nvvm_memory_barrier(memory_order::acq_rel), + memory_order::seq_cst => nvvm_memory_barrier(memory_order::seq_cst), _ => () } +} - f(); - +fn @nvvm_legacy_atomic_memory_barrier_post(order:memory_order) { match order { - memory_order::acquire => nvvm_threadfence(), - memory_order::acq_rel => nvvm_threadfence(), - memory_order::seq_cst => nvvm_threadfence(), + memory_order::acquire => nvvm_memory_barrier(memory_order::acquire), + memory_order::acq_rel => nvvm_memory_barrier(memory_order::acq_rel), + memory_order::seq_cst => nvvm_memory_barrier(memory_order::seq_cst), _ => () } } fn @nvvm_legacy_atomic_memory_order_wrap_load[T](f: fn(&addrspace(1) T) -> T) { @|location:&addrspace(1) T, order:memory_order| -> T { - let mut res:T; - nvvm_legacy_atomic_memory_order_wrap(order, @|| res = f(location)); + nvvm_legacy_atomic_memory_barrier_pre(order); + let res = f(location); + nvvm_legacy_atomic_memory_barrier_post(order); res } } fn @nvvm_legacy_atomic_memory_order_wrap_store[T](f: fn(&mut addrspace(1) T, T) -> ()) { @|location:&mut addrspace(1) T, value:T, order:memory_order| -> () { - nvvm_legacy_atomic_memory_order_wrap(order, @|| f(location, value)); + nvvm_legacy_atomic_memory_barrier_pre(order); + f(location, value); + nvvm_legacy_atomic_memory_barrier_post(order); } } @@ -169,7 +189,7 @@ fn @nvvm_thread(idx: fn(i32) -> u32, gid: fn() -> u32, body: fn(thread_context) atomic_inc_global_u32 = nvvm_atomic_inc_global_u32, - memory_barrier = @|_| nvvm_threadfence(), + memory_barrier = nvvm_memory_barrier, timestamp = @|| nvvm_globaltimer() as i64, timestamp32 = @|| nvvm_globaltimer_lo() as i32, From 87905f5ef732809dfe5109bd8928042596c1a72d Mon Sep 17 00:00:00 2001 From: Michael Kenzel Date: Fri, 24 Nov 2023 06:07:41 +0100 Subject: [PATCH 4/6] improved atomic loads and stores --- src/mapping_cuda.art | 108 +++++++++++++++++++++++++++++++++++++------ src/mapping_nvvm.art | 98 +++++++++++++++++++++++++++++++++++---- 2 files changed, 185 insertions(+), 21 deletions(-) diff --git a/src/mapping_cuda.art b/src/mapping_cuda.art index 9ecb2ee1..f965457a 100644 --- a/src/mapping_cuda.art +++ b/src/mapping_cuda.art @@ -116,16 +116,98 @@ fn @cuda_legacy_atomic_memory_order_wrap_cas[T](f: fn(&mut addrspace(1) T, T, T) } } -fn @cuda_atomic_wait_and_transition[T](f: fn(&mut addrspace(1) T, T, T) -> T, cmp: fn(T, T) -> bool) { - @|location:&mut addrspace(1) T, expected:T, desired:T, order:memory_order, _debug_msg:&[u8]| -> () { + +fn @atomic_load_global_i32(location: &addrspace(1) i32, order: memory_order) -> i32 = atomic_load_global_u32(location as &addrspace(1) u32, order) as i32; +fn @atomic_load_global_u32(location: &addrspace(1) u32, order: memory_order) -> u32 { + let mut value: u32; + + if cuda_device_arch() >= 700 { + (@|| { + match order { + memory_order::relaxed => asm("ld.relaxed.gpu.b32 %0, [%1];" : "=r"(value) : "l"(location) : "memory"), + memory_order::acquire => asm("ld.acquire.gpu.b32 %0, [%1];" : "=r"(value) : "l"(location) : "memory"), + _ => cuda_trap() + } + })(); + } + else { cuda_legacy_atomic_memory_barrier_pre(order); - while !cmp(f(location, expected, desired), expected) { - cuda_threadfence(); - } + asm("ld.volatile.global.b32 %0, [%1];" : "=r"(value) : "l"(location) : "memory"); + cuda_legacy_atomic_memory_barrier_post(order); + } + + value +} + +fn @atomic_load_global_i64(location: &addrspace(1) i64, order: memory_order) = atomic_load_global_u64(location as &addrspace(1) u64, order) as i64; +fn @atomic_load_global_u64(location: &addrspace(1) u64, order: memory_order) -> u64 { + let mut value: u64; + + if cuda_device_arch() >= 700 { + (@|| { + match order { + memory_order::relaxed => asm("ld.relaxed.gpu.b64 %0, [%1];" : "=l"(value) : "l"(location) : "memory"), + memory_order::acquire => asm("ld.acquire.gpu.b64 %0, [%1];" : "=l"(value) : "l"(location) : "memory"), + _ => cuda_trap() + } + })(); + } + else { + cuda_legacy_atomic_memory_barrier_pre(order); + asm("ld.volatile.global.b64 %0, [%1];" : "=l"(value) : "l"(location) : "memory"); + cuda_legacy_atomic_memory_barrier_post(order); + } + + value +} + +fn @atomic_store_global_i32(location: &addrspace(1) i32, value: i32, order: memory_order) = atomic_store_global_u32(location as &addrspace(1) u32, value as u32, order); +fn @atomic_store_global_u32(location: &addrspace(1) u32, value: u32, order: memory_order) -> () { + if cuda_device_arch() >= 700 { + (@|| { + match order { + memory_order::relaxed => asm("st.relaxed.gpu.b32 [%0], %1;" : : "l"(location), "r"(value) : "memory"), + memory_order::release => asm("st.release.gpu.b32 [%0], %1;" : : "l"(location), "r"(value) : "memory"), + _ => cuda_trap() + } + })() + } + else { + cuda_legacy_atomic_memory_barrier_pre(order); + asm("st.volatile.global.b32 [%0], %1;" : : "l"(location), "r"(value) : "memory"); cuda_legacy_atomic_memory_barrier_post(order); } } +fn @atomic_store_global_i64(location: &addrspace(1) i64, value: i64, order: memory_order) = atomic_store_global_u64(location as &addrspace(1) u64, value as u64, order); +fn @atomic_store_global_u64(location: &addrspace(1) u64, value: u64, order: memory_order) -> () { + if cuda_device_arch() >= 700 { + (@|| { + match order { + memory_order::relaxed => asm("st.relaxed.gpu.b64 [%0], %1;" : : "l"(location), "l"(value) : "memory"), + memory_order::release => asm("st.release.gpu.b64 [%0], %1;" : : "l"(location), "l"(value) : "memory"), + _ => cuda_trap() + } + })() + } + else { + cuda_legacy_atomic_memory_barrier_pre(order); + asm("st.volatile.global.b64 [%0], %1;" : : "l"(location), "l"(value) : "memory"); + cuda_legacy_atomic_memory_barrier_post(order); + } +} + + +// fn @cuda_atomic_wait_and_transition[T](f: fn(&mut addrspace(1) T, T, T) -> T, cmp: fn(T, T) -> bool) { +// @|location:&mut addrspace(1) T, expected:T, desired:T, order:memory_order, _debug_msg:&[u8]| -> () { +// cuda_legacy_atomic_memory_barrier_pre(order); +// while !cmp(f(location, expected, desired), expected) { +// cuda_threadfence(); +// } +// cuda_legacy_atomic_memory_barrier_post(order); +// } +// } + fn @cuda_thread(idx: fn(i32) -> u32, gid: fn() -> u32, body: fn(thread_context) -> ()) -> () { let sleep = @|t: u32| { if cuda_device_arch() >= 700 { @@ -142,20 +224,20 @@ fn @cuda_thread(idx: fn(i32) -> u32, gid: fn() -> u32, body: fn(thread_context) gid = gid, uid = @|| gid() as i32, - atomic_load_global_i32 = cuda_legacy_atomic_memory_order_wrap_load[i32](@|location| { let mut res:i32; asm("atom.global.add.s32 %0, [%1], 0;" : "=r"(res) : "l"(location) : "memory"); res }), - atomic_load_global_u32 = cuda_legacy_atomic_memory_order_wrap_load[u32](@|location| { let mut res:u32; asm("atom.global.add.u32 %0, [%1], 0;" : "=r"(res) : "l"(location) : "memory"); res }), - atomic_load_global_i64 = cuda_legacy_atomic_memory_order_wrap_load[i64](@|location| { let mut res:i64; asm("atom.global.add.u64 %0, [%1], 0;" : "=l"(res) : "l"(location) : "memory"); res }), - atomic_load_global_u64 = cuda_legacy_atomic_memory_order_wrap_load[u64](@|location| { let mut res:u64; asm("atom.global.add.u64 %0, [%1], 0;" : "=l"(res) : "l"(location) : "memory"); res }), + atomic_load_global_i32 = atomic_load_global_i32, + atomic_load_global_u32 = atomic_load_global_u32, + atomic_load_global_i64 = atomic_load_global_i64, + atomic_load_global_u64 = atomic_load_global_u64, atomic_load_global_i32_coalesced = cuda_legacy_atomic_memory_order_wrap_load[i32](@|location| { let mut res:i32; asm("ld.volatile.global.b32 %0, [%1];" : "=r"(res) : "l"(location) : "memory"); res }), atomic_load_global_u32_coalesced = cuda_legacy_atomic_memory_order_wrap_load[u32](@|location| { let mut res:u32; asm("ld.volatile.global.b32 %0, [%1];" : "=r"(res) : "l"(location) : "memory"); res }), atomic_load_global_i64_coalesced = cuda_legacy_atomic_memory_order_wrap_load[i64](@|location| { let mut res:i64; asm("ld.volatile.global.b64 %0, [%1];" : "=l"(res) : "l"(location) : "memory"); res }), atomic_load_global_u64_coalesced = cuda_legacy_atomic_memory_order_wrap_load[u64](@|location| { let mut res:u64; asm("ld.volatile.global.b64 %0, [%1];" : "=l"(res) : "l"(location) : "memory"); res }), - atomic_store_global_i32 = cuda_legacy_atomic_memory_order_wrap_store[i32](@|location, value| { let mut res:i32; asm("atom.global.exch.b32 %0, [%1], %2;" : "=r"(res) : "l"(location), "r"(value) : "memory"); }), - atomic_store_global_u32 = cuda_legacy_atomic_memory_order_wrap_store[u32](@|location, value| { let mut res:u32; asm("atom.global.exch.b32 %0, [%1], %2;" : "=r"(res) : "l"(location), "r"(value) : "memory"); }), - atomic_store_global_i64 = cuda_legacy_atomic_memory_order_wrap_store[i64](@|location, value| { let mut res:i64; asm("atom.global.exch.b64 %0, [%1], %2;" : "=l"(res) : "l"(location), "l"(value) : "memory"); }), - atomic_store_global_u64 = cuda_legacy_atomic_memory_order_wrap_store[u64](@|location, value| { let mut res:u64; asm("atom.global.exch.b64 %0, [%1], %2;" : "=l"(res) : "l"(location), "l"(value) : "memory"); }), + atomic_store_global_i32 = atomic_store_global_i32, + atomic_store_global_u32 = atomic_store_global_u32, + atomic_store_global_i64 = atomic_store_global_i64, + atomic_store_global_u64 = atomic_store_global_u64, atomic_store_global_i32_coalesced = cuda_legacy_atomic_memory_order_wrap_store[i32](@|location, value| asm("st.volatile.global.b32 [%0], %1;" : : "l"(location), "r"(value) : "memory")), atomic_store_global_u32_coalesced = cuda_legacy_atomic_memory_order_wrap_store[u32](@|location, value| asm("st.volatile.global.b32 [%0], %1;" : : "l"(location), "r"(value) : "memory")), diff --git a/src/mapping_nvvm.art b/src/mapping_nvvm.art index 4df13ecc..752b1be7 100644 --- a/src/mapping_nvvm.art +++ b/src/mapping_nvvm.art @@ -103,6 +103,88 @@ fn @nvvm_atomic_cas[T](location:&mut addrspace(1) T, expected: T, desired: T, me fn @nvvm_atomic_cas_weak[T](location:&mut addrspace(1) T, expected: T, desired: T, memory_order_succ: memory_order, memory_order_fail: memory_order) = cmpxchg_weak_p1[T](location, expected, desired, builtin_memory_order(memory_order_succ), builtin_memory_order(memory_order_fail), ""); + +fn @atomic_load_global_i32(location: &addrspace(1) i32, order: memory_order) -> i32 = atomic_load_global_u32(location as &addrspace(1) u32, order) as i32; +fn @atomic_load_global_u32(location: &addrspace(1) u32, order: memory_order) -> u32 { + let mut value: u32; + + if nvvm_device_arch() >= 700 { + (@|| { + match order { + memory_order::relaxed => asm("ld.relaxed.gpu.b32 $0, [$1];" : "=r"(value) : "l"(location) : "memory"), + memory_order::acquire => asm("ld.acquire.gpu.b32 $0, [$1];" : "=r"(value) : "l"(location) : "memory"), + _ => nvvm_trap() + } + })(); + } + else { + nvvm_legacy_atomic_memory_barrier_pre(order); + asm("ld.volatile.global.b32 $0, [$1];" : "=r"(value) : "l"(location) : "memory"); + nvvm_legacy_atomic_memory_barrier_post(order); + } + + value +} + +fn @atomic_load_global_i64(location: &addrspace(1) i64, order: memory_order) = atomic_load_global_u64(location as &addrspace(1) u64, order) as i64; +fn @atomic_load_global_u64(location: &addrspace(1) u64, order: memory_order) -> u64 { + let mut value: u64; + + if nvvm_device_arch() >= 700 { + (@|| { + match order { + memory_order::relaxed => asm("ld.relaxed.gpu.b64 $0, [$1];" : "=l"(value) : "l"(location) : "memory"), + memory_order::acquire => asm("ld.acquire.gpu.b64 $0, [$1];" : "=l"(value) : "l"(location) : "memory"), + _ => nvvm_trap() + } + })(); + } + else { + nvvm_legacy_atomic_memory_barrier_pre(order); + asm("ld.volatile.global.b64 $0, [$1];" : "=l"(value) : "l"(location) : "memory"); + nvvm_legacy_atomic_memory_barrier_post(order); + } + + value +} + +fn @atomic_store_global_i32(location: &addrspace(1) i32, value: i32, order: memory_order) = atomic_store_global_u32(location as &addrspace(1) u32, value as u32, order); +fn @atomic_store_global_u32(location: &addrspace(1) u32, value: u32, order: memory_order) -> () { + if nvvm_device_arch() >= 700 { + (@|| { + match order { + memory_order::relaxed => asm("st.relaxed.gpu.b32 [$0], $1;" : : "l"(location), "r"(value) : "memory"), + memory_order::release => asm("st.release.gpu.b32 [$0], $1;" : : "l"(location), "r"(value) : "memory"), + _ => nvvm_trap() + } + })() + } + else { + nvvm_legacy_atomic_memory_barrier_pre(order); + asm("st.volatile.global.b32 [$0], $1;" : : "l"(location), "r"(value) : "memory"); + nvvm_legacy_atomic_memory_barrier_post(order); + } +} + +fn @atomic_store_global_i64(location: &addrspace(1) i64, value: i64, order: memory_order) = atomic_store_global_u64(location as &addrspace(1) u64, value as u64, order); +fn @atomic_store_global_u64(location: &addrspace(1) u64, value: u64, order: memory_order) -> () { + if nvvm_device_arch() >= 700 { + (@|| { + match order { + memory_order::relaxed => asm("st.relaxed.gpu.b64 [$0], $1;" : : "l"(location), "l"(value) : "memory"), + memory_order::release => asm("st.release.gpu.b64 [$0], $1;" : : "l"(location), "l"(value) : "memory"), + _ => nvvm_trap() + } + })() + } + else { + nvvm_legacy_atomic_memory_barrier_pre(order); + asm("st.volatile.global.b64 [$0], $1;" : : "l"(location), "l"(value) : "memory"); + nvvm_legacy_atomic_memory_barrier_post(order); + } +} + + // fn @nvvm_atomic_wait_and_transition[T](location:&mut addrspace(1) T, expected:T, desired:T, order:memory_order, _debug_msg:&[u8]) -> () { // while !nvvm_atomic_cas_weak[T](location, expected, desired, order, memory_order::relaxed).1 { // nvvm_threadfence(); @@ -125,20 +207,20 @@ fn @nvvm_thread(idx: fn(i32) -> u32, gid: fn() -> u32, body: fn(thread_context) gid = gid, uid = @|| gid() as i32, - atomic_load_global_i32 = nvvm_legacy_atomic_memory_order_wrap_load[i32](@|location| { let mut res:i32; asm("atom.global.add.s32 $0, [$1], 0;" : "=r"(res) : "l"(location) : "memory"); res }), - atomic_load_global_u32 = nvvm_legacy_atomic_memory_order_wrap_load[u32](@|location| { let mut res:u32; asm("atom.global.add.u32 $0, [$1], 0;" : "=r"(res) : "l"(location) : "memory"); res }), - atomic_load_global_i64 = nvvm_legacy_atomic_memory_order_wrap_load[i64](@|location| { let mut res:i64; asm("atom.global.add.u64 $0, [$1], 0;" : "=l"(res) : "l"(location) : "memory"); res }), - atomic_load_global_u64 = nvvm_legacy_atomic_memory_order_wrap_load[u64](@|location| { let mut res:u64; asm("atom.global.add.u64 $0, [$1], 0;" : "=l"(res) : "l"(location) : "memory"); res }), + atomic_load_global_i32 = atomic_load_global_i32, + atomic_load_global_u32 = atomic_load_global_u32, + atomic_load_global_i64 = atomic_load_global_i64, + atomic_load_global_u64 = atomic_load_global_u64, atomic_load_global_i32_coalesced = nvvm_legacy_atomic_memory_order_wrap_load[i32](@|location| { let mut res:i32; asm("ld.volatile.global.b32 $0, [$1];" : "=r"(res) : "l"(location) : "memory"); res }), atomic_load_global_u32_coalesced = nvvm_legacy_atomic_memory_order_wrap_load[u32](@|location| { let mut res:u32; asm("ld.volatile.global.b32 $0, [$1];" : "=r"(res) : "l"(location) : "memory"); res }), atomic_load_global_i64_coalesced = nvvm_legacy_atomic_memory_order_wrap_load[i64](@|location| { let mut res:i64; asm("ld.volatile.global.b64 $0, [$1];" : "=l"(res) : "l"(location) : "memory"); res }), atomic_load_global_u64_coalesced = nvvm_legacy_atomic_memory_order_wrap_load[u64](@|location| { let mut res:u64; asm("ld.volatile.global.b64 $0, [$1];" : "=l"(res) : "l"(location) : "memory"); res }), - atomic_store_global_i32 = nvvm_legacy_atomic_memory_order_wrap_store[i32](@|location, value| { let mut res:i32; asm("atom.global.exch.b32 $0, [$1], $2;" : "=r"(res) : "l"(location), "r"(value) : "memory"); }), - atomic_store_global_u32 = nvvm_legacy_atomic_memory_order_wrap_store[u32](@|location, value| { let mut res:u32; asm("atom.global.exch.b32 $0, [$1], $2;" : "=r"(res) : "l"(location), "r"(value) : "memory"); }), - atomic_store_global_i64 = nvvm_legacy_atomic_memory_order_wrap_store[i64](@|location, value| { let mut res:i64; asm("atom.global.exch.b64 $0, [$1], $2;" : "=l"(res) : "l"(location), "l"(value) : "memory"); }), - atomic_store_global_u64 = nvvm_legacy_atomic_memory_order_wrap_store[u64](@|location, value| { let mut res:u64; asm("atom.global.exch.b64 $0, [$1], $2;" : "=l"(res) : "l"(location), "l"(value) : "memory"); }), + atomic_store_global_i32 = atomic_store_global_i32, + atomic_store_global_u32 = atomic_store_global_u32, + atomic_store_global_i64 = atomic_store_global_i64, + atomic_store_global_u64 = atomic_store_global_u64, atomic_store_global_i32_coalesced = nvvm_legacy_atomic_memory_order_wrap_store[i32](@|location, value| asm("st.volatile.global.b32 [$0], $1;" : : "l"(location), "r"(value) : "memory")), atomic_store_global_u32_coalesced = nvvm_legacy_atomic_memory_order_wrap_store[u32](@|location, value| asm("st.volatile.global.b32 [$0], $1;" : : "l"(location), "r"(value) : "memory")), From 08ddcc9b3b10b7b58f49400fad2a793c24fb6563 Mon Sep 17 00:00:00 2001 From: Michael Kenzel Date: Fri, 24 Nov 2023 06:14:05 +0100 Subject: [PATCH 5/6] improved broker queue implementation --- src/queues/BQ/broker_queue.art | 157 ++++++++++++++++++++------------- src/queues/BQ/queue.cmake | 6 +- 2 files changed, 100 insertions(+), 63 deletions(-) diff --git a/src/queues/BQ/broker_queue.art b/src/queues/BQ/broker_queue.art index 9821e0af..a99265dc 100644 --- a/src/queues/BQ/broker_queue.art +++ b/src/queues/BQ/broker_queue.art @@ -16,12 +16,12 @@ enum bwd_alloc_result { Failed } -struct BWDAllocator { +struct BQAllocator { alloc: fn(i64) -> bwd_alloc_result, release: fn(Buffer) -> () } -fn @createBrokerWorkDistributorQueue_internal[T](queue_size: i32, allocator: BWDAllocator, reference_impl: bool) -> create_queue_result[T] { +fn @createBrokerQueue_internal[T](queue_size: i32, allocator: BQAllocator, bwd: bool, reference_impl: bool) -> create_queue_result[T] { if queue_size < 0 { return(create_queue_result[T]::Err("invalid queue size")) } @@ -49,80 +49,83 @@ fn @createBrokerWorkDistributorQueue_internal[T](queue_size: i32, allocator: BWD let buffer = &mut queue_device_memory(buffer_offset) as &mut addrspace(1) [T]; let queue = &mut queue_device_memory(queue_offset) as &mut addrspace(1) BWD::Queue; + // fn @atomic_head_tail(thread: thread_context) { + // let ht = thread.atomic_load_global_u64(&mut queue.head as &mut addrspace(1) u64, memory_order::relaxed); + // ((ht & 0xFFFFFFFF) as u32, (ht >> 32) as u32) + // } fn @wait_for_ticket(i: u32, number: u32, thread: thread_context) -> () { let load_ticket = if reference_impl { - @|| thread.atomic_load_global_u32_coalesced(tickets(i), memory_order::relaxed) + @|| thread.atomic_load_global_u32(&mut tickets(i), memory_order::relaxed) } else { - @|| thread.atomic_load_global_u32(tickets(i), memory_order::relaxed) + // @|| thread.atomic_xor_global_u32(&mut tickets(i), 0, memory_order::relaxed) + @|| thread.atomic_load_global_u32(&mut tickets(i), memory_order::relaxed) }; - thread.wait(@|| load_ticket() == number, "BWD waiting for ticket"); + thread.wait(@|| load_ticket() == number, "BWD waiting for ticket"); } fn @ensure_dequeue(thread: thread_context) -> bool { - let load_size = if reference_impl { - @|| queue.size + let mut num = if reference_impl { + thread.atomic_load_global_i32(&mut queue.size, memory_order::relaxed) } else { - @|| thread.atomic_load_global_i32(queue.size, memory_order::relaxed) + thread.atomic_xor_global_i32(&mut queue.size, 0, memory_order::relaxed) }; - let mut num = load_size(); - - let mut ensurance = false; - - while !ensurance && num > 0 { - if thread.atomic_sub_global_i32(queue.size, 1, memory_order::relaxed) > 0 { - ensurance = true; + while true { + if num <= 0 { + return(false) } - else { - num = thread.atomic_add_global_i32(queue.size, 1, memory_order::relaxed) + 1; + + if thread.atomic_sub_global_i32(&mut queue.size, 1, memory_order::relaxed) > 0 { + break() } + + num = thread.atomic_add_global_i32(&mut queue.size, 1, memory_order::relaxed) + 1; } - ensurance + true } fn @ensure_enqueue(thread: thread_context) -> bool { - let load_size = if reference_impl { - @|| queue.size + let mut num = if reference_impl { + thread.atomic_load_global_i32(&mut queue.size, memory_order::relaxed) } else { - @|| thread.atomic_load_global_i32(queue.size, memory_order::relaxed) + thread.atomic_xor_global_i32(&mut queue.size, 0, memory_order::relaxed) }; - let mut num = load_size(); - - let mut ensurance = false; - - while !ensurance && num < queue_size { - if thread.atomic_add_global_i32(queue.size, 1, memory_order::relaxed) < queue_size { - ensurance = true; + while true { + if num >= queue_size { + return(false) } - else { - num = thread.atomic_sub_global_i32(queue.size, 1, memory_order::relaxed) - 1; + + if thread.atomic_add_global_i32(&mut queue.size, 1, memory_order::relaxed) < queue_size { + break() } + + num = thread.atomic_sub_global_i32(&mut queue.size, 1, memory_order::relaxed) - 1; } - ensurance + true } fn @read_data(sink: fn(T) -> (), thread: thread_context) -> () { let store_ticket = if reference_impl { - @|p:u32, value:u32| { thread.atomic_store_global_u32_coalesced(tickets(p), value, memory_order::relaxed); } + @|p:u32, value:u32| { thread.atomic_store_global_u32(&mut tickets(p), value, memory_order::release); } } else { - @|p:u32, value:u32| { tickets(p) = value; } + // @|p:u32, value:u32| { thread.atomic_exch_global_u32(&mut tickets(p), value, memory_order::release); } + @|p:u32, value:u32| { thread.atomic_store_global_u32(&mut tickets(p), value, memory_order::release); } }; - let pos = thread.atomic_add_global_u32(queue.head, 1, memory_order::relaxed); + let pos = thread.atomic_add_global_u32(&mut queue.head, 1, memory_order::relaxed); let p = pos % queue_size as u32; - wait_for_ticket(p, 2 * (pos / queue_size as u32) + 1, thread); + thread.memory_barrier(memory_order::acquire); let val = buffer(p); - thread.memory_barrier(memory_order::acq_rel); store_ticket(p, 2 * ((pos + queue_size as u32) / queue_size as u32)); sink(val); @@ -130,51 +133,77 @@ fn @createBrokerWorkDistributorQueue_internal[T](queue_size: i32, allocator: BWD fn @put_data(source: fn() -> T, thread: thread_context) -> () { let store_ticket = if reference_impl { - @|p:u32, value:u32| { thread.atomic_store_global_u32_coalesced(tickets(p), value, memory_order::relaxed); } + @|p:u32, value:u32| { thread.atomic_store_global_u32(&mut tickets(p), value, memory_order::release); } } else { - @|p:u32, value:u32| { tickets(p) = value; } + // @|p:u32, value:u32| { thread.atomic_exch_global_u32(&mut tickets(p), value, memory_order::release); } + @|p:u32, value:u32| { thread.atomic_store_global_u32(&mut tickets(p), value, memory_order::release); } }; - let pos = thread.atomic_add_global_u32(queue.tail, 1, memory_order::relaxed); - let p = pos % queue_size as u32; - let b = 2 * (pos / queue_size as u32); - let val = source(); - wait_for_ticket(p, b, thread); + let pos = thread.atomic_add_global_u32(&mut queue.tail, 1, memory_order::relaxed); + let p = pos % queue_size as u32; + wait_for_ticket(p, 2 * (pos / queue_size as u32), thread); buffer(p) = val; - thread.memory_barrier(memory_order::release); - store_ticket(p, b + 1); + store_ticket(p, 2 * (pos / queue_size as u32) + 1); } create_queue_result[T]::Ok(ProducerConsumerQueue[T] { push = @|source| @|thread| { - if ensure_enqueue(thread) { - put_data(source, thread); - 1 + if bwd { + if ensure_enqueue(thread) { + put_data(source, thread); + 1 + } + else { + 0 + } } else { 0 + // while !ensure_enqueue() { + // let (head, tail) = atomic_head_tail(thread); + // if N <= tail - head < N + MaxThreads/2 { + // return(0) + // } + // } + + // put_data(source, thread); + // 1 } }, pop = @|sink| @|thread| { - if ensure_dequeue(thread) { - read_data(sink, thread); - 1 + if bwd { + if ensure_dequeue(thread) { + read_data(sink, thread); + 1 + } + else { + 0 + } } else { 0 + // while !ensure_enqueue() { + // let (head, tail) = atomic_head_tail(thread); + // if N + MaxThreads/2 <= tail - head - 1 { + // return(0) + // } + // } + + // read_data(sink, thread); + // 1 } }, size = @|thread| { if reference_impl { - queue.size + thread.atomic_load_global_i32(&mut queue.size, memory_order::relaxed) } else { - thread.atomic_load_global_i32(queue.size, memory_order::relaxed) + thread.atomic_load_global_i32(&mut queue.size, memory_order::relaxed) } }, @@ -201,20 +230,26 @@ fn @createBrokerWorkDistributorQueue_internal[T](queue_size: i32, allocator: BWD }) } -fn @bwd_dynamic_alloc(device: AccDevice) = BWDAllocator { +fn @bwd_dynamic_alloc(device: AccDevice) = BQAllocator { alloc = @|size: i64| bwd_alloc_result::Ok(device.alloc(size)), release = @|buffer| release(buffer) }; fn @createBrokerWorkDistributorQueue[T](device: AccDevice, queue_size: i32) { - createBrokerWorkDistributorQueue_internal[T](queue_size, bwd_dynamic_alloc(device), false) + createBrokerQueue_internal[T](queue_size, bwd_dynamic_alloc(device), true, false) } -static mut bwd_static_queue_buffer: [u8 * 268435456]; +fn @createBrokerWorkDistributorQueueOrig[T](device: AccDevice, queue_size: i32) { + createBrokerQueue_internal[T](queue_size, bwd_dynamic_alloc(device), true, true) +} -fn @bwd_static_alloc(device: AccDevice) = BWDAllocator { +static mut bwd_static_queue_buffer: [u8 * 536870912]; + +fn @bwd_static_alloc(device: AccDevice) = BQAllocator { alloc = @|size: i64| { - if size <= 268435456 { + if size <= 536870912 && (device.platform_device & 0xF) != 0 { + // ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + // cannot access static buffer from non-host device bwd_alloc_result::Ok(Buffer { data = &mut bwd_static_queue_buffer as &mut [i8], size = size, @@ -229,9 +264,9 @@ fn @bwd_static_alloc(device: AccDevice) = BWDAllocator { }; fn @createBrokerWorkDistributorQueueStatic[T](device: AccDevice, queue_size: i32) { - createBrokerWorkDistributorQueue_internal[T](queue_size, bwd_static_alloc(device), false) + createBrokerQueue_internal[T](queue_size, bwd_static_alloc(device), true, false) } -fn @createBrokerWorkDistributorQueueOrig[T](device: AccDevice, queue_size: i32) { - createBrokerWorkDistributorQueue_internal[T](queue_size, bwd_static_alloc(device), true) +fn @createBrokerWorkDistributorQueueOrigStatic[T](device: AccDevice, queue_size: i32) { + createBrokerQueue_internal[T](queue_size, bwd_static_alloc(device), true, true) } diff --git a/src/queues/BQ/queue.cmake b/src/queues/BQ/queue.cmake index 1b50828b..ea9d1dd0 100644 --- a/src/queues/BQ/queue.cmake +++ b/src/queues/BQ/queue.cmake @@ -1,13 +1,15 @@ set(BrokerWorkDistributorQueue_short_name BWD) set(BrokerWorkDistributorQueueStatic_short_name BWD_static) set(BrokerWorkDistributorQueueOrig_short_name BWD_orig) -set(BrokerWorkDistributorQueueOrigCUDA_short_name BWD_cuda_orig) +set(BrokerWorkDistributorQueueOrigStatic_short_name BWD_orig_static) +set(BrokerWorkDistributorQueueOrigCUDA_short_name BWD_orig_cuda) set(BrokerWorkDistributorQueueCUDA_short_name BWD_cuda) set(BrokerWorkDistributorQueueCUDAIndirect_short_name BWD_cuda_indirect) set(BrokerWorkDistributorQueue_sources ${CMAKE_CURRENT_LIST_DIR}/broker_queue.art) set(BrokerWorkDistributorQueueStatic_sources ${CMAKE_CURRENT_LIST_DIR}/broker_queue.art) set(BrokerWorkDistributorQueueOrig_sources ${CMAKE_CURRENT_LIST_DIR}/broker_queue.art) +set(BrokerWorkDistributorQueueOrigStatic_sources ${CMAKE_CURRENT_LIST_DIR}/broker_queue.art) set(BrokerWorkDistributorQueueOrigCUDA_sources ${CMAKE_CURRENT_LIST_DIR}/broker_queue_cuda.art) set(BrokerWorkDistributorQueueCUDA_sources ${CMAKE_CURRENT_LIST_DIR}/broker_queue_cuda.art) set(BrokerWorkDistributorQueueCUDAIndirect_sources ${CMAKE_CURRENT_LIST_DIR}/broker_queue_cuda_indirect.art) @@ -27,7 +29,7 @@ function (BrokerWorkDistributorQueueCUDA_configure_target target patch_includes) get_target_property(_bin_dir ${target} ANYDSL_BINARY_DIR) get_target_property(_name ${target} NAME) - set(cuda_src "${_bin_dir}/${_name}") + set(cuda_src "${_bin_dir}/$/${_name}") add_custom_command( OUTPUT ${cuda_src}.ll From 57bbedbb3815c83e5e62490943940e13bb2d88c7 Mon Sep 17 00:00:00 2001 From: Michael Kenzel Date: Fri, 24 Nov 2023 06:21:36 +0100 Subject: [PATCH 6/6] udpate .gitignore --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 6c478c5e..dba27acc 100644 --- a/.gitignore +++ b/.gitignore @@ -3,3 +3,4 @@ __pycache__/** dependencies/** build/** html/** +plot_data/**