diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cu index e76ff3128fdd..4baf49e1fbc3 100644 --- a/ggml/src/ggml-cuda/ggml-cuda.cu +++ b/ggml/src/ggml-cuda/ggml-cuda.cu @@ -305,7 +305,11 @@ static ggml_cuda_device_info ggml_cuda_init() { info.default_tensor_split[id] = total_vram; total_vram += device_vram; +#if defined(GGML_USE_HIP) + info.devices[id].integrated = prop.integrated; +#else info.devices[id].integrated = false; // Temporarily disabled due to issues with corrupted output (e.g. #15034) +#endif info.devices[id].nsm = prop.multiProcessorCount; info.devices[id].smpb = prop.sharedMemPerBlock; info.devices[id].warp_size = prop.warpSize; diff --git a/src/llama-context.cpp b/src/llama-context.cpp index 8675f6087336..aea52c724e38 100644 --- a/src/llama-context.cpp +++ b/src/llama-context.cpp @@ -424,6 +424,25 @@ llama_context::llama_context( LLAMA_LOG_DEBUG("%s: backend_ptrs.size() = %zu\n", __func__, backend_ptrs.size()); + // graph inputs live in the CPU backend's buffer. if a GPU can use that buffer directly, the scheduler + // does not copy the inputs to the device, so nothing orders the next set_inputs after the previous compute + for (size_t i = 0; i < backend_ptrs.size(); ++i) { + if (ggml_backend_dev_type(ggml_backend_get_device(backend_ptrs[i])) != GGML_BACKEND_DEVICE_TYPE_CPU) { + continue; + } + for (auto * backend : backend_ptrs) { + auto * dev = ggml_backend_get_device(backend); + auto dev_type = ggml_backend_dev_type(dev); + if ((dev_type == GGML_BACKEND_DEVICE_TYPE_GPU || dev_type == GGML_BACKEND_DEVICE_TYPE_IGPU) && + ggml_backend_dev_supports_buft(dev, backend_buft[i])) { + sync_host_inputs = true; + } + } + } + if (sync_host_inputs) { + LLAMA_LOG_INFO("%s: GPU reads host input buffer in place, synchronizing before set_inputs\n", __func__); + } + // TODO: move these checks to ggml_backend_sched // enabling pipeline parallelism in the scheduler increases memory usage, so it is only done when necessary bool pipeline_parallel = @@ -1445,6 +1464,11 @@ llm_graph_result * llama_context::process_ubatch(const llama_ubatch & ubatch, ll { //const auto t_start_us = ggml_time_us(); + // the previous graph_compute_async may still be reading the inputs that are about to be overwritten + if (sync_host_inputs) { + ggml_backend_sched_synchronize(sched.get()); + } + // FIXME this call causes a crash if any model inputs were not used in the graph and were therefore not allocated res->set_inputs(&ubatch); diff --git a/src/llama-context.h b/src/llama-context.h index b403b099b76f..c09ec7c77258 100644 --- a/src/llama-context.h +++ b/src/llama-context.h @@ -389,6 +389,10 @@ struct llama_context { // env: LLAMA_GRAPH_REUSE_DISABLE bool graph_reuse_disable = false; + // true if a GPU backend reads the CPU backend's (host) buffer in place, e.g. integrated GPUs + // in that case set_inputs must not overwrite graph inputs while a previous compute is in flight + bool sync_host_inputs = false; + // perf mutable int64_t t_start_us = 0; mutable int64_t t_load_us = 0;