diff --git a/common/arg.cpp b/common/arg.cpp index ec24cc008c14..164f53728586 100644 --- a/common/arg.cpp +++ b/common/arg.cpp @@ -2900,6 +2900,13 @@ common_params_context common_params_parser_init(common_params & params, llama_ex params.no_op_offload = !value; } )); + add_opt(common_arg( + {"-pw", "--prefetch-weights"}, "0|1", + string_format("prefetch weight transfers to overlap CPU->GPU copies with compute (default: %d)", (int) params.prefetch_weights), + [](common_params & params, int value) { + params.prefetch_weights = value != 0; + } + )); add_opt(common_arg( {"--lora"}, "FNAME", "path to LoRA adapter (use comma-separated values to load multiple adapters)", diff --git a/common/common.cpp b/common/common.cpp index 663cc4ace780..189dc1569efd 100644 --- a/common/common.cpp +++ b/common/common.cpp @@ -1720,6 +1720,7 @@ struct llama_context_params common_context_params_to_llama(const common_params & cparams.op_offload = !params.no_op_offload; cparams.swa_full = params.swa_full; cparams.kv_unified = params.kv_unified; + cparams.prefetch_weights = params.prefetch_weights; cparams.type_k = params.cache_type_k; cparams.type_v = params.cache_type_v; diff --git a/common/common.h b/common/common.h index a8ecb0bc0df8..101c96de167d 100644 --- a/common/common.h +++ b/common/common.h @@ -593,6 +593,7 @@ struct common_params { bool warmup = true; // warmup run bool check_tensors = false; // validate tensor data bool no_op_offload = false; // globally disable offload host tensor operations to device + bool prefetch_weights = false; // prefetch weight transfers to overlap CPU->GPU copies with compute bool no_extra_bufts = false; // disable extra buffer types (used for weight repacking) common_moe_cache_params moe_cache; bool no_host = false; // bypass host buffer allowing extra buffers to be used diff --git a/ggml/include/ggml-backend.h b/ggml/include/ggml-backend.h index 2924fdbe9884..99090fef7887 100644 --- a/ggml/include/ggml-backend.h +++ b/ggml/include/ggml-backend.h @@ -154,6 +154,8 @@ extern "C" { bool buffer_from_host_ptr; // event synchronization bool events; + // dedicated copy stream for compute/transfer overlap + bool copy_stream; }; // all the device properties @@ -351,6 +353,9 @@ extern "C" { // Set a callback to be called for each resulting node during graph compute GGML_API void ggml_backend_sched_set_eval_callback(ggml_backend_sched_t sched, ggml_backend_sched_eval_callback callback, void * user_data); + // Enable async weight prefetching to overlap CPU->GPU transfers with compute + GGML_API void ggml_backend_sched_set_prefetch_weights(ggml_backend_sched_t sched, bool enabled); + // // Meta backend // diff --git a/ggml/src/ggml-backend-meta.cpp b/ggml/src/ggml-backend-meta.cpp index 499c3b767ce3..26f9e3d7fd8d 100644 --- a/ggml/src/ggml-backend-meta.cpp +++ b/ggml/src/ggml-backend-meta.cpp @@ -132,6 +132,7 @@ static void ggml_backend_meta_device_get_props(ggml_backend_dev_t dev, ggml_back /* .host_buffer = */ false, // Not implemented. /* .buffer_from_host_ptr = */ false, // Not implemented. /* .events = */ false, // Not implemented. + /* .copy_stream = */ false, // Not available }; for (ggml_backend_dev_t simple_dev : meta_dev_ctx->simple_devs) { ggml_backend_dev_props tmp_props; diff --git a/ggml/src/ggml-backend.cpp b/ggml/src/ggml-backend.cpp index 7debcde5278e..9fd2df6d2186 100644 --- a/ggml/src/ggml-backend.cpp +++ b/ggml/src/ggml-backend.cpp @@ -975,6 +975,12 @@ struct ggml_backend_sched { size_t context_buffer_size; bool op_offload; + bool prefetch_weights; + + // prefetch support: copy backends and events for compute/transfer overlap + ggml_backend_t copy_backends[GGML_SCHED_MAX_BACKENDS]; + ggml_backend_event_t copy_events[GGML_SCHED_MAX_BACKENDS]; // copy -> compute sync + ggml_backend_event_t compute_events[GGML_SCHED_MAX_BACKENDS]; // compute -> copy sync int debug; @@ -1497,6 +1503,47 @@ void ggml_backend_sched_split_graph(ggml_backend_sched_t sched, struct ggml_cgra } } + // when prefetch is enabled and a new split is needed, progressively fuse + // consecutive nodes that also require weight offloading to the same backend. + // intermediate nodes without weight offload (e.g. GLU between up and down in MoE) + // are included in the fused range. a max lookahead distance prevents scanning + // too far past the last offloading node. + int fuse_end = i; // inclusive end index in graph + if (sched->prefetch_weights && need_new_split) { + const int max_lookahead = 8; + int last_offload_k = i; + for (int k = i + 1; k < graph->n_nodes; k++) { + struct ggml_tensor * next = graph->nodes[k]; + if (ggml_is_view_op(next->op)) { + continue; + } + if (tensor_backend_id(next) != node_backend_id) { + break; + } + if (k - last_offload_k > max_lookahead) { + break; + } + bool has_weight_offload = false; + for (int j = 0; j < GGML_MAX_SRC; j++) { + struct ggml_tensor * src = next->src[j]; + if (src == NULL) { + continue; + } + if (src->buffer != NULL && src->buffer->usage == GGML_BACKEND_BUFFER_USAGE_WEIGHTS) { + int src_backend_id = tensor_backend_id(src); + if (src_backend_id != node_backend_id && !ggml_backend_sched_buffer_supported(sched, src, node_backend_id)) { + has_weight_offload = true; + break; + } + } + } + if (has_weight_offload) { + fuse_end = k; + last_offload_k = k; + } + } + } + if (node_backend_id != cur_backend_id || need_new_split) { split->i_end = i; i_split++; @@ -1518,63 +1565,71 @@ void ggml_backend_sched_split_graph(ggml_backend_sched_t sched, struct ggml_cgra } // find inputs that are not on the same backend - for (int j = 0; j < GGML_MAX_SRC; j++) { - struct ggml_tensor * src = node->src[j]; - if (src == NULL) { + for (int fi = i; fi <= fuse_end; fi++) { + struct ggml_tensor * fnode = graph->nodes[fi]; + if (ggml_is_view_op(fnode->op)) { continue; } - - size_t src_id = hash_id(src); - const int src_backend_id = sched->hv_tensor_backend_ids[src_id]; - GGML_ASSERT(src_backend_id != -1); // all inputs should be assigned by now - - if (src->flags & GGML_TENSOR_FLAG_INPUT && sched->n_copies > 1) { - if (tensor_id_copy(src_id, src_backend_id, 0) == NULL) { - ggml_backend_t backend = sched->backends[src_backend_id]; - for (int c = 0; c < sched->n_copies; c++) { - struct ggml_tensor * tensor_copy; - if (c == sched->cur_copy) { - tensor_copy = src; // use the original tensor as the current copy - } else { - tensor_copy = ggml_dup_tensor_layout(sched->ctx, src); - ggml_format_name(tensor_copy, "%s#%s#%d", ggml_backend_name(backend), src->name, c); - } - ggml_set_input(tensor_copy); - ggml_set_output(tensor_copy); // prevent ggml-alloc from overwriting the tensor - tensor_id_copy(src_id, src_backend_id, c) = tensor_copy; - SET_CAUSE(tensor_copy, "4.cpy"); - } - int n_graph_inputs = sched->n_graph_inputs++; - if (n_graph_inputs >= sched->graph_inputs_capacity) { - ggml_backend_sched_graph_inputs_grow(sched); - } - sched->graph_inputs[n_graph_inputs] = src; + for (int j = 0; j < GGML_MAX_SRC; j++) { + struct ggml_tensor * src = fnode->src[j]; + if (src == NULL) { + continue; } - } - if (src_backend_id != cur_backend_id && !ggml_backend_sched_buffer_supported(sched, src, cur_backend_id)) { - // create a copy of the input in the split's backend - if (tensor_id_copy(src_id, cur_backend_id, 0) == NULL) { - ggml_backend_t backend = sched->backends[cur_backend_id]; - for (int c = 0; c < sched->n_copies; c++) { - struct ggml_tensor * tensor_copy = ggml_dup_tensor_layout(sched->ctx, src); - ggml_format_name(tensor_copy, "%s#%s#%d", ggml_backend_name(backend), src->name, c); - if (sched->n_copies > 1) { + size_t src_id = hash_id(src); + const int src_backend_id = sched->hv_tensor_backend_ids[src_id]; + GGML_ASSERT(src_backend_id != -1); // all inputs should be assigned by now + + if (src->flags & GGML_TENSOR_FLAG_INPUT && sched->n_copies > 1) { + if (tensor_id_copy(src_id, src_backend_id, 0) == NULL) { + ggml_backend_t backend = sched->backends[src_backend_id]; + for (int c = 0; c < sched->n_copies; c++) { + struct ggml_tensor * tensor_copy; + if (c == sched->cur_copy) { + tensor_copy = src; // use the original tensor as the current copy + } else { + tensor_copy = ggml_dup_tensor_layout(sched->ctx, src); + ggml_format_name(tensor_copy, "%s#%s#%d", ggml_backend_name(backend), src->name, c); + } ggml_set_input(tensor_copy); ggml_set_output(tensor_copy); // prevent ggml-alloc from overwriting the tensor + tensor_id_copy(src_id, src_backend_id, c) = tensor_copy; + SET_CAUSE(tensor_copy, "4.cpy"); + } + int n_graph_inputs = sched->n_graph_inputs++; + if (n_graph_inputs >= sched->graph_inputs_capacity) { + ggml_backend_sched_graph_inputs_grow(sched); } - tensor_id_copy(src_id, cur_backend_id, c) = tensor_copy; - SET_CAUSE(tensor_copy, "4.cpy"); + sched->graph_inputs[n_graph_inputs] = src; } - int n_inputs = split->n_inputs++; - if (n_inputs >= split->inputs_capacity) { - ggml_backend_sched_split_inputs_grow(split); + } + + if (src_backend_id != cur_backend_id && !ggml_backend_sched_buffer_supported(sched, src, cur_backend_id)) { + // create a copy of the input in the split's backend + if (tensor_id_copy(src_id, cur_backend_id, 0) == NULL) { + ggml_backend_t backend = sched->backends[cur_backend_id]; + for (int c = 0; c < sched->n_copies; c++) { + struct ggml_tensor * tensor_copy = ggml_dup_tensor_layout(sched->ctx, src); + ggml_format_name(tensor_copy, "%s#%s#%d", ggml_backend_name(backend), src->name, c); + if (sched->n_copies > 1) { + ggml_set_input(tensor_copy); + ggml_set_output(tensor_copy); // prevent ggml-alloc from overwriting the tensor + } + tensor_id_copy(src_id, cur_backend_id, c) = tensor_copy; + SET_CAUSE(tensor_copy, "4.cpy"); + } + int n_inputs = split->n_inputs++; + if (n_inputs >= split->inputs_capacity) { + ggml_backend_sched_split_inputs_grow(split); + } + split->inputs[n_inputs] = src; } - split->inputs[n_inputs] = src; + fnode->src[j] = tensor_id_copy(src_id, cur_backend_id, sched->cur_copy); } - node->src[j] = tensor_id_copy(src_id, cur_backend_id, sched->cur_copy); } } + + i = fuse_end; } split->i_end = graph->n_nodes; sched->n_splits = i_split + 1; @@ -1599,7 +1654,9 @@ void ggml_backend_sched_split_graph(ggml_backend_sched_t sched, struct ggml_cgra for (int i = 0; i < sched->n_splits; i++) { total_inputs += sched->splits[i].n_inputs; } - int graph_size = std::max(graph->n_nodes, graph->n_leafs) + total_inputs * 2 * sched->n_copies; + // extra nodes per split input: 2 (dep + copy), plus 2 more for keepalive nodes when prefetching + const int nodes_per_input = sched->prefetch_weights ? 4 : 2; + int graph_size = std::max(graph->n_nodes, graph->n_leafs) + total_inputs * nodes_per_input * sched->n_copies; // remember the actual graph_size for performing reallocation checks later [GGML_SCHED_DEBUG_REALLOC] sched->debug_prev_graph_size = sched->debug_graph_size; @@ -1644,11 +1701,51 @@ void ggml_backend_sched_split_graph(ggml_backend_sched_t sched, struct ggml_cgra graph_copy->nodes[graph_copy->n_nodes++] = input_cpy; } + // prefetch double-buffer: reserve next split's weight copy memory BEFORE compute + // so the allocator doesn't reuse it for intermediates in this split + if (sched->prefetch_weights && i + 1 < sched->n_splits) { + struct ggml_backend_sched_split * next = &sched->splits[i + 1]; + if (next->backend_id == split->backend_id) { + for (int j = 0; j < next->n_inputs; j++) { + struct ggml_tensor * next_input = next->inputs[j]; + if (next_input->buffer != NULL && + ggml_backend_buffer_get_usage(next_input->buffer) == GGML_BACKEND_BUFFER_USAGE_WEIGHTS && + ggml_backend_buffer_is_host(next_input->buffer)) { + const size_t id = hash_id(next_input); + struct ggml_tensor * next_cpy = tensor_id_copy(id, next->backend_id, sched->cur_copy); + assert(graph_copy->size > graph_copy->n_nodes); + struct ggml_tensor * keepalive = ggml_view_tensor(sched->ctx, next_cpy); + keepalive->src[0] = next_cpy; + sched->node_backend_ids[graph_copy->n_nodes] = next->backend_id; + graph_copy->nodes[graph_copy->n_nodes++] = keepalive; + } + } + } + } + for (int j = split->i_start; j < split->i_end; j++) { assert(graph_copy->size > graph_copy->n_nodes); sched->node_backend_ids[graph_copy->n_nodes] = tensor_backend_id(graph->nodes[j]); graph_copy->nodes[graph_copy->n_nodes++] = graph->nodes[j]; } + + // extend current split's weight copies lifetime to here (after next's are allocated above) + if (sched->prefetch_weights) { + for (int j = 0; j < split->n_inputs; j++) { + struct ggml_tensor * input = split->inputs[j]; + if (input->buffer != NULL && + ggml_backend_buffer_get_usage(input->buffer) == GGML_BACKEND_BUFFER_USAGE_WEIGHTS && + ggml_backend_buffer_is_host(input->buffer)) { + const size_t id = hash_id(input); + struct ggml_tensor * curr_cpy = tensor_id_copy(id, split->backend_id, sched->cur_copy); + assert(graph_copy->size > graph_copy->n_nodes); + struct ggml_tensor * keepalive = ggml_view_tensor(sched->ctx, curr_cpy); + keepalive->src[0] = curr_cpy; + sched->node_backend_ids[graph_copy->n_nodes] = split->backend_id; + graph_copy->nodes[graph_copy->n_nodes++] = keepalive; + } + } + } } if (sched->n_copies > 1) { @@ -1735,6 +1832,9 @@ static bool ggml_backend_sched_alloc_splits(ggml_backend_sched_t sched) { // synchronize without ggml_backend_sched_synchronize to avoid changing cur_copy for (int i = 0; i < sched->n_backends; i++) { ggml_backend_synchronize(sched->backends[i]); + if (sched->copy_backends[i] != NULL) { + ggml_backend_synchronize(sched->copy_backends[i]); + } } ggml_gallocr_reserve_n(sched->galloc, &sched->graph, sched->node_backend_ids, sched->leaf_backend_ids); @@ -1783,17 +1883,62 @@ static enum ggml_status ggml_backend_sched_compute_splits(ggml_backend_sched_t s std::vector ids; std::vector used_ids; + bool next_weights_prefetched = false; + for (int split_id = 0; split_id < sched->n_splits; split_id++) { struct ggml_backend_sched_split * split = &splits[split_id]; int split_backend_id = split->backend_id; ggml_backend_t split_backend = sched->backends[split_backend_id]; + bool weights_prefetched = next_weights_prefetched; + next_weights_prefetched = false; + + if (sched->prefetch_weights) { + ggml_backend_t copy_backend = sched->copy_backends[split_backend_id]; + if (copy_backend != NULL) { + // compute stream waits for previous prefetch to complete + ggml_backend_event_wait(split_backend, sched->copy_events[split_backend_id]); + + // prefetch next split's weights on the copy stream + if (split_id + 1 < sched->n_splits) { + struct ggml_backend_sched_split * next = &splits[split_id + 1]; + // only prefetch when next split is on the same device + if (next->backend_id == split_backend_id) { + // copy stream waits for previous compute to complete + ggml_backend_event_wait(copy_backend, sched->compute_events[split_backend_id]); + + for (int input_id = 0; input_id < next->n_inputs; input_id++) { + struct ggml_tensor * input = next->inputs[input_id]; + if (input->buffer != NULL && + ggml_backend_buffer_get_usage(input->buffer) == GGML_BACKEND_BUFFER_USAGE_WEIGHTS && + ggml_backend_buffer_is_host(input->buffer)) { + struct ggml_tensor * input_cpy = tensor_copy(input, next->backend_id, sched->cur_copy); + ggml_backend_tensor_set_async(copy_backend, input_cpy, input->data, 0, ggml_nbytes(input)); + next_weights_prefetched = true; + } + } + + // signal that prefetch is done + ggml_backend_event_record(sched->copy_events[split_backend_id], copy_backend); + } + } + } + } + // copy the input tensors to the split backend for (int input_id = 0; input_id < split->n_inputs; input_id++) { ggml_backend_t input_backend = ggml_backend_sched_get_tensor_backend(sched, split->inputs[input_id]); struct ggml_tensor * input = split->inputs[input_id]; struct ggml_tensor * input_cpy = tensor_copy(input, split_backend_id, sched->cur_copy); + // skip weight inputs that were already prefetched by the previous split + if (weights_prefetched && + input->buffer != NULL && + ggml_backend_buffer_get_usage(input->buffer) == GGML_BACKEND_BUFFER_USAGE_WEIGHTS && + ggml_backend_buffer_is_host(input->buffer)) { + continue; + } + if (input->flags & GGML_TENSOR_FLAG_INPUT) { // inputs from the user must be copied immediately to prevent the user overwriting the data before the copy is done if (sched->events[split_backend_id][sched->cur_copy] != NULL) { @@ -1816,7 +1961,6 @@ static enum ggml_status ggml_backend_sched_compute_splits(ggml_backend_sched_t s ggml_backend_buffer_get_usage(input->buffer) == GGML_BACKEND_BUFFER_USAGE_WEIGHTS && ggml_backend_buffer_is_host(input->buffer) && ( (node->src[0] == input_cpy && node->op == GGML_OP_MUL_MAT_ID) - //|| (node->src[1] == input_cpy && node->op == GGML_OP_ADD_ID) /* GGML_OP_ADD_ID weights are small and not worth splitting */ )) { const int64_t n_expert = node->op == GGML_OP_MUL_MAT_ID ? input->ne[2] : input->ne[1]; @@ -1950,12 +2094,18 @@ static enum ggml_status ggml_backend_sched_compute_splits(ggml_backend_sched_t s } } + // record compute done for copy stream sync + if (sched->compute_events[split_backend_id] != NULL) { + ggml_backend_event_record(sched->compute_events[split_backend_id], split_backend); + } + // record the event of this copy if (split->n_inputs > 0) { if (sched->events[split_backend_id][sched->cur_copy] != NULL) { ggml_backend_event_record(sched->events[split_backend_id][sched->cur_copy], split_backend); } } + } return GGML_STATUS_SUCCESS; @@ -1994,7 +2144,7 @@ ggml_backend_sched_t ggml_backend_sched_new( sched->hv_tensor_copies = (ggml_tensor **) malloc(sched->hash_set.size * sched->n_backends * sched->n_copies * sizeof(struct ggml_tensor *)); const size_t ggml_sched_max_splits = graph_size; // at most there is one split for each node in the graph - const size_t nodes_size = graph_size + ggml_sched_max_splits*GGML_SCHED_MAX_SPLIT_INPUTS*2; + const size_t nodes_size = graph_size + ggml_sched_max_splits*GGML_SCHED_MAX_SPLIT_INPUTS*4; sched->node_backend_ids = (int *) calloc(nodes_size, sizeof(sched->node_backend_ids[0])); sched->leaf_backend_ids = (int *) calloc(nodes_size, sizeof(sched->leaf_backend_ids[0])); sched->prev_node_backend_ids = (int *) calloc(nodes_size, sizeof(sched->prev_node_backend_ids[0])); @@ -2003,7 +2153,7 @@ ggml_backend_sched_t ggml_backend_sched_new( sched->debug_graph_size = 0; sched->debug_prev_graph_size = 0; - sched->context_buffer_size = ggml_sched_max_splits*GGML_SCHED_MAX_SPLIT_INPUTS*2*sizeof(struct ggml_tensor) + ggml_graph_overhead_custom(graph_size, false); + sched->context_buffer_size = ggml_sched_max_splits*GGML_SCHED_MAX_SPLIT_INPUTS*4*sizeof(struct ggml_tensor) + ggml_graph_overhead_custom(graph_size, false); sched->context_buffer = (char *) malloc(sched->context_buffer_size); const int initial_splits_capacity = 16; @@ -2106,6 +2256,11 @@ void ggml_backend_sched_free(ggml_backend_sched_t sched) { for (int c = 0; c < sched->n_copies; c++) { ggml_backend_event_free(sched->events[b][c]); } + ggml_backend_event_free(sched->copy_events[b]); + ggml_backend_event_free(sched->compute_events[b]); + if (sched->copy_backends[b] != NULL) { + ggml_backend_free(sched->copy_backends[b]); + } } ggml_gallocr_free(sched->galloc); ggml_free(sched->ctx); @@ -2214,6 +2369,9 @@ void ggml_backend_sched_synchronize(ggml_backend_sched_t sched) { GGML_ASSERT(sched); for (int i = 0; i < sched->n_backends; i++) { ggml_backend_synchronize(sched->backends[i]); + if (sched->copy_backends[i] != NULL) { + ggml_backend_synchronize(sched->copy_backends[i]); + } } if (!sched->is_alloc) { // if the graph is not already allocated, always use copy 0 after a synchronization @@ -2229,6 +2387,41 @@ void ggml_backend_sched_set_eval_callback(ggml_backend_sched_t sched, ggml_backe sched->callback_eval_user_data = user_data; } +void ggml_backend_sched_set_prefetch_weights(ggml_backend_sched_t sched, bool enabled) { + GGML_ASSERT(sched); + sched->prefetch_weights = enabled; + + if (enabled) { + for (int b = 0; b < sched->n_backends; b++) { + if (sched->copy_backends[b] != NULL) { + continue; + } + ggml_backend_dev_t dev = ggml_backend_get_device(sched->backends[b]); + if (dev == NULL) { + continue; + } + struct ggml_backend_dev_props props; + ggml_backend_dev_get_props(dev, &props); + if (props.caps.copy_stream) { + sched->copy_backends[b] = ggml_backend_dev_init(dev, NULL); + sched->copy_events[b] = ggml_backend_event_new(dev); + sched->compute_events[b] = ggml_backend_event_new(dev); + } + } + } else { + for (int b = 0; b < sched->n_backends; b++) { + ggml_backend_event_free(sched->copy_events[b]); + sched->copy_events[b] = NULL; + ggml_backend_event_free(sched->compute_events[b]); + sched->compute_events[b] = NULL; + if (sched->copy_backends[b] != NULL) { + ggml_backend_free(sched->copy_backends[b]); + sched->copy_backends[b] = NULL; + } + } + } +} + int ggml_backend_sched_get_n_splits(ggml_backend_sched_t sched) { GGML_ASSERT(sched); return sched->n_splits; diff --git a/ggml/src/ggml-blas/ggml-blas.cpp b/ggml/src/ggml-blas/ggml-blas.cpp index 9745fa29f5db..d3e91b892713 100644 --- a/ggml/src/ggml-blas/ggml-blas.cpp +++ b/ggml/src/ggml-blas/ggml-blas.cpp @@ -367,6 +367,7 @@ static void ggml_backend_blas_device_get_props(ggml_backend_dev_t dev, struct gg /* .host_buffer = */ false, /* .buffer_from_host_ptr = */ true, /* .events = */ false, + /* .copy_stream = */ false, }; } diff --git a/ggml/src/ggml-cann/ggml-cann.cpp b/ggml/src/ggml-cann/ggml-cann.cpp index 5f51ea3bb3c8..106268008070 100644 --- a/ggml/src/ggml-cann/ggml-cann.cpp +++ b/ggml/src/ggml-cann/ggml-cann.cpp @@ -2815,6 +2815,7 @@ static void ggml_backend_cann_device_get_props(ggml_backend_dev_t dev, ggml_back /* .host_buffer = */ host_buffer, /* .buffer_from_host_ptr = */ false, /* .events = */ true, + /* .copy_stream = */ false, }; } diff --git a/ggml/src/ggml-cpu/ggml-cpu.cpp b/ggml/src/ggml-cpu/ggml-cpu.cpp index 16cc5116c545..859a4c8676db 100644 --- a/ggml/src/ggml-cpu/ggml-cpu.cpp +++ b/ggml/src/ggml-cpu/ggml-cpu.cpp @@ -397,6 +397,7 @@ static void ggml_backend_cpu_device_get_props(ggml_backend_dev_t dev, struct ggm /* .host_buffer = */ false, /* .buffer_from_host_ptr = */ true, /* .events = */ false, + /* .copy_stream = */ false, }; } diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cu index dc1439b2c761..d86ac8158fc4 100644 --- a/ggml/src/ggml-cuda/ggml-cuda.cu +++ b/ggml/src/ggml-cuda/ggml-cuda.cu @@ -4871,6 +4871,7 @@ static void ggml_backend_cuda_device_get_props(ggml_backend_dev_t dev, ggml_back /* .host_buffer = */ host_buffer, /* .buffer_from_host_ptr = */ false, /* .events = */ events, + /* .copy_stream = */ true, }; } diff --git a/ggml/src/ggml-hexagon/ggml-hexagon.cpp b/ggml/src/ggml-hexagon/ggml-hexagon.cpp index bdb8af0820a3..09be7db8e037 100644 --- a/ggml/src/ggml-hexagon/ggml-hexagon.cpp +++ b/ggml/src/ggml-hexagon/ggml-hexagon.cpp @@ -3930,6 +3930,7 @@ static void ggml_backend_hexagon_device_get_props(ggml_backend_dev_t dev, struct /* .host_buffer = */ (bool) opt_hostbuf, /* .buffer_from_host_ptr = */ false, /* .events = */ false, + /* .copy_stream = */ false, }; } diff --git a/ggml/src/ggml-metal/ggml-metal.cpp b/ggml/src/ggml-metal/ggml-metal.cpp index 32a6272e112f..80a720b39413 100644 --- a/ggml/src/ggml-metal/ggml-metal.cpp +++ b/ggml/src/ggml-metal/ggml-metal.cpp @@ -683,6 +683,7 @@ static void ggml_backend_metal_device_get_props(ggml_backend_dev_t dev, ggml_bac /* .host_buffer = */ false, /* .buffer_from_host_ptr = */ true, /* .events = */ true, + /* .copy_stream = */ false, }; } diff --git a/ggml/src/ggml-opencl/ggml-opencl.cpp b/ggml/src/ggml-opencl/ggml-opencl.cpp index fc0fce0d780a..d81bcd5ef11e 100644 --- a/ggml/src/ggml-opencl/ggml-opencl.cpp +++ b/ggml/src/ggml-opencl/ggml-opencl.cpp @@ -10777,6 +10777,7 @@ static void ggml_backend_opencl_device_get_props(ggml_backend_dev_t dev, struct /* .host_buffer = */ false, /* .buffer_from_host_ptr = */ false, /* .events = */ false, + /* .copy_stream = */ false, }; } diff --git a/ggml/src/ggml-openvino/ggml-openvino.cpp b/ggml/src/ggml-openvino/ggml-openvino.cpp index 0e7501fefe38..4ce19d3de1a3 100644 --- a/ggml/src/ggml-openvino/ggml-openvino.cpp +++ b/ggml/src/ggml-openvino/ggml-openvino.cpp @@ -763,6 +763,7 @@ static void ggml_backend_openvino_device_get_props(ggml_backend_dev_t dev, ggml_ /* .host_buffer = */ false, /* .buffer_from_host_ptr = */ false, /* .events = */ false, + /* .copy_stream = */ false, }; } diff --git a/ggml/src/ggml-rpc/ggml-rpc.cpp b/ggml/src/ggml-rpc/ggml-rpc.cpp index 17c53a5f049e..340087de1aff 100644 --- a/ggml/src/ggml-rpc/ggml-rpc.cpp +++ b/ggml/src/ggml-rpc/ggml-rpc.cpp @@ -1881,6 +1881,7 @@ static void ggml_backend_rpc_device_get_props(ggml_backend_dev_t dev, struct ggm /* .host_buffer = */ false, /* .buffer_from_host_ptr = */ false, /* .events = */ false, + /* .copy_stream = */ false, }; } diff --git a/ggml/src/ggml-sycl/ggml-sycl.cpp b/ggml/src/ggml-sycl/ggml-sycl.cpp index 12cc3fdc5a8a..45f271aca38c 100644 --- a/ggml/src/ggml-sycl/ggml-sycl.cpp +++ b/ggml/src/ggml-sycl/ggml-sycl.cpp @@ -5639,6 +5639,7 @@ static void ggml_backend_sycl_device_get_props(ggml_backend_dev_t dev, ggml_back /* .host_buffer = */ host_buffer, /* .buffer_from_host_ptr = */ false, /* .events = */ events, + /* .copy_stream = */ false, }; } diff --git a/ggml/src/ggml-virtgpu/ggml-backend-device.cpp b/ggml/src/ggml-virtgpu/ggml-backend-device.cpp index a978812cd908..8c896d1bc76e 100644 --- a/ggml/src/ggml-virtgpu/ggml-backend-device.cpp +++ b/ggml/src/ggml-virtgpu/ggml-backend-device.cpp @@ -70,6 +70,7 @@ static void ggml_backend_remoting_device_get_props(ggml_backend_dev_t dev, ggml_ props->caps.buffer_from_host_ptr = false; props->caps.async = false; props->caps.events = false; + props->caps.copy_stream = false; } ggml_backend_buffer_type_t ggml_backend_remoting_device_get_buffer_type(ggml_backend_dev_t dev) { diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vulkan.cpp index 405ff9235178..b2dab3bb9b5c 100644 --- a/ggml/src/ggml-vulkan/ggml-vulkan.cpp +++ b/ggml/src/ggml-vulkan/ggml-vulkan.cpp @@ -18159,6 +18159,7 @@ static void ggml_backend_vk_device_get_props(ggml_backend_dev_t dev, struct ggml /* .host_buffer = */ true, /* .buffer_from_host_ptr = */ false, /* .events = */ true, + /* .copy_stream = */ false, }; } diff --git a/ggml/src/ggml-webgpu/ggml-webgpu.cpp b/ggml/src/ggml-webgpu/ggml-webgpu.cpp index c001cda7d116..5ce9a4c59277 100644 --- a/ggml/src/ggml-webgpu/ggml-webgpu.cpp +++ b/ggml/src/ggml-webgpu/ggml-webgpu.cpp @@ -3954,6 +3954,7 @@ static void ggml_backend_webgpu_device_get_props(ggml_backend_dev_t dev, struct /* .host_buffer = */ false, /* .buffer_from_host_ptr = */ false, /* .events = */ false, + /* .copy_stream = */ false, }; } diff --git a/ggml/src/ggml-zdnn/ggml-zdnn.cpp b/ggml/src/ggml-zdnn/ggml-zdnn.cpp index 639b818d128e..60ecb22467c2 100644 --- a/ggml/src/ggml-zdnn/ggml-zdnn.cpp +++ b/ggml/src/ggml-zdnn/ggml-zdnn.cpp @@ -419,22 +419,22 @@ static enum ggml_status ggml_backend_zdnn_graph_compute(ggml_backend_t backend, } static ggml_backend_i ggml_backend_zdnn_i = { - /* .get_name = */ ggml_backend_zdnn_name, - /* .free = */ ggml_backend_zdnn_free, - /* .set_tensor_async = */ NULL, - /* .get_tensor_async = */ NULL, - /* .set_tensor_2d_async = */ NULL, - /* .get_tensor_2d_async = */ NULL, - /* .cpy_tensor_async = */ NULL, - /* .synchronize = */ NULL, - /* .graph_plan_create = */ NULL, - /* .graph_plan_free = */ NULL, - /* .graph_plan_update = */ NULL, - /* .graph_plan_compute = */ NULL, - /* .graph_compute = */ ggml_backend_zdnn_graph_compute, - /* .event_record = */ NULL, - /* .event_wait = */ NULL, - /* .graph_optimize = */ NULL, + /* .get_name = */ ggml_backend_zdnn_name, + /* .free = */ ggml_backend_zdnn_free, + /* .set_tensor_async = */ NULL, + /* .get_tensor_async = */ NULL, + /* .set_tensor_2d_async = */ NULL, + /* .get_tensor_2d_async = */ NULL, + /* .cpy_tensor_async = */ NULL, + /* .synchronize = */ NULL, + /* .graph_plan_create = */ NULL, + /* .graph_plan_free = */ NULL, + /* .graph_plan_update = */ NULL, + /* .graph_plan_compute = */ NULL, + /* .graph_compute = */ ggml_backend_zdnn_graph_compute, + /* .event_record = */ NULL, + /* .event_wait = */ NULL, + /* .graph_optimize = */ NULL, }; static ggml_guid_t ggml_backend_zdnn_guid(void) { @@ -487,7 +487,8 @@ static void ggml_backend_zdnn_device_get_props(ggml_backend_dev_t dev, ggml_back /* .async = */ false, /* .host_buffer = */ false, /* .buffer_from_host_ptr = */ false, - /* .events = */ false + /* .events = */ false, + /* .copy_stream = */ false, }; } diff --git a/ggml/src/ggml-zendnn/ggml-zendnn.cpp b/ggml/src/ggml-zendnn/ggml-zendnn.cpp index e6a9b51b7925..330d7d8df9d0 100644 --- a/ggml/src/ggml-zendnn/ggml-zendnn.cpp +++ b/ggml/src/ggml-zendnn/ggml-zendnn.cpp @@ -654,7 +654,8 @@ static void ggml_backend_zendnn_device_get_props(ggml_backend_dev_t dev, struct /* .async = */ false, /* .host_buffer = */ false, /* .buffer_from_host_ptr = */ true, - /* .events = */ false + /* .events = */ false, + /* .copy_stream = */ false, }; } diff --git a/include/llama.h b/include/llama.h index cb79ab838563..6e2cfe661301 100644 --- a/include/llama.h +++ b/include/llama.h @@ -409,6 +409,8 @@ extern "C" { // try to disable when n_seq_max > 1 for improved performance when the sequences do not share a large prefix // ref: https://github.com/ggml-org/llama.cpp/pull/14363 + bool prefetch_weights; // prefetch weight transfers to overlap CPU->GPU copies with compute + // [EXPERIMENTAL] // backend sampler chain configuration (make sure the caller keeps the sampler chains alive) // note: the samplers must be sampler chains (i.e. use llama_sampler_chain_init) diff --git a/src/llama-context.cpp b/src/llama-context.cpp index 2b91f1b545e5..fea22fcb0ac1 100644 --- a/src/llama-context.cpp +++ b/src/llama-context.cpp @@ -296,8 +296,9 @@ llama_context::llama_context( cparams.n_outputs_max = params.n_outputs_max == 0 || llama_model_has_encoder(&model) ? cparams.n_batch : params.n_outputs_max; - cparams.op_offload = params.op_offload; - cparams.kv_unified = params.kv_unified; + cparams.op_offload = params.op_offload; + cparams.kv_unified = params.kv_unified; + cparams.prefetch_weights = params.prefetch_weights; // initialized later cparams.pipeline_parallel = false; @@ -740,6 +741,7 @@ void llama_context::sched_reserve() { ggml_backend_sched_set_moe_cache( sched.get(), moe_cache_mode, cparams.moe_cache_budget_mib); + ggml_backend_sched_set_prefetch_weights(sched.get(), cparams.prefetch_weights); llama_memory_context_ptr mctx; if (memory) { @@ -832,6 +834,7 @@ void llama_context::sched_reserve() { ggml_backend_sched_set_moe_cache( sched.get(), moe_cache_mode, cparams.moe_cache_budget_mib); + ggml_backend_sched_set_prefetch_weights(sched.get(), cparams.prefetch_weights); gf = graph_reserve(n_tokens, n_seqs, n_outputs_pp, mctx.get()); } if (!gf) { @@ -3810,6 +3813,7 @@ llama_context_params llama_context_default_params() { /*.op_offload =*/ true, /*.swa_full =*/ true, /*.kv_unified =*/ false, + /*.prefetch_weights =*/ false, /*.sampler =*/ nullptr, /*.n_sampler =*/ 0, /*.ctx_other =*/ nullptr, diff --git a/src/llama-cparams.h b/src/llama-cparams.h index 7f04941e3c8c..0fef395d9d5a 100644 --- a/src/llama-cparams.h +++ b/src/llama-cparams.h @@ -56,6 +56,7 @@ struct llama_cparams { bool op_offload; bool kv_unified; bool pipeline_parallel; + bool prefetch_weights; std::vector embeddings_layer_inp; // [n_layer()] extract input embeddings for layer diff --git a/tools/llama-bench/llama-bench.cpp b/tools/llama-bench/llama-bench.cpp index 802f9f963cb0..d86c017cdac6 100644 --- a/tools/llama-bench/llama-bench.cpp +++ b/tools/llama-bench/llama-bench.cpp @@ -433,6 +433,7 @@ struct cmd_params { std::vector> tensor_buft_overrides; std::vector embeddings; std::vector no_op_offload; + std::vector prefetch_weights; std::vector no_host; std::vector fit_params_target; std::vector fit_params_min_ctx; @@ -483,6 +484,7 @@ static const cmd_params cmd_params_defaults = { /* tensor_buft_overrides*/ { std::vector{ { nullptr, nullptr } } }, /* embeddings */ { false }, /* no_op_offload */ { false }, + /* prefetch_weights */ { false }, /* no_host */ { false }, /* fit_params_target */ { 0 }, /* fit_params_min_ctx */ { 0 }, @@ -1041,6 +1043,13 @@ static cmd_params parse_cmd_params(int argc, char ** argv) { } auto p = string_split(argv[i], split_delim); params.no_op_offload.insert(params.no_op_offload.end(), p.begin(), p.end()); + } else if (arg == "-pw" || arg == "--prefetch-weights") { + if (++i >= argc) { + invalid_param = true; + break; + } + auto p = string_split(argv[i], split_delim); + params.prefetch_weights.insert(params.prefetch_weights.end(), p.begin(), p.end()); } else if (arg == "--no-host") { if (++i >= argc) { invalid_param = true; @@ -1317,6 +1326,9 @@ static cmd_params parse_cmd_params(int argc, char ** argv) { if (params.no_op_offload.empty()) { params.no_op_offload = cmd_params_defaults.no_op_offload; } + if (params.prefetch_weights.empty()) { + params.prefetch_weights = cmd_params_defaults.prefetch_weights; + } if (params.no_host.empty()) { params.no_host = cmd_params_defaults.no_host; } @@ -1381,6 +1393,7 @@ struct cmd_params_instance { std::vector tensor_buft_overrides; bool embeddings; bool no_op_offload; + bool prefetch_weights; bool no_host; size_t fit_target; uint32_t fit_min_ctx; @@ -1459,6 +1472,7 @@ struct cmd_params_instance { cparams.flash_attn_type = flash_attn; cparams.embeddings = embeddings; cparams.op_offload = !no_op_offload; + cparams.prefetch_weights = prefetch_weights; cparams.swa_full = false; return cparams; @@ -1485,6 +1499,7 @@ static std::vector get_cmd_params_instances(const cmd_param for (const auto & noh : params.no_host) for (const auto & embd : params.embeddings) for (const auto & nopo : params.no_op_offload) + for (const auto & pw : params.prefetch_weights) for (const auto & nb : params.n_batch) for (const auto & nub : params.n_ubatch) for (const auto & tk : params.type_k) @@ -1528,6 +1543,7 @@ static std::vector get_cmd_params_instances(const cmd_param /* .tensor_buft_overrides = */ ot, /* .embeddings = */ embd, /* .no_op_offload = */ nopo, + /* .prefetch_weights = */ pw, /* .no_host = */ noh, /* .fit_target = */ fpt, /* .fit_min_ctx = */ fpc, @@ -1567,6 +1583,7 @@ static std::vector get_cmd_params_instances(const cmd_param /* .tensor_buft_overrides = */ ot, /* .embeddings = */ embd, /* .no_op_offload = */ nopo, + /* .prefetch_weights = */ pw, /* .no_host = */ noh, /* .fit_target = */ fpt, /* .fit_min_ctx = */ fpc, @@ -1606,6 +1623,7 @@ static std::vector get_cmd_params_instances(const cmd_param /* .tensor_buft_overrides = */ ot, /* .embeddings = */ embd, /* .no_op_offload = */ nopo, + /* .prefetch_weights = */ pw, /* .no_host = */ noh, /* .fit_target = */ fpt, /* .fit_min_ctx = */ fpc, @@ -1650,6 +1668,7 @@ struct test { std::vector tensor_buft_overrides; bool embeddings; bool no_op_offload; + bool prefetch_weights; bool no_host; size_t fit_target; uint32_t fit_min_ctx; @@ -1715,6 +1734,7 @@ struct test { } embeddings = inst.embeddings; no_op_offload = inst.no_op_offload; + prefetch_weights = inst.prefetch_weights; no_host = inst.no_host; fit_target = inst.fit_target; fit_min_ctx = inst.fit_min_ctx; @@ -1777,7 +1797,7 @@ struct test { "moe_cache_fit", "repack", "split_mode", "main_gpu", "no_kv_offload", "flash_attn", "devices", "tensor_split", "tensor_buft_overrides", "load_mode", "embeddings", - "no_op_offload", "no_host", "fit_target", "fit_min_ctx", + "no_op_offload", "prefetch_weights", "no_host", "fit_target", "fit_min_ctx", "n_prompt", "n_gen", "n_gen_warmup", "n_depth", "test_time", "avg_ns", "stddev_ns", "avg_ts", "stddev_ts" }; @@ -1791,7 +1811,7 @@ struct test { field == "poll" || field == "model_size" || field == "model_n_params" || field == "n_gpu_layers" || field == "main_gpu" || field == "n_prompt" || field == "n_gen" || field == "n_gen_warmup" || field == "n_depth" || field == "avg_ns" || - field == "stddev_ns" || field == "no_op_offload" || field == "n_cpu_moe" || + field == "stddev_ns" || field == "no_op_offload" || field == "prefetch_weights" || field == "n_cpu_moe" || field == "fit_target" || field == "fit_min_ctx" || field == "flash_attn") { return INT; } @@ -1877,6 +1897,7 @@ struct test { llama_load_mode_name(load_mode), std::to_string(embeddings), std::to_string(no_op_offload), + std::to_string(prefetch_weights), std::to_string(no_host), std::to_string(fit_target), std::to_string(fit_min_ctx), @@ -2082,6 +2103,9 @@ struct markdown_printer : public printer { if (field == "no_op_offload") { return 4; } + if (field == "prefetch_weights") { + return 2; + } if (field == "no_host") { return 4; } @@ -2128,6 +2152,9 @@ struct markdown_printer : public printer { if (field == "no_op_offload") { return "nopo"; } + if (field == "prefetch_weights") { + return "pw"; + } if (field == "no_host") { return "noh"; } @@ -2239,6 +2266,9 @@ struct markdown_printer : public printer { if (params.no_op_offload.size() > 1 || params.no_op_offload != cmd_params_defaults.no_op_offload) { fields.emplace_back("no_op_offload"); } + if (params.prefetch_weights.size() > 1 || params.prefetch_weights != cmd_params_defaults.prefetch_weights) { + fields.emplace_back("prefetch_weights"); + } if (params.no_host.size() > 1 || params.no_host != cmd_params_defaults.no_host) { fields.emplace_back("no_host"); }