diff --git a/src/llama-graph.cpp b/src/llama-graph.cpp index b5d4642a2..f9dd3e0f6 100644 --- a/src/llama-graph.cpp +++ b/src/llama-graph.cpp @@ -1418,6 +1418,7 @@ void llm_graph_result::reset() { t_sampled_probs.clear(); t_sampled_logits.clear(); t_candidates.clear(); + t_sampling_all.clear(); params = {}; @@ -1466,22 +1467,8 @@ void llm_graph_result::set_outputs(const llm_graph_params & params) { } } } - for (auto * tensor : t_sampled) { - if (tensor != nullptr) { - ggml_set_output(tensor); - } - } - for (auto * tensor : t_sampled_probs) { - if (tensor != nullptr) { - ggml_set_output(tensor); - } - } - for (auto * tensor : t_sampled_logits) { - if (tensor != nullptr) { - ggml_set_output(tensor); - } - } - for (auto * tensor : t_candidates) { + // the chains are read on the host after compute, so keep every one alive + for (auto * tensor : t_sampling_all) { if (tensor != nullptr) { ggml_set_output(tensor); } @@ -4455,6 +4442,11 @@ void llm_graph_context::build_sampling() const { assert(sampler->iface->backend_apply); sampler->iface->backend_apply(sampler, ctx0, gf, &data); + if (data.sampled != nullptr) { res->t_sampling_all.push_back(data.sampled); } + if (data.probs != nullptr) { res->t_sampling_all.push_back(data.probs); } + if (data.logits != nullptr) { res->t_sampling_all.push_back(data.logits); } + if (data.candidates != nullptr) { res->t_sampling_all.push_back(data.candidates); } + if (data.sampled != nullptr) { if (active) { res->t_sampled[row] = data.sampled; diff --git a/src/llama-graph.h b/src/llama-graph.h index 9867c9540..da473f043 100644 --- a/src/llama-graph.h +++ b/src/llama-graph.h @@ -979,6 +979,8 @@ public: std::vector t_sampled_logits; std::vector t_candidates; + std::vector t_sampling_all; + std::vector inputs; std::vector fused_nodes;