Merge pull request #32015 from ggml-org/gg/fix-sampling-output-aliasing
This commit is contained in:
2 files changed
+10
-16
No files matched your search
+8
-16
@@ -1428,6 +1428,7 @@ void llm_graph_result::reset() {
|
||||
t_sampled_probs.clear();
|
||||
t_sampled_logits.clear();
|
||||
t_candidates.clear();
|
||||
t_sampling_all.clear();
|
||||
|
||||
params = {};
|
||||
|
||||
@@ -1476,22 +1477,8 @@ void llm_graph_result::set_outputs(const llm_graph_params & params) {
|
||||
}
|
||||
}
|
||||
}
|
||||
for (auto * tensor : t_sampled) {
|
||||
if (tensor != nullptr) {
|
||||
ggml_set_output(tensor);
|
||||
}
|
||||
}
|
||||
for (auto * tensor : t_sampled_probs) {
|
||||
if (tensor != nullptr) {
|
||||
ggml_set_output(tensor);
|
||||
}
|
||||
}
|
||||
for (auto * tensor : t_sampled_logits) {
|
||||
if (tensor != nullptr) {
|
||||
ggml_set_output(tensor);
|
||||
}
|
||||
}
|
||||
for (auto * tensor : t_candidates) {
|
||||
// the chains are read on the host after compute, so keep every one alive
|
||||
for (auto * tensor : t_sampling_all) {
|
||||
if (tensor != nullptr) {
|
||||
ggml_set_output(tensor);
|
||||
}
|
||||
@@ -4507,6 +4494,11 @@ void llm_graph_context::build_sampling() const {
|
||||
assert(sampler->iface->backend_apply);
|
||||
sampler->iface->backend_apply(sampler, ctx0, gf, &data);
|
||||
|
||||
if (data.sampled != nullptr) { res->t_sampling_all.push_back(data.sampled); }
|
||||
if (data.probs != nullptr) { res->t_sampling_all.push_back(data.probs); }
|
||||
if (data.logits != nullptr) { res->t_sampling_all.push_back(data.logits); }
|
||||
if (data.candidates != nullptr) { res->t_sampling_all.push_back(data.candidates); }
|
||||
|
||||
if (data.sampled != nullptr) {
|
||||
if (active) {
|
||||
res->t_sampled[row] = data.sampled;
|
||||
|
||||
Reference in new issue
Block a user