Repository navigation
Expand file tree
/
Copy pathcuda_graph_overrides.h
More file actions
94 lines (87 loc) · 3.42 KB
/
Copy pathcuda_graph_overrides.h
File metadata and controls
94 lines (87 loc) · 3.42 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
#pragma once
#include "ggml-cuda.h"
namespace luce::common {
// Thread-local CUDA/HIP graph controls used while executing a graph whose
// topology or backend metadata requires a temporary policy override. The
// setters return their prior state, which makes this scope safe to nest.
class ScopedCudaGraphOverrides {
public:
explicit ScopedCudaGraphOverrides(
bool disable_graphs = false,
int mmvq_max_ncols = 0,
bool skip_property_check = false,
int ds4_mix_mmv_max_tokens = 0,
bool mmvq_batch_invariant = false,
int mmvq_invariant_max_ncols = 0)
: disable_graphs_(disable_graphs),
override_mmvq_(mmvq_max_ncols > 0),
skip_property_check_(skip_property_check),
override_ds4_mix_(ds4_mix_mmv_max_tokens > 0),
batch_invariant_(mmvq_batch_invariant),
override_invariant_width_(mmvq_invariant_max_ncols > 0) {
if (batch_invariant_) {
previous_batch_invariant_ =
ggml_backend_cuda_set_mmvq_batch_invariant(true);
}
if (override_invariant_width_) {
previous_invariant_width_ =
ggml_backend_cuda_set_mmvq_invariant_max_ncols(mmvq_invariant_max_ncols);
}
if (disable_graphs_) {
previous_graphs_disabled_ =
ggml_backend_cuda_set_graphs_disabled_override(true);
}
if (override_mmvq_) {
previous_mmvq_max_ncols_ =
ggml_backend_cuda_set_mmvq_max_ncols_override(
mmvq_max_ncols);
}
if (skip_property_check_) {
previous_skip_property_check_ =
ggml_backend_cuda_set_skip_props_check(true);
}
if (override_ds4_mix_) {
previous_ds4_mix_max_tokens_ =
ggml_backend_cuda_set_ds4_mix_mmv_max_tokens_override(ds4_mix_mmv_max_tokens);
}
}
~ScopedCudaGraphOverrides() {
if (override_invariant_width_) {
ggml_backend_cuda_set_mmvq_invariant_max_ncols(previous_invariant_width_);
}
if (batch_invariant_) {
ggml_backend_cuda_set_mmvq_batch_invariant(previous_batch_invariant_);
}
if (override_ds4_mix_) {
ggml_backend_cuda_set_ds4_mix_mmv_max_tokens_override(previous_ds4_mix_max_tokens_);
}
if (skip_property_check_) {
ggml_backend_cuda_set_skip_props_check(
previous_skip_property_check_);
}
if (override_mmvq_) {
ggml_backend_cuda_set_mmvq_max_ncols_override(
previous_mmvq_max_ncols_);
}
if (disable_graphs_) {
ggml_backend_cuda_set_graphs_disabled_override(
previous_graphs_disabled_);
}
}
ScopedCudaGraphOverrides(const ScopedCudaGraphOverrides &) = delete;
ScopedCudaGraphOverrides & operator=(const ScopedCudaGraphOverrides &) = delete;
private:
bool disable_graphs_ = false;
bool override_mmvq_ = false;
bool skip_property_check_ = false;
bool override_ds4_mix_ = false;
bool batch_invariant_ = false;
bool override_invariant_width_ = false;
bool previous_graphs_disabled_ = false;
bool previous_batch_invariant_ = false;
bool previous_skip_property_check_ = false;
int previous_mmvq_max_ncols_ = 0;
int previous_ds4_mix_max_tokens_ = 0;
int previous_invariant_width_ = 0;
};
} // namespace luce::common