Repository navigation
Expand file tree
/
Copy pathdflash_draft_kv.h
More file actions
149 lines (128 loc) · 6.28 KB
/
Copy pathdflash_draft_kv.h
File metadata and controls
149 lines (128 loc) · 6.28 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
// dflash_draft_kv.h — drafter context-KV ring cache (target-agnostic).
//
// The DFlash drafter recomputes K/V over its whole feature window (up to
// draft_ctx_max tokens) every step, which dominates draft latency once the
// window fills (~10ms at 2048 on RTX 3090 vs ~3ms on short prompts). The
// window is append-only per commit, so this module keeps per-layer K/V ring
// caches on the draft backend and only computes the newly committed rows.
//
// One fixed-topology step graph is built at init: a fold-in append stage
// (up to a_step new rows per step, padded rows land in a trash slot) followed
// by the noise-token forward flash-attending over the cache. Every input
// tensor lives in a dedicated backend buffer (never gallocr-recycled), so the
// graph replays as a CUDA graph indefinitely — no per-step rebuilds at all.
//
// RoPE uses absolute positions; scores depend only on position differences,
// so results match the legacy rebased-window math while cached entries stay
// valid as the window slides (ring slot = position % cap).
#pragma once
#include "dflash_feature_ring.h"
#include "draft/draft_graph.h"
#include "internal.h" // DraftWeights
#include "ggml.h"
#include "ggml-alloc.h"
#include "ggml-backend.h"
#include <cstdint>
#include <vector>
namespace luce::common {
struct DraftKvState {
DraftKvState() = default;
// Owns raw ggml contexts/buffers/graphs: copying would double-free.
DraftKvState(const DraftKvState &) = delete;
DraftKvState & operator=(const DraftKvState &) = delete;
// geometry
int cap = 0; // ctx ring slots (== drafter feature window cap)
int q_len = 0; // noise block size
int kv_total = 0; // cache rows: cap + q_len + trash + mask alignment
int a_step = 0; // fold-in append capacity per step
int trash_slot = 0; // destination for padded append rows
int fc_in = 0;
bool any_full = false, any_swa = false;
// persistent device memory: caches + every graph input.
ggml_context * mem_ctx = nullptr;
ggml_backend_buffer_t mem_buf = nullptr;
DraftKvCacheRefs cache;
ggml_tensor * inp_embed = nullptr; // [hidden, q_len] f32 (caller fills)
ggml_tensor * pos_q = nullptr; // [q_len] i32
ggml_tensor * noise_rows = nullptr; // [q_len] i32 (static)
ggml_tensor * mask_full = nullptr; // [kv_total, q_len] f16
ggml_tensor * mask_swa = nullptr; // [kv_total, q_len] f16
ggml_tensor * ap_feat = nullptr; // [fc_in, a_step] f32
ggml_tensor * ap_pos = nullptr; // [a_step] i32
ggml_tensor * ap_rows = nullptr; // [a_step] i32
// the once-built step graph (fold-in append + noise forward)
std::vector<uint8_t> meta_arena;
ggml_context * g_ctx = nullptr;
ggml_cgraph * gf = nullptr;
ggml_gallocr_t galloc = nullptr;
ggml_tensor * hidden_states = nullptr;
ggml_tensor * logits = nullptr; // iff lm_head passed at init
// host bookkeeping
const void * built_for = nullptr; // DraftWeights the graph was built against
int64_t next_pos = 0; // first ctx position not yet appended
std::vector<int32_t> slot_pos; // absolute position per ring slot, -1 empty
std::vector<uint16_t> mask_hbuf;
std::vector<int32_t> i32_hbuf;
};
// Allocate caches + inputs and build the step graph. `cap` is the drafter
// feature-window capacity (ring size). lm_head may be null (hidden-only).
bool draft_kv_init(DraftKvState & st,
const DraftWeights & dw,
ggml_backend_t backend,
int cap,
ggml_tensor * lm_head);
// Allocate persistent lane state without the single-lane graph. Concurrent
// decoding binds these states to DraftKvBatchGraph instead.
bool draft_kv_init_batched(DraftKvState & st,
const DraftWeights & dw,
ggml_backend_t backend,
int cap);
// Invalidate all cached rows (new/rewound request). Cheap; the next
// begin_step bulk-appends the live window from the feature ring.
void draft_kv_reset(DraftKvState & st);
void draft_kv_free(DraftKvState & st);
// Bring the cache up to date with `committed` (bulk-appending from the
// feature ring if more than a_step rows are missing) and fill all step
// inputs (append rows, positions, masks). After this the caller uploads
// inp_embed and computes st.gf; the draft hidden is st.hidden_states.
// width: noise positions the next step drafts, at most st.q_len
// (0 = st.q_len). The full-attention mask hides the noise scratch rows past
// it, which still hold the K/V of an earlier, wider step.
bool draft_kv_begin_step(DraftKvState & st,
const DraftWeights & dw,
ggml_backend_t backend,
const DraftFeatureMirror & ring,
int committed,
int width = 0);
struct DraftKvBatchGraph {
DraftKvBatchGraph() = default;
DraftKvBatchGraph(const DraftKvBatchGraph &) = delete;
DraftKvBatchGraph & operator=(const DraftKvBatchGraph &) = delete;
int n_lanes = 0;
int q_len = 0; // noise positions per lane
const void * built_for = nullptr;
ggml_backend_t backend = nullptr;
std::vector<DraftKvState *> lane_states;
std::vector<uint8_t> meta_arena;
ggml_context * g_ctx = nullptr;
ggml_cgraph * gf = nullptr;
ggml_gallocr_t galloc = nullptr;
std::vector<ggml_tensor *> hidden_by_lane;
};
void draft_kv_batch_free(DraftKvBatchGraph & batch);
// All lane states must already have draft_kv_begin_step() inputs and
// inp_embed uploaded. The packed graph computes the shared backbone and
// returns one host-visible hidden-state block per lane.
// width: noise positions each lane drafts, at most dw.block_size
// (0 = dw.block_size). A narrower step reads the leading positions of the
// lane inputs, filled by draft_kv_begin_step for the same width. A graph is
// rebuilt when the width changes, so a caller that alternates widths keeps
// one DraftKvBatchGraph per width.
bool draft_kv_batch_compute(
DraftKvBatchGraph & batch,
const DraftWeights & dw,
ggml_backend_t backend,
const std::vector<DraftKvState *> & lane_states,
std::vector<std::vector<float>> & hidden_by_lane,
int width = 0);
} // namespace luce::common