Repository navigation
Expand file tree
/
Copy pathdflash_feature_ring.h
More file actions
158 lines (136 loc) · 6.08 KB
/
Copy pathdflash_feature_ring.h
File metadata and controls
158 lines (136 loc) · 6.08 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
// dflash_feature_ring.h — DFlash draft feature ring buffer (target-agnostic).
//
// Hosts the ring buffer that mirrors target hidden-state captures on the
// draft GPU, plus the helpers that move data across the storage/input dtype
// boundary:
// - target activation tensor → ring slot
// - ring range → contiguous draft input tensor
// - target BF16 feature cache tensor → ring (with dtype conversion,
// possibly across devices)
//
// Lives in common/ so any DFlash target architecture (qwen35, gemma4,
// laguna, ...) can reuse it without depending on architecture-specific
// weight or cache structs.
#pragma once
#include "ggml.h"
#include "ggml-backend.h"
#include <cstddef>
#include <cstdint>
#include <cstdlib>
#include <vector>
namespace luce::common {
// Drafter's trained sliding-window (feature-mirror) context. The drafter's
// feature ring must span at least this many tokens or it only sees a shallow
// suffix of the context and spec-decode accept collapses once the committed
// prefix exceeds it. Pending the converter injecting the real context_length
// into the drafter GGUF (currently a 262144 placeholder).
inline constexpr int DFLASH_DRAFTER_TRAINED_CTX = 40960;
inline constexpr int DFLASH_FEATURE_RING_DEFAULT = 4096;
// Deep drafter feature window. On an opted-in model, the target feature ring
// and feature mirror must use this same value: the mirror is synced from the
// target ring, so mismatched windows feed the drafter aliased features.
// `LUCE_FEAT_RING_CAP` is the memory valve (lower clamps both rings);
// unset/0 means the trained window.
inline int dflash_drafter_window(int max_ctx) {
int cap = max_ctx < DFLASH_DRAFTER_TRAINED_CTX ? max_ctx : DFLASH_DRAFTER_TRAINED_CTX;
if (const char * s = std::getenv("LUCE_FEAT_RING_CAP")) {
const int e = std::atoi(s);
if (e > 0 && e < cap) cap = e;
}
return cap;
}
// Opt the affected target/drafter pair into deep history without changing the
// established ring on other model families.
inline int dflash_feature_ring_cap(int max_ctx, bool needs_deep_history) {
return needs_deep_history
? dflash_drafter_window(max_ctx)
: (max_ctx < DFLASH_FEATURE_RING_DEFAULT
? max_ctx : DFLASH_FEATURE_RING_DEFAULT);
}
// Bound the feature history consumed by one draft step. The feature ring may
// be deeper than the active window: qwen35moe needs a 40K history ring to keep
// old captures coherent, but its measured throughput sweet spot is an 8K
// active window. Keep the configured compute cap on local and remote paths.
inline int dflash_draft_context_cap(int ring_cap, int draft_ctx_max) {
return draft_ctx_max > 0 && draft_ctx_max < ring_cap
? draft_ctx_max : ring_cap;
}
ggml_type dflash_feature_dtype();
struct DraftFeatureMirror {
ggml_context * ctx = nullptr;
ggml_backend_buffer_t buf = nullptr;
ggml_tensor * target_feat = nullptr; // [n_target_layers*hidden_size, cap]
void * staging = nullptr;
size_t staging_bytes = 0;
int device = 0;
int target_device = 0;
int cap = 0;
int n_target_layers = 0;
int hidden_size = 0;
ggml_type storage_type = GGML_TYPE_F32;
};
void draft_feature_mirror_free(DraftFeatureMirror & mirror);
bool draft_feature_mirror_init(DraftFeatureMirror & mirror,
ggml_backend_t backend,
int device,
int target_device,
int cap,
int n_target_layers,
int hidden_size);
// Check whether the mirror ring buffer can provide a contiguous view of
// ctx_len slots ending at committed. Returns true and writes slot0 (the
// starting slot in the ring buffer) on success.
bool draft_feature_mirror_can_view(const DraftFeatureMirror & mirror,
int committed,
int ctx_len,
int & slot0);
// Copy and convert BF16→F32 for n_tokens starting at start_pos from a
// target-side BF16 feature ring (`src_target_feat` / `src_cap`) into the
// draft-side mirror ring buffer.
bool draft_feature_mirror_sync_range(const ggml_tensor * src_target_feat,
int src_cap,
DraftFeatureMirror & mirror,
int start_pos,
int n_tokens);
// Convenience: sync the last `committed` tokens (or mirror.cap, whichever is smaller).
bool draft_feature_mirror_sync_tail(const ggml_tensor * src_target_feat,
int src_cap,
DraftFeatureMirror & mirror,
int committed);
// ── Ring ↔ tensor copy helpers (target-agnostic) ────────────────────
// Copy one capture slice from a target layer's activation output into the
// DraftFeatureMirror ring buffer. src_device is the GPU device of act_out.
bool copy_capture_slice_to_draft_ring(
DraftFeatureMirror & feature_ring,
int capture_idx,
const ggml_tensor * act_out,
int src_device,
int chunk_start,
int start_pos,
int n_tokens);
// Copy one host-side F32 capture slice into the DraftFeatureMirror ring buffer.
bool copy_host_capture_slice_to_draft_ring(
DraftFeatureMirror & feature_ring,
int capture_idx,
int start_pos,
int n_tokens,
const float * host,
size_t host_elems);
// Copy n_tokens rows from the DraftFeatureMirror ring buffer into a
// destination tensor (typically the draft graph's target_hidden_cat input).
bool copy_feature_ring_range_to_tensor(
const DraftFeatureMirror & feature_ring,
ggml_tensor * dst,
int start_pos,
int n_tokens);
bool copy_feature_ring_range_to_host_f32(
const DraftFeatureMirror & feature_ring,
int start_pos,
int n_tokens,
std::vector<float> & out);
bool copy_host_f32_to_feature_ring_range(
DraftFeatureMirror & feature_ring,
int start_pos,
int n_tokens,
const std::vector<float> & src);
} // namespace luce::common