Repository navigation
Expand file tree
/
Copy pathbackend_args.h
More file actions
142 lines (123 loc) · 6.09 KB
/
Copy pathbackend_args.h
File metadata and controls
142 lines (123 loc) · 6.09 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
// Raw backend construction arguments.
//
// This contains only caller-requested configuration. Runtime facts derived
// from the model or compiled binary belong in BackendPlan instead.
#pragma once
#include <limits>
#include <optional>
#include <string>
#include "ggml.h"
#include "placement/draft_residency.h"
#include "placement/placement_config.h"
#include "placement/remote_draft_config.h"
#include "placement/remote_target_shard_config.h"
#include "prefill_attention_mode.h"
#include "cluster/cluster_config.h"
namespace luce::common {
enum class KvFlashRequest {
Off,
Auto,
Fixed,
};
// Server-owned facts that participate in backend admission without becoming
// backend construction arguments. This is the only projection the HTTP server
// may pass into backend preparation.
struct BackendAdmissionContext {
bool pflash_enabled = false;
bool pflash_drafter_configured = false;
DraftResidencyPolicy draft_residency = DraftResidencyPolicy::Auto;
// Automatic sizing remains backend-owned because only the initialized
// backend has the VRAM budget. Fixed pools can participate in admission.
KvFlashRequest kvflash = KvFlashRequest::Off;
bool kvflash_requested() const {
return kvflash != KvFlashRequest::Off;
}
bool fixed_kvflash_requested() const {
return kvflash == KvFlashRequest::Fixed;
}
};
// A superset of all per-architecture config fields. Preparation projects only
// the effective fields into BackendPlan's concern-specific snapshots.
struct BackendArgs {
// Required
std::string model_path; // target .gguf
// Optional: speculative decode draft model (qwen35 only)
std::optional<std::string> draft_path;
// Optional: vision projector .gguf (deepseek4 only)
std::optional<std::string> mmproj_path;
// Optional: GPU for the vision encoder when it should not share the
// target's (deepseek4, one-GPU layout only).
std::optional<DevicePlacement> mmproj_device;
// Device placement
DevicePlacement device;
DevicePlacement draft_device;
RemoteDraftConfig remote_draft;
RemoteTargetShardConfig remote_target_shard;
// Split MoE placement (qwen4exp): routed experts on a second device, and the
// most-routed of them also copied to the target. DeepSeek V4 reads
// --expert-device through its own expert-parallel path instead.
std::optional<DevicePlacement> expert_device;
// I/O — only used when running under daemon_loop (legacy). The new
// server passes -1 and uses on_token callbacks instead.
int stream_fd = -1;
// Chunked prefill
int chunk = 512;
bool chunk_set = false; // --chunk given explicitly
PrefillAttentionMode ds4_prefill_mode = PrefillAttentionMode::Exact;
bool ds4_prefill_mode_set = false;
// deepseek4-specific decode options
int ds4_expert_top_k = 0; // 0 = model default
bool ds4_fused_decode = false;
bool ds4_fused_verify_f16_kv = false;
// Expert ownership and routing files (see docs/DS41.md); empty = unused.
std::string expert_placement; // --expert-placement (alias --ds4-expert-placement)
std::string ds4_router_bias;
// Expert-parallel cluster rank (--cluster-*); size 0 = a single box.
cluster::ClusterConfig cluster;
std::string ds4_protected_experts;
// Attention and speculative-decode options. Individual backends consume
// only the fields they support.
// Explicit per-model overrides; COUNT preserves environment/family defaults.
ggml_type cache_type_k = GGML_TYPE_COUNT;
ggml_type cache_type_v = GGML_TYPE_COUNT;
int fa_window = 0; // 0 = full attention. qwen3.6 full-attn layers must see the whole context; a finite window drops the system prompt/tools -> breaks tool calls.
bool paged_attention = false; // model-specific paged K/V blocks for AR decode
// Concurrent decode slots (--max-concurrency). > 1 requires paged_attention;
// the backend serves that many sequences through the seq_* slot API.
int max_concurrency = 1;
// Total paged K/V pool in tokens shared by all slots (--kv-pool-tokens;
// model-page-rounded). 0 = derive capacity from available device memory.
long long kv_pool_tokens = 0;
// Device memory an automatically sized pool leaves free for a model that
// runs beside concurrent decode (the PFlash drafter). Bytes; 0 = none.
long long kv_pool_reserve_bytes = 0;
// Device memory a model that sizes itself to its GPUs (qwen4exp) leaves
// free for the PFlash drafter scoring beside it: on GPU pflash_reserve_gpu
// (-1: none), its weights plus a workspace per context position.
int pflash_reserve_gpu = -1;
long long pflash_reserve_bytes = 0;
long long pflash_reserve_bytes_per_token = 0;
int kq_stride_pad = 32;
int draft_block_size = 0; // 0 = drafter metadata
int draft_swa_window = 0;
int draft_ctx_max = 4096;
bool fast_rollback = true;
bool seq_verify = false;
bool specla_mode = false;
int specla_top_k = 4;
bool specla_top_k_explicit = false;
bool ddtree_mode = false;
int ddtree_budget = 22;
float ddtree_temp = 1.0f;
bool ddtree_chain_seed = true;
float ddtree_tau = std::numeric_limits<float>::infinity();
bool ddtree_tau_explicit = false;
int verify_width = 0; // chain spec verify width; 0 = adaptive
bool use_feature_mirror = false;
// MoE backend requests. The server currently realizes these through
// environment variables, but admission still treats them as explicit
// operator input rather than server-owned context.
bool routing_stats_requested = false;
bool adaptive_experts_requested = false;
};
} // namespace luce::common