-
Notifications
You must be signed in to change notification settings - Fork 5
Expand file tree
/
Copy pathhost.rs
More file actions
2413 lines (2188 loc) · 111 KB
/
Copy pathhost.rs
File metadata and controls
2413 lines (2188 loc) · 111 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
use std::collections::{BTreeSet, HashMap};
use std::path::{Path, PathBuf};
use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
use std::sync::{Arc, OnceLock, RwLock};
use std::time::Duration;
use animus_plugin_protocol::{
error_codes, EnvRequirement, HealthCheckResult, HostCapabilities, HostInfo, InitializeParams, InitializeResult,
RpcError, RpcNotification, RpcRequest, RpcResponse, PROTOCOL_VERSION,
};
use anyhow::{anyhow, Result};
use semver::Version;
use serde_json::Value;
use thiserror::Error;
use tokio::io::{AsyncBufReadExt, AsyncRead, AsyncReadExt, AsyncWrite, AsyncWriteExt};
use tokio::process::Child;
use tokio::sync::{broadcast, oneshot, Mutex};
use tokio::task::JoinHandle;
use tracing::{debug, warn};
/// Universal shell environment variables that every plugin gets regardless of
/// its declared `env_required` manifest. These are the locale + shell + Rust
/// telemetry vars that practically every CLI tool expects; withholding them
/// breaks even well-behaved plugins for no security gain (none of them carry
/// secrets).
///
/// Anything **not** in this list and **not** explicitly declared by the
/// plugin's manifest is scrubbed from the spawn environment via
/// [`std::process::Command::env_clear`].
pub const PLUGIN_BASE_ENV_ALLOWLIST: &[&str] =
&["PATH", "HOME", "USER", "SHELL", "TERM", "TMPDIR", "LANG", "LC_ALL", "RUST_LOG", "RUST_BACKTRACE", "TZ"];
/// Maximum cumulative bytes (sum of `key.len() + value.len()`) the spawn
/// path will merge from the installed [`SecretSnapshotProvider`] into a
/// child plugin's environment. The cap protects against a runaway
/// keychain index from blowing past the platform's `ARG_MAX`. Mirrors
/// `orchestrator_core::MAX_INJECTED_ENV_BYTES`.
pub const MAX_INJECTED_SECRET_BYTES: usize = 1024 * 1024;
/// Compiled default for the per-host notification broadcast channel capacity.
///
/// Used when neither [`PluginManifest::notification_buffer_size`] nor the
/// `ANIMUS_PLUGIN_BROADCAST_CAPACITY` env override is set. Mirrors the
/// session-backend convention of ~256 in-flight notification slots per
/// subscriber.
///
/// [`PluginManifest::notification_buffer_size`]: animus_plugin_protocol::PluginManifest::notification_buffer_size
pub const DEFAULT_NOTIFICATION_BROADCAST_CAPACITY: usize = 256;
/// Environment variable operators set to override the per-plugin broadcast
/// channel capacity. Lower precedence than the plugin manifest hint, higher
/// precedence than [`DEFAULT_NOTIFICATION_BROADCAST_CAPACITY`].
pub const NOTIFICATION_BROADCAST_CAPACITY_ENV: &str = "ANIMUS_PLUGIN_BROADCAST_CAPACITY";
/// Deadline the [`PluginHost::shutdown`] flow waits for the child to exit
/// after sending the `shutdown` RPC.
const SHUTDOWN_GRACE: Duration = Duration::from_secs(2);
/// Generous upper bound for a single frame write (mutex + write_all + flush)
/// on the untimed request path. A healthy plugin drains its stdin promptly;
/// one that stops reading would otherwise block the writer mutex forever and
/// wedge every queued request, including shutdown. Expiry marks the host
/// dead and surfaces as `ConnectionLost` to callers.
const WRITE_FRAME_TIMEOUT: Duration = Duration::from_secs(30);
/// Cap on the reader's unparsed frame buffer. A plugin that streams an
/// endless (or endlessly malformed) frame without ever completing it would
/// otherwise grow the buffer without bound; past this point the router tears
/// down and every awaiter observes [`HostError::ConnectionLost`].
const READER_BUFFER_CAP: usize = 8 * 1024 * 1024;
/// How many trailing stderr lines to retain per plugin for failure diagnostics.
const STDERR_TAIL_CAP: usize = 40;
/// Max bytes retained per captured stderr line (bounds memory for a plugin that
/// emits pathologically long lines; the tail is diagnostics, not a full log).
const MAX_STDERR_LINE_LEN: usize = 512;
/// Truncate `line` to at most [`MAX_STDERR_LINE_LEN`] bytes on a char boundary,
/// appending an ellipsis when clipped. Keeps the ring buffer bounded.
fn clip_stderr_line(line: String) -> String {
if line.len() <= MAX_STDERR_LINE_LEN {
return line;
}
let mut end = MAX_STDERR_LINE_LEN;
while end > 0 && !line.is_char_boundary(end) {
end -= 1;
}
format!("{}…", &line[..end])
}
/// Matches a credential label anywhere in a stderr line, tolerating word breaks
/// (`api key`, `api_key`, `api-key`) and any following separator/spacing. Once
/// matched, everything from the label to end-of-line is masked, so the value is
/// redacted regardless of how it is delimited or whether it spans tokens (PEM
/// blob, JSON, `password = hunter2`, `Authorization: Bearer <tok>`).
fn secret_marker_regex() -> &'static regex::Regex {
static RE: std::sync::OnceLock<regex::Regex> = std::sync::OnceLock::new();
RE.get_or_init(|| {
regex::Regex::new(
r"(?i)(pass(?:word|wd)?|pwd|secret|token|api[ _-]?key|access[ _-]?key|client[ _-]?secret|authorization|credential|private[ _-]?key|session[ _-]?key|bearer|basic)",
)
.expect("valid secret-marker regex")
})
}
/// Redact credentials from a stderr line before it is surfaced in a user-visible
/// error, so secrets stay in operator logs and do not leak into RPC/CLI errors.
///
/// Two passes: (1) mask the password of any `scheme://user:pass@host` connection
/// string (a Postgres plugin echoing its DATABASE_URL on a connect failure), even
/// when no credential *label* is present; (2) if a credential label (`password`,
/// `api key`, `Authorization`, `bearer`, ...) appears, keep the text up to and
/// including the label and mask everything after it. This over-redacts the tail of
/// such a line — the stderr tail is failure diagnostics, not a full log — but
/// never leaks the value regardless of separator, spacing, or token spanning.
fn redact_stderr_line(line: &str) -> String {
let url_redacted = redact_url_userinfo(line);
if let Some(m) = secret_marker_regex().find(&url_redacted) {
return format!("{} ***", url_redacted[..m.end()].trim_end());
}
url_redacted
}
/// Mask the password in any `scheme://user:pass@host` token: `user:***@host`.
/// Uses the LAST `@` so an unescaped `@` inside the password cannot leave a
/// suffix unmasked (`u:p@ss@host` -> `u:***@host`).
fn redact_url_userinfo(line: &str) -> String {
line.split(' ')
.map(|token| {
if let Some(scheme_end) = token.find("://") {
let after = &token[scheme_end + 3..];
if let Some(at) = after.rfind('@') {
let userinfo = &after[..at];
if let Some(colon) = userinfo.find(':') {
return format!("{}{}:***@{}", &token[..scheme_end + 3], &userinfo[..colon], &after[at + 1..]);
}
}
}
token.to_string()
})
.collect::<Vec<_>>()
.join(" ")
}
/// Deadline the [`PluginHost::shutdown_transport`] flow waits for a transport
/// plugin's `transport/shutdown` reply before moving on to the generic
/// shutdown. Spec-compliant transports drain in-flight requests during this
/// call; a misbehaving plugin must not block daemon teardown so the upper
/// bound is enforced here.
const TRANSPORT_SHUTDOWN_GRACE: Duration = Duration::from_secs(5);
/// JSON-RPC method name the host issues to ask a `transport_backend` plugin
/// to bind its external listener. Kept as a string constant so this crate
/// avoids a build-time dependency on `animus-transport-protocol`; the spec
/// freezes the literal at `transport/start` (see
/// `animus-transport-protocol::TRANSPORT_METHOD_START`).
pub const TRANSPORT_METHOD_START: &str = "transport/start";
/// JSON-RPC method name the host issues to ask a `transport_backend` plugin
/// to drain in-flight requests and release its bound address. Mirrors
/// `animus-transport-protocol::TRANSPORT_METHOD_SHUTDOWN`.
pub const TRANSPORT_METHOD_SHUTDOWN: &str = "transport/shutdown";
/// Structured plugin-host errors that benefit from being matched on by
/// callers. The supervisor pattern-matches on this enum to decide whether a
/// failure is death-like (retry-once safe) or a structured plugin-side error
/// (retry would just re-elicit). Constructing one of these at the point of
/// failure (vs coercing everything to `RpcError { code: INTERNAL_ERROR, ... }`
/// and parsing message substrings later) is the architectural fix shipped in
/// the typed-classifier refactor.
#[derive(Debug, Error)]
pub enum HostError {
/// The plugin advertised a `protocol_version` that the host cannot speak.
///
/// Major-version mismatch (or non-semver gibberish) trips this. The host
/// should quarantine the plugin and surface the message so users can see
/// which plugin is wedged.
#[error("incompatible plugin protocol: {0}")]
IncompatibleProtocol(String),
/// The plugin transport closed (or never opened) while an awaiter was
/// waiting for a response.
///
/// Surfaced when the child process exits, its stdout closes, or the
/// reader task observes a fatal I/O error. The host is no longer usable
/// after this error; the supervisor should respawn.
#[error("plugin connection lost")]
ConnectionLost,
/// A [`PluginHost::request_with_timeout`] call exceeded its deadline.
///
/// The pending awaiter is removed from the router map so any late
/// response from the plugin is silently discarded.
#[error("plugin request timed out after {0:?}")]
Timeout(Duration),
/// The plugin child process exited mid-request with a non-zero (or
/// known-fatal) status. Reserved for future use by callers that watch the
/// child's wait status directly; the in-tree dispatch path currently
/// observes process death indirectly via [`Self::ConnectionLost`] when
/// stdout closes.
#[error("plugin process exited: {0}")]
ProcessExited(String),
/// The plugin returned a structured JSON-RPC error frame in response to
/// a request. The plugin process is still alive; retrying would just
/// re-elicit the same error. The supervisor uses this distinction to
/// avoid wasting a restart budget on plugin-author bugs.
#[error("plugin returned RPC error {}: {}", .0.code, .0.message)]
Rpc(RpcError),
/// The plugin did not advertise the capability the host is trying to
/// invoke. Returned by higher-level callers (e.g. the session backend's
/// cancel routing) when the plugin's handshake-reported
/// [`PluginCapabilities`](animus_plugin_protocol::PluginCapabilities) does
/// not include the required feature.
///
/// Carries the capability name so callers can surface a useful message
/// (e.g. "plugin 'foo' does not advertise capability 'cancellation'").
#[error("plugin does not advertise capability: {0}")]
CapabilityNotSupported(String),
}
impl From<HostError> for RpcError {
fn from(err: HostError) -> Self {
match err {
HostError::Rpc(inner) => inner,
HostError::Timeout(duration) => {
RpcError { code: error_codes::TIMEOUT, message: HostError::Timeout(duration).to_string(), data: None }
}
other => RpcError { code: error_codes::INTERNAL_ERROR, message: other.to_string(), data: None },
}
}
}
/// Validate that a plugin's advertised `protocol_version` is wire-compatible
/// with the host's [`PROTOCOL_VERSION`].
///
/// Compatibility is gated by the semver major component. Plugins reporting a
/// matching major are accepted (minor/patch drift is treated as additive and
/// backwards-compatible). Plugins reporting a different major — or a
/// non-semver string — are rejected with [`HostError::IncompatibleProtocol`].
pub fn check_protocol_compat(plugin_version: &str) -> Result<(), HostError> {
let host: Version = PROTOCOL_VERSION
.parse()
.map_err(|err| HostError::IncompatibleProtocol(format!("host protocol version is not valid semver: {err}")))?;
let plugin: Version = plugin_version.parse().map_err(|_| {
HostError::IncompatibleProtocol(format!(
"plugin advertised non-semver protocol_version '{plugin_version}' (host speaks {PROTOCOL_VERSION})"
))
})?;
if plugin.major != host.major {
return Err(HostError::IncompatibleProtocol(format!(
"plugin protocol_version {plugin_version} incompatible with host {PROTOCOL_VERSION} (major version mismatch)"
)));
}
Ok(())
}
/// Sink for plugin stderr lines. Receives `(plugin_name, line)` on each stderr line.
pub type PluginStderrSink = Arc<dyn Fn(&str, &str) + Send + Sync>;
/// Caller-supplied options that drive how the plugin host spawns a plugin
/// process.
///
/// Use [`PluginSpawnOptions::for_manifest`] to derive an environment allowlist
/// from a plugin's [`PluginManifest::env_required`](animus_plugin_protocol::PluginManifest::env_required)
/// list. See [`PLUGIN_BASE_ENV_ALLOWLIST`] for the universally-forwarded vars.
#[derive(Default, Clone)]
pub struct PluginSpawnOptions {
/// Routes every stderr line through this sink in addition to the standard
/// `tracing::warn!` log. Useful for surfacing plugin diagnostics into a
/// project's structured events log.
pub stderr_sink: Option<PluginStderrSink>,
/// Names of environment variables the plugin is allowed to see. The host
/// always forwards [`PLUGIN_BASE_ENV_ALLOWLIST`] on top of this list.
/// Anything else is scrubbed.
pub env_allowlist: Vec<String>,
/// Plugin-name label used in any spawn-time warnings (e.g. missing
/// required env). When empty, the host falls back to the binary file name.
pub plugin_label: Option<String>,
/// Required-but-missing env variable names. The host emits a `warn!` for
/// each at spawn time so operators can see why the plugin will likely
/// fail.
pub missing_required_env: Vec<String>,
/// Optional override for the broadcast channel capacity used for plugin
/// notifications. When `Some`, it wins over both the manifest hint and
/// the env override. Used by tests; production callers typically leave
/// this `None` and rely on
/// [`PluginManifest::notification_buffer_size`] +
/// [`NOTIFICATION_BROADCAST_CAPACITY_ENV`].
///
/// [`PluginManifest::notification_buffer_size`]: animus_plugin_protocol::PluginManifest::notification_buffer_size
pub notification_capacity: Option<usize>,
/// The plugin manifest's declared
/// [`PluginManifest::notification_buffer_size`] hint. Lower precedence
/// than `notification_capacity`, higher than the env override and the
/// compiled default. Set via [`PluginSpawnOptions::with_notification_buffer_hint`]
/// by callers that hold the plugin's manifest at spawn time.
///
/// [`PluginManifest::notification_buffer_size`]: animus_plugin_protocol::PluginManifest::notification_buffer_size
pub notification_buffer_hint: Option<usize>,
/// Optional working directory for the spawned plugin process. When set,
/// the host pins the child's cwd here instead of inheriting the
/// caller's cwd. Subject-backend and provider plugins use cwd-relative
/// paths for their on-disk state (e.g. `.animus/subjects/tasks.db`), so
/// the daemon must pin cwd to `--project-root` rather than letting it
/// depend on which shell happened to start the daemon. Leave `None`
/// when the plugin has no cwd-relative state — the spawn then inherits
/// the parent's cwd, matching pre-fix behavior.
pub working_dir: Option<PathBuf>,
}
impl PluginSpawnOptions {
/// Build options for a plugin whose manifest declares the supplied env
/// requirements. Returns the assembled options and a list of declared-as
/// `required = true` vars that are not currently set in the host process.
///
/// The returned options force the spawn to scrub the daemon's environment
/// to [`PLUGIN_BASE_ENV_ALLOWLIST`] plus the manifest's declared variables
/// plus any explicit `extra` names supplied by the caller (e.g. one-off
/// runtime overrides).
pub fn for_manifest(
plugin_label: impl Into<String>,
env_required: &[EnvRequirement],
extra_env_vars: impl IntoIterator<Item = String>,
stderr_sink: Option<PluginStderrSink>,
) -> Self {
let plugin_label = plugin_label.into();
let mut allow: BTreeSet<String> = env_required.iter().map(|requirement| requirement.name.clone()).collect();
allow.extend(extra_env_vars);
let missing_required: Vec<String> = env_required
.iter()
.filter(|requirement| requirement.required)
.filter(|requirement| std::env::var_os(&requirement.name).is_none())
.map(|requirement| requirement.name.clone())
.collect();
Self {
stderr_sink,
env_allowlist: allow.into_iter().collect(),
plugin_label: if plugin_label.is_empty() { None } else { Some(plugin_label) },
missing_required_env: missing_required,
notification_capacity: None,
notification_buffer_hint: None,
working_dir: None,
}
}
/// Carry the plugin manifest's `notification_buffer_size` hint into the
/// spawn so [`resolve_broadcast_capacity`]'s documented priority chain
/// (explicit override → manifest hint → env override → default) actually
/// sees the manifest value.
#[must_use]
pub fn with_notification_buffer_hint(mut self, hint: Option<usize>) -> Self {
self.notification_buffer_hint = hint;
self
}
/// Pin the spawned plugin's working directory. Used for subject-backend
/// and provider plugins so their cwd-relative state paths
/// (e.g. `.animus/subjects/tasks.db`) resolve under the project root
/// rather than under whatever cwd the daemon happened to be started
/// from.
#[must_use]
pub fn with_working_dir(mut self, dir: impl Into<PathBuf>) -> Self {
self.working_dir = Some(dir.into());
self
}
}
/// Receiver for plugin-emitted JSON-RPC notifications (frames without `id`).
///
/// Returned by [`PluginHost::subscribe_notifications`]. Each subscriber gets
/// an independent receiver fed by the host's single-reader router task; a
/// slow subscriber observes [`broadcast::error::RecvError::Lagged`] rather
/// than backpressuring the request path.
pub type PluginNotificationRx = broadcast::Receiver<RpcNotification>;
/// Choose the notification broadcast capacity for a plugin host using the
/// documented priority: explicit option override → plugin manifest hint →
/// env override → compiled default. Always returns a non-zero capacity (a
/// `broadcast::channel` with capacity 0 panics).
pub(crate) fn resolve_broadcast_capacity(spawn_override: Option<usize>, manifest_hint: Option<usize>) -> usize {
if let Some(cap) = spawn_override {
if cap > 0 {
return cap;
}
}
if let Some(cap) = manifest_hint {
if cap > 0 {
return cap;
}
}
if let Ok(raw) = std::env::var(NOTIFICATION_BROADCAST_CAPACITY_ENV) {
if let Ok(cap) = raw.trim().parse::<usize>() {
if cap > 0 {
return cap;
}
}
}
DEFAULT_NOTIFICATION_BROADCAST_CAPACITY
}
/// Opaque RAII guard returned by [`ProcessSlotFactory::acquire`]. Dropping it
/// must release the underlying quota slot. The plugin host holds one of these
/// alongside the spawned child for the child's lifetime, so a slot is held for
/// exactly the same duration as the live plugin process.
///
/// The marker trait is intentionally empty: the only behaviour the host cares
/// about is `Drop`. Implementors typically wrap a concrete RAII type owned by
/// the quota module (e.g. `orchestrator_daemon_runtime::PluginProcessSlot`).
pub trait ProcessSlotGuard: Send + Sync + std::fmt::Debug {}
/// Boxed trait object alias used everywhere the host stores or returns a slot.
pub type BoxedProcessSlotGuard = Box<dyn ProcessSlotGuard>;
/// Structured error returned by [`ProcessSlotFactory::acquire`] when the
/// configured per-process plugin cap is at its limit. The host translates this
/// into an `anyhow::Error` at the spawn site so callers see a single error
/// type from `spawn_with_options`.
#[derive(Debug, Clone)]
pub struct ProcessSlotError {
/// Currently-live plugin process count as observed by the factory.
pub current: usize,
/// Configured cap (e.g. `RuntimeQuotas::plugin_process_max`).
pub cap: usize,
/// Human-readable diagnostic appended by the factory (often the factory's
/// own `Display` formatting of its native error type).
pub message: String,
}
impl std::fmt::Display for ProcessSlotError {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(f, "{}", self.message)
}
}
impl std::error::Error for ProcessSlotError {}
/// Quota-enforcement boundary the plugin host uses at the spawn site.
///
/// The plugin-host crate intentionally does NOT depend on
/// `orchestrator-daemon-runtime` (that crate already depends on this one;
/// adding a reverse dep would form a cycle). Instead the daemon installs an
/// implementation of this trait at startup via [`install_process_slot_factory`].
/// When no factory is installed, the host falls back to a no-op slot and
/// behaviour is identical to pre-quota releases (used by unit tests and any
/// embedder that hasn't opted in).
pub trait ProcessSlotFactory: Send + Sync + 'static {
/// Try to claim a slot. Returns `Err` if the cap is reached; the host
/// surfaces this as a spawn failure rather than queuing or blocking.
fn acquire(&self) -> Result<BoxedProcessSlotGuard, ProcessSlotError>;
}
/// Lazy-init container for the process-wide factory. Production daemon
/// startup installs exactly once; tests may swap via
/// [`install_process_slot_factory_for_test`] under a serializing mutex.
fn process_slot_factory_slot() -> &'static RwLock<Option<Arc<dyn ProcessSlotFactory>>> {
static SLOT: OnceLock<RwLock<Option<Arc<dyn ProcessSlotFactory>>>> = OnceLock::new();
SLOT.get_or_init(|| RwLock::new(None))
}
/// Install the process-wide [`ProcessSlotFactory`]. First-installer-wins:
/// subsequent calls return `false` and leave the existing factory in place so
/// a test that pre-installed a stub keeps its override even if the daemon
/// startup path also runs.
pub fn install_process_slot_factory(factory: Arc<dyn ProcessSlotFactory>) -> bool {
let mut guard = process_slot_factory_slot().write().expect("process slot factory lock poisoned");
if guard.is_some() {
return false;
}
*guard = Some(factory);
true
}
/// Test-only: unconditionally replace the installed factory. Production code
/// must never call this; the daemon startup path uses
/// [`install_process_slot_factory`] which is first-installer-wins.
#[cfg(any(test, feature = "test-support"))]
pub fn install_process_slot_factory_for_test(factory: Arc<dyn ProcessSlotFactory>) {
let mut guard = process_slot_factory_slot().write().expect("process slot factory lock poisoned");
*guard = Some(factory);
}
/// Test-only: clear the installed factory so the spawn path falls back to
/// the no-quota path.
#[cfg(any(test, feature = "test-support"))]
pub fn clear_process_slot_factory_for_test() {
let mut guard = process_slot_factory_slot().write().expect("process slot factory lock poisoned");
*guard = None;
}
/// Snapshot of the currently-installed factory. Cloned `Arc` so the caller
/// doesn't hold the lock across an `.acquire()` call.
fn current_process_slot_factory() -> Option<Arc<dyn ProcessSlotFactory>> {
process_slot_factory_slot().read().expect("process slot factory lock poisoned").clone()
}
/// Process-wide hook that supplies the keychain-backed secret snapshot the
/// spawn path merges into each plugin's child environment.
///
/// Decoupled from `orchestrator-core` so this crate stays dependency-free
/// w.r.t. the secret store: the daemon installs a real implementation at
/// startup, tests may install a mock, and any embedder that hasn't opted
/// in gets the historical "no extra env" behaviour.
pub trait SecretSnapshotProvider: Send + Sync + 'static {
/// Return the (KEY, VALUE) pairs the host should merge into the
/// next-spawned plugin's environment, **before** the caller's process
/// environment is applied. Existing parent-process env wins on
/// collision so explicit `KEY=val animus daemon start` overrides the
/// keychain entry.
///
/// Implementations MUST cap their own output at
/// `orchestrator_core::MAX_INJECTED_ENV_BYTES` bytes cumulative
/// (sum of key.len() + value.len()) and return the empty map on any
/// error so a broken keychain never blocks plugin spawn.
fn snapshot(&self) -> std::collections::BTreeMap<String, String>;
/// Return only the entries whose KEYs appear in `requested`. Default
/// impl scans the full snapshot; production providers SHOULD
/// override this to avoid touching keychain items the caller does
/// not need (so a plugin with no `env_required` block never causes
/// the OS to prompt for unrelated secrets). (codex round-3 P2.)
fn snapshot_filtered(&self, requested: &[String]) -> std::collections::BTreeMap<String, String> {
if requested.is_empty() {
return std::collections::BTreeMap::new();
}
let mut all = self.snapshot();
all.retain(|k, _| requested.iter().any(|r| r == k));
all
}
}
fn secret_snapshot_provider_slot() -> &'static RwLock<Option<Arc<dyn SecretSnapshotProvider>>> {
static SLOT: OnceLock<RwLock<Option<Arc<dyn SecretSnapshotProvider>>>> = OnceLock::new();
SLOT.get_or_init(|| RwLock::new(None))
}
/// Install the process-wide [`SecretSnapshotProvider`]. First-installer-wins
/// to match [`install_process_slot_factory`] semantics; tests use
/// [`install_secret_snapshot_provider_for_test`] to swap unconditionally.
pub fn install_secret_snapshot_provider(provider: Arc<dyn SecretSnapshotProvider>) -> bool {
let mut guard = secret_snapshot_provider_slot().write().expect("secret snapshot provider lock poisoned");
if guard.is_some() {
return false;
}
*guard = Some(provider);
true
}
/// Test-only: unconditionally replace the installed provider.
#[cfg(any(test, feature = "test-support"))]
pub fn install_secret_snapshot_provider_for_test(provider: Arc<dyn SecretSnapshotProvider>) {
let mut guard = secret_snapshot_provider_slot().write().expect("secret snapshot provider lock poisoned");
*guard = Some(provider);
}
/// Test-only: clear the installed provider so the spawn path skips the
/// keychain merge entirely.
#[cfg(any(test, feature = "test-support"))]
pub fn clear_secret_snapshot_provider_for_test() {
let mut guard = secret_snapshot_provider_slot().write().expect("secret snapshot provider lock poisoned");
*guard = None;
}
/// Snapshot of the currently-installed [`SecretSnapshotProvider`], if
/// any. Exposed so out-of-tree subprocess spawn paths (e.g. the daemon's
/// `ProcessManager` workflow-runner spawn) can merge keychain entries
/// into their own command env without going through
/// [`PluginHost::spawn_with_options`].
pub fn current_secret_snapshot_provider() -> Option<Arc<dyn SecretSnapshotProvider>> {
secret_snapshot_provider_slot().read().expect("secret snapshot provider lock poisoned").clone()
}
/// Shared inner state for a [`PluginHost`]. One per spawned plugin process.
///
/// The host follows the single-reader-router pattern: one tokio task owns
/// the transport's read half and demultiplexes inbound frames. Frames with
/// an `id` field route to the pending-map awaiter via a oneshot channel;
/// frames without an `id` fan out via [`broadcast`] to every subscriber.
/// Writes go through `transport_write` so concurrent `request()` calls
/// serialize cleanly on the line-delimited wire.
pub struct PluginHostInner {
/// Human-readable plugin name, used in log messages and shutdown.
pub name: String,
/// Locked write half of the stdio transport. Concurrent senders
/// interleave one frame at a time.
transport_write: Mutex<Box<dyn AsyncWrite + Send + Unpin>>,
/// Pending request awaiters keyed by JSON-RPC id. Populated by
/// `request()` / `request_with_timeout()`, drained by the reader task
/// (or by the host itself on shutdown).
pending: Mutex<HashMap<u64, oneshot::Sender<RpcResponse>>>,
/// Sender owned by the reader task; subscribers come and go via
/// [`PluginHost::subscribe_notifications`].
notifications_tx: broadcast::Sender<RpcNotification>,
/// Monotonic JSON-RPC id allocator. We allocate from `1` so a freshly
/// constructed host doesn't collide with the spec's "null id" sentinel.
next_id: AtomicU64,
/// The plugin child process. Owned so [`PluginHost::shutdown`] can kill
/// it if `shutdown` RPC times out. `None` for hosts constructed from
/// in-memory pipes (tests).
child: Mutex<Option<Child>>,
/// Reader task handle. `Some` until [`PluginHost::shutdown`] reaps it.
/// Held under a sync mutex so [`PluginHost::launch`] can stash it
/// immediately (no awaits) before returning the host to callers.
reader_handle: std::sync::Mutex<Option<JoinHandle<()>>>,
/// Flips to `false` when the reader task exits (EOF, fatal error, or
/// shutdown). New requests issued after this point short-circuit with
/// [`HostError::ConnectionLost`] instead of inserting an awaiter that
/// would never be answered.
alive: AtomicBool,
/// Process-quota RAII guard acquired at spawn time. Held for the lifetime
/// of the host (and therefore the child); dropped when the `Arc<...Inner>`
/// goes away, which is after [`PluginHost::shutdown`] has reaped the
/// child. `None` for tests / embedders that haven't installed a
/// [`ProcessSlotFactory`].
///
/// Held inside a mutex purely so [`PluginHost::shutdown`] can take the
/// guard and drop it eagerly after the child wait completes, ahead of the
/// last `Arc` drop. In steady state nothing else touches this field.
_process_slot: std::sync::Mutex<Option<BoxedProcessSlotGuard>>,
/// Ring buffer of the plugin's most recent stderr lines (last
/// [`STDERR_TAIL_CAP`]), captured by the stderr reader task. Surfaced in
/// handshake / ConnectionLost errors so a plugin that dies during startup
/// reports WHY (its stderr) instead of an opaque "connection lost". Empty
/// for in-memory test hosts.
stderr_tail: Arc<std::sync::Mutex<std::collections::VecDeque<String>>>,
}
/// Single-process JSON-RPC plugin host.
///
/// Cloning a [`PluginHost`] hands out another shared reference to the same
/// underlying transport — all methods take `&self` and may be called
/// concurrently. The router task is single-reader; writes are serialized
/// through an internal mutex so frames stay intact on the wire.
///
/// Construct one via [`PluginHost::spawn_with_options`] for a real child
/// process or [`PluginHost::from_streams`] for in-memory tests.
#[derive(Clone)]
pub struct PluginHost {
inner: Arc<PluginHostInner>,
}
impl PluginHost {
/// Spawn a plugin without forwarding any environment beyond
/// [`PLUGIN_BASE_ENV_ALLOWLIST`]. Most production callers should use
/// [`PluginHost::spawn_with_options`] instead so the plugin sees the
/// env it declared in its manifest.
pub async fn spawn(binary_path: &Path, args: &[&str]) -> Result<Self> {
Self::spawn_with_options(binary_path, args, PluginSpawnOptions::default()).await
}
/// Spawn a plugin and route every stderr line through the supplied sink in addition
/// to the standard `tracing::warn!` log. Use this from the host runtime so plugin
/// diagnostics land in the project's structured `events.jsonl`.
///
/// Note: this convenience does not forward any plugin-specific env vars.
/// Prefer [`PluginHost::spawn_with_options`] (with options built via
/// [`PluginSpawnOptions::for_manifest`]) for production spawns so the
/// plugin's manifest-declared environment is honored.
pub async fn spawn_with_stderr(
binary_path: &Path,
args: &[&str],
stderr_sink: Option<PluginStderrSink>,
) -> Result<Self> {
let options = PluginSpawnOptions { stderr_sink, ..PluginSpawnOptions::default() };
Self::spawn_with_options(binary_path, args, options).await
}
/// Spawn a plugin under the supplied [`PluginSpawnOptions`].
///
/// The host always calls `env_clear()` on the child process and forwards
/// only the union of [`PLUGIN_BASE_ENV_ALLOWLIST`] and
/// `options.env_allowlist`. This is the v0.4.x trust boundary: plugins
/// only see secrets they explicitly declared in their manifest.
pub async fn spawn_with_options(binary_path: &Path, args: &[&str], options: PluginSpawnOptions) -> Result<Self> {
let binary_name = binary_path.file_name().and_then(|value| value.to_str()).unwrap_or("plugin").to_string();
let name = options.plugin_label.clone().unwrap_or_else(|| binary_name.clone());
// Quota check BEFORE the fork: if the daemon has installed a
// ProcessSlotFactory and the per-process cap is reached, refuse
// the spawn instead of letting the fd/memory pressure build. The
// slot is held alongside the child for the rest of its lifetime;
// dropping it (in shutdown or when the Arc<...Inner> goes away)
// releases capacity for the next spawn.
let process_slot = match current_process_slot_factory() {
Some(factory) => Some(factory.acquire().map_err(|err| {
warn!(plugin = %name, error = %err, "refused plugin spawn: process slot cap reached");
anyhow!("{err}")
})?),
None => None,
};
let mut command = tokio::process::Command::new(binary_path);
command
.args(args)
.kill_on_drop(true)
.stdin(std::process::Stdio::piped())
.stdout(std::process::Stdio::piped())
.stderr(std::process::Stdio::piped());
if let Some(working_dir) = options.working_dir.as_ref() {
command.current_dir(working_dir);
}
// Build the allowlist: universal base + caller-declared. Deduplicate
// case-sensitively (env var names are case-sensitive on POSIX).
let mut allow: BTreeSet<&str> = PLUGIN_BASE_ENV_ALLOWLIST.iter().copied().collect();
for var in &options.env_allowlist {
allow.insert(var.as_str());
}
command.env_clear();
// Precedence (lowest to highest): keychain entries -> parent env.
// Applying keychain first lets explicit `KEY=val animus daemon start`
// still win, matching the documented contract in
// `docs/reference/secrets.md`.
let mut injected_keys: BTreeSet<String> = BTreeSet::new();
if let Some(provider) = current_secret_snapshot_provider() {
// Request only the secrets the plugin's manifest declares
// AND that are not already satisfied by the parent
// environment. The latter avoids touching the keychain for
// entries an explicit `KEY=val animus daemon start` will
// overwrite anyway — important on locked-keychain
// platforms that prompt on access. (codex round-7 P2.)
let requested: Vec<String> = options
.env_allowlist
.iter()
.filter(|name| std::env::var_os(name.as_str()).is_none())
.cloned()
.collect();
let snapshot = if requested.is_empty() {
std::collections::BTreeMap::new()
} else {
provider.snapshot_filtered(&requested)
};
let mut total: usize = 0;
let mut injected = 0usize;
let mut skipped = 0usize;
for (key, value) in snapshot {
let next = total.saturating_add(key.len()).saturating_add(value.len());
if next > MAX_INJECTED_SECRET_BYTES {
skipped += 1;
warn!(
plugin = %name,
skipped_key = %key,
"secret entry skipped: would exceed {MAX_INJECTED_SECRET_BYTES}-byte cumulative cap"
);
continue;
}
command.env(&key, value);
injected_keys.insert(key);
total = next;
injected += 1;
}
if injected > 0 || skipped > 0 {
debug!(plugin = %name, injected, skipped, "merged keychain-backed secrets into plugin env");
}
}
for var in &allow {
if let Some(value) = std::env::var_os(var) {
command.env(var, value);
}
}
// Bound each plugin's tokio runtime. A bare `#[tokio::main]` (every Animus
// stdio plugin) sizes its multi-thread worker pool to
// `available_parallelism()` — all CPU cores. With v0.6's resident-plugin
// fleet (config_source + subject backends + queue + workflow_runner +
// providers + transport) that is hundreds of threads on a many-core host,
// exhausting the PID/thread budget so new forks — including the provider CLI
// an agent phase spawns — fail with EAGAIN and the run hangs. Plugins are
// I/O-bound stdio RPC servers, so a tiny pool is sufficient. `env_clear()`
// above dropped any inherited value (TOKIO_WORKER_THREADS is not in the base
// allowlist), so set it explicitly here, honoring an operator override on the
// daemon env so a deploy can still tune it up or down.
command.env(
"TOKIO_WORKER_THREADS",
std::env::var_os("TOKIO_WORKER_THREADS").unwrap_or_else(|| std::ffi::OsString::from("2")),
);
for missing in &options.missing_required_env {
// Suppress the warning when the keychain already satisfied
// the requirement during the snapshot merge above. The
// `missing_required_env` list was computed at
// `for_manifest` time from `std::env` only, so without this
// check every successful keychain-backed spawn would print
// a spurious "plugin will likely fail" warning.
// (codex round-2 P3.)
if injected_keys.contains(missing) {
continue;
}
warn!(
plugin = %name,
env_var = %missing,
"plugin declared env_required={{name={missing}, required=true}} but the host environment does not have it set; the plugin will likely fail to start"
);
}
let mut child = command.spawn()?;
let stdin = child.stdin.take().ok_or_else(|| anyhow!("failed to take plugin stdin"))?;
let stdout = child.stdout.take().ok_or_else(|| anyhow!("failed to take plugin stdout"))?;
let stderr = child.stderr.take().ok_or_else(|| anyhow!("failed to take plugin stderr"))?;
let stderr_plugin_name = name.clone();
let stderr_sink = options.stderr_sink.clone();
let capacity = resolve_broadcast_capacity(options.notification_capacity, options.notification_buffer_hint);
let host = Self::launch_with_slot(name, Box::new(stdout), Box::new(stdin), Some(child), capacity, process_slot);
// Capture the plugin's stderr into a bounded ring buffer (in addition to
// the standard warn! + optional sink) so a startup/handshake failure can
// report the plugin's own last words instead of a bare "connection lost".
let stderr_tail = host.inner.stderr_tail.clone();
tokio::spawn(async move {
let mut lines = tokio::io::BufReader::new(stderr).lines();
while let Ok(Some(line)) = lines.next_line().await {
warn!(plugin = %stderr_plugin_name, "{}", line);
if let Some(sink) = stderr_sink.as_ref() {
sink(&stderr_plugin_name, &line);
}
if let Ok(mut buf) = stderr_tail.lock() {
if buf.len() >= STDERR_TAIL_CAP {
buf.pop_front();
}
// Redact BEFORE clipping: clipping first could split a
// `scheme://user:pass@host` across the byte cutoff and defeat
// the URL detector, leaking the password prefix. The raw line
// still reaches operator logs above via `warn!`/`sink`.
buf.push_back(clip_stderr_line(redact_stderr_line(&line)));
}
}
});
Ok(host)
}
/// Build a host from caller-supplied in-memory streams. Used by tests
/// that script a plugin in-process without spawning a real binary.
///
/// The reader and writer are boxed and erased; the resulting
/// [`PluginHost`] is identical in behavior to a spawned-process host.
pub fn from_streams<R, W>(name: impl Into<String>, reader: R, writer: W) -> Self
where
R: AsyncRead + Send + Unpin + 'static,
W: AsyncWrite + Send + Unpin + 'static,
{
Self::launch(name.into(), Box::new(reader), Box::new(writer), None, DEFAULT_NOTIFICATION_BROADCAST_CAPACITY)
}
/// Build a host from in-memory streams with an explicit broadcast
/// capacity override. Convenience for tests that need to exercise the
/// `Lagged` path.
pub fn from_streams_with_capacity<R, W>(name: impl Into<String>, reader: R, writer: W, capacity: usize) -> Self
where
R: AsyncRead + Send + Unpin + 'static,
W: AsyncWrite + Send + Unpin + 'static,
{
let capacity = capacity.max(1);
Self::launch(name.into(), Box::new(reader), Box::new(writer), None, capacity)
}
/// Internal hot-path constructor: wires up the pending-map, broadcast
/// channel, and reader task in one place so both `spawn_with_options`
/// and `from_streams` produce the same shape of host.
fn launch(
name: String,
reader: Box<dyn AsyncRead + Send + Unpin>,
writer: Box<dyn AsyncWrite + Send + Unpin>,
child: Option<Child>,
notification_capacity: usize,
) -> Self {
Self::launch_with_slot(name, reader, writer, child, notification_capacity, None)
}
/// Test-only: build an in-memory host with an explicit reader buffer cap
/// so the overflow teardown path can be exercised without pushing the
/// production [`READER_BUFFER_CAP`] worth of bytes through a duplex.
#[cfg(test)]
fn from_streams_with_reader_buffer_cap<R, W>(name: impl Into<String>, reader: R, writer: W, cap: usize) -> Self
where
R: AsyncRead + Send + Unpin + 'static,
W: AsyncWrite + Send + Unpin + 'static,
{
Self::launch_full(
name.into(),
Box::new(reader),
Box::new(writer),
None,
DEFAULT_NOTIFICATION_BROADCAST_CAPACITY,
None,
cap,
)
}
/// Variant of [`Self::launch`] that also stashes the process-quota slot
/// alongside the child. Only `spawn_with_options` calls this with a
/// `Some` slot; in-memory stream constructors pass `None`.
fn launch_with_slot(
name: String,
reader: Box<dyn AsyncRead + Send + Unpin>,
writer: Box<dyn AsyncWrite + Send + Unpin>,
child: Option<Child>,
notification_capacity: usize,
process_slot: Option<BoxedProcessSlotGuard>,
) -> Self {
Self::launch_full(name, reader, writer, child, notification_capacity, process_slot, READER_BUFFER_CAP)
}
#[allow(clippy::too_many_arguments)]
fn launch_full(
name: String,
reader: Box<dyn AsyncRead + Send + Unpin>,
writer: Box<dyn AsyncWrite + Send + Unpin>,
child: Option<Child>,
notification_capacity: usize,
process_slot: Option<BoxedProcessSlotGuard>,
reader_buffer_cap: usize,
) -> Self {
let (notifications_tx, _) = broadcast::channel::<RpcNotification>(notification_capacity);
let inner = Arc::new(PluginHostInner {
name,
transport_write: Mutex::new(writer),
pending: Mutex::new(HashMap::new()),
notifications_tx: notifications_tx.clone(),
next_id: AtomicU64::new(1),
child: Mutex::new(child),
reader_handle: std::sync::Mutex::new(None),
alive: AtomicBool::new(true),
_process_slot: std::sync::Mutex::new(process_slot),
stderr_tail: Arc::new(std::sync::Mutex::new(std::collections::VecDeque::new())),
});
let reader_inner = inner.clone();
let handle = tokio::spawn(reader_loop(reader, reader_inner, notifications_tx, reader_buffer_cap));
// Stash the handle synchronously so shutdown() can find it without
// racing the spawn that owns the reader loop.
*inner.reader_handle.lock().expect("reader_handle mutex poisoned at launch") = Some(handle);
Self { inner }
}
/// Plugin name (label) — same as the `name` field passed to spawn.
/// Return the OS process id of the spawned child, if one is owned by
/// this host. Returns `None` for in-memory hosts constructed by tests
/// via [`Self::from_streams`], or when the child mutex is currently
/// held by another task (so the synchronous accessor never blocks).
pub fn child_pid(&self) -> Option<u32> {
let guard = self.inner.child.try_lock().ok()?;
guard.as_ref().and_then(|child| child.id())
}
pub fn name(&self) -> &str {
&self.inner.name
}
/// Subscribe to JSON-RPC notifications (frames with no `id`) emitted by
/// the plugin. Each call returns an independent receiver fed by the
/// shared broadcast channel; subscribers are responsible for keeping up
/// (and observing `Lagged` if they don't).
pub fn subscribe_notifications(&self) -> PluginNotificationRx {
self.inner.notifications_tx.subscribe()
}
/// The next request id this host will allocate. Useful for tests; not
/// part of the steady-state API.
pub fn next_request_id(&self) -> u64 {
self.inner.next_id.load(Ordering::Relaxed)
}
/// Send a JSON-RPC request and await its response.
///
/// Multiple concurrent calls share the transport but each gets its own
/// pending-map entry; they multiplex independently.
///
/// This is the legacy-shape API (`Result<Value, RpcError>`) preserved for
/// callers that don't care about the structural distinction between
/// process-death and a plugin-side error. New callers should prefer
/// [`PluginHost::request_typed`], which returns the typed [`HostError`]
/// enum so the supervisor can pattern-match instead of parsing message
/// substrings.
pub async fn request(&self, method: impl Into<String>, params: Option<Value>) -> Result<Value, RpcError> {
self.request_typed(method, params).await.map_err(RpcError::from)
}
/// Typed variant of [`PluginHost::request`]: surfaces process-death
/// (`HostError::ConnectionLost`) and plugin-side RPC errors
/// (`HostError::Rpc(_)`) as distinct enum variants. The dispatcher
/// classifier in `orchestrator-session-host` matches on this enum to
/// decide whether a retry-once is safe.
pub async fn request_typed(&self, method: impl Into<String>, params: Option<Value>) -> Result<Value, HostError> {
let method = method.into();
let response = self.request_raw(&method, params).await?;
match response.error {
Some(error) => Err(HostError::Rpc(error)),
None => Ok(response.result.unwrap_or(Value::Null)),
}
}
/// Same as [`PluginHost::request`] but bails with [`HostError::Timeout`]
/// if the plugin doesn't respond within `timeout`. The pending awaiter
/// is removed from the router map so any late response is silently