From 550e50f58d0e3236217531acb8f90b5e4f583632 Mon Sep 17 00:00:00 2001 From: clux Date: Wed, 5 Feb 2025 23:59:53 +0000 Subject: [PATCH 01/16] Experiment with multi_reflectors since the question has showed up a few times. ultimately this is awkward because the hard generic use in ObjectRef and in Store which ties them to a particular resource. It feels like it shouldn't have to be this way because the Store::get takes an ObjectRef, which feels dynamically typed, but actually isnt. Tried out a couple of potentials and had to just resort to the dumb thing instead. Maybe there's some other things we can do for dynamics. Maybe worth exploring in an issue? Signed-off-by: clux --- examples/Cargo.toml | 4 ++ examples/multi_reflector.rs | 137 ++++++++++++++++++++++++++++++++++++ 2 files changed, 141 insertions(+) create mode 100644 examples/multi_reflector.rs diff --git a/examples/Cargo.toml b/examples/Cargo.toml index 3abb06521..1c7740f1c 100644 --- a/examples/Cargo.toml +++ b/examples/Cargo.toml @@ -130,6 +130,10 @@ path = "log_stream.rs" name = "multi_watcher" path = "multi_watcher.rs" +[[example]] +name = "multi_reflector" +path = "multi_reflector.rs" + [[example]] name = "pod_api" path = "pod_api.rs" diff --git a/examples/multi_reflector.rs b/examples/multi_reflector.rs new file mode 100644 index 000000000..9f3c4537d --- /dev/null +++ b/examples/multi_reflector.rs @@ -0,0 +1,137 @@ +use futures::{future, StreamExt}; +use k8s_openapi::api::{ + apps::v1::Deployment, + core::v1::{ConfigMap, Secret}, +}; +use kube::{ + runtime::{ + reflector, + reflector::{ObjectRef, Store}, + watcher, WatchStreamExt, + }, + Api, Client, +}; +use std::sync::Arc; +use tracing::*; + +// This does not work because Resource trait is not dyn safe. +/* +use std::any::TypeId; +use std::collections::HashMap; +use k8s_openapi::NamespaceResourceScope; +use kube::api::{Resource, ResourceExt}; +struct MultiStore { + stores: HashMap>>, + } +impl MultiStore { + fn get>(&self, name: &str, ns: &str) -> Option> { + let oref = ObjectRef::::new(name).within(ns); + if let Some(store) = self.stores.get(&TypeId::of::()) { + store.get(oref) + } else { + None + } + } +}*/ + +// explicit store can work +struct MultiStore { + deploys: Store, + cms: Store, + secs: Store, +} +// but using generics to help out won't because the K needs to be concretised +/* +impl MultiStore { + fn get>(&self, name: &str, ns: &str) -> Option>> { + let oref = ObjectRef::::new(name).within(ns); + let kind = K::kind(&()).to_owned(); + match kind.as_ref() { + "Deployment" => self.deploys.get(&ObjectRef::new(name).within(ns)), + "ConfigMap" => self.cms.get(&ObjectRef::new(name).within(ns)), + "Secret" => self.secs.get(&ObjectRef::new(name).within(ns)), + _ => None, + } + None + } +} +*/ +// so left with this + +impl MultiStore { + fn get_deploy(&self, name: &str, ns: &str) -> Option> { + self.deploys.get(&ObjectRef::::new(name).within(ns)) + } + + fn get_secret(&self, name: &str, ns: &str) -> Option> { + self.secs.get(&ObjectRef::::new(name).within(ns)) + } + + fn get_cm(&self, name: &str, ns: &str) -> Option> { + self.cms.get(&ObjectRef::::new(name).within(ns)) + } +} + +#[tokio::main] +async fn main() -> anyhow::Result<()> { + tracing_subscriber::fmt::init(); + let client = Client::try_default().await?; + + let deploys: Api = Api::default_namespaced(client.clone()); + let cms: Api = Api::default_namespaced(client.clone()); + let secret: Api = Api::default_namespaced(client.clone()); + + let (dep_reader, dep_writer) = reflector::store::(); + let (cm_reader, cm_writer) = reflector::store::(); + let (sec_reader, sec_writer) = reflector::store::(); + + let cfg = watcher::Config::default(); + let dep_watcher = watcher(deploys, cfg.clone()) + .reflect(dep_writer) + .applied_objects() + .for_each(|_| future::ready(())); + let cm_watcher = watcher(cms, cfg.clone()) + .reflect(cm_writer) + .applied_objects() + .for_each(|_| future::ready(())); + let sec_watcher = watcher(secret, cfg) + .reflect(sec_writer) + .applied_objects() + .for_each(|_| future::ready(())); + // poll these forever + + // multistore + let stores = MultiStore { + deploys: dep_reader, + cms: cm_reader, + secs: sec_reader, + }; + + // simulate doing stuff with the stores from some other thread + tokio::spawn(async move { + // Show state every 5 seconds of watching + info!("waiting for them to be ready"); + stores.deploys.wait_until_ready().await.unwrap(); + stores.cms.wait_until_ready().await.unwrap(); + stores.secs.wait_until_ready().await.unwrap(); + info!("stores initialised"); + // can use helper accessors + info!( + "common cm: {:?}", + stores.get_cm("kube-root-ca.crt", "kube-system").unwrap() + ); + loop { + tokio::time::sleep(std::time::Duration::from_secs(5)).await; + // access individual sub stores + info!("Current deploys count: {}", stores.deploys.state().len()); + } + }); + // info!("long watches starting"); + tokio::select! { + r = dep_watcher => println!("dep watcher exit: {r:?}"), + r = cm_watcher => println!("cm watcher exit: {r:?}"), + r = sec_watcher => println!("sec watcher exit: {r:?}"), + } + + Ok(()) +} From a4035ee632e021895eb9cc845d05da81261ff77b Mon Sep 17 00:00:00 2001 From: Danil-Grigorev Date: Mon, 10 Feb 2025 16:53:59 +0100 Subject: [PATCH 02/16] Exploring shared cache based on multi reflectors Signed-off-by: Danil-Grigorev --- examples/Cargo.toml | 1 + examples/multi_reflector.rs | 192 +++++++++++++++------------- kube-core/src/gvk.rs | 2 +- kube-core/src/lib.rs | 2 +- kube-core/src/metadata.rs | 137 +++++++++++++++++++- kube-core/src/object.rs | 51 +++++++- kube-core/src/resource.rs | 7 + kube-runtime/src/controller/mod.rs | 4 +- kube-runtime/src/reflector/mod.rs | 1 + kube-runtime/src/reflector/store.rs | 69 +++++----- kube-runtime/src/utils/reflect.rs | 23 +++- kube-runtime/src/utils/watch_ext.rs | 11 +- 12 files changed, 357 insertions(+), 143 deletions(-) diff --git a/examples/Cargo.toml b/examples/Cargo.toml index 1c7740f1c..e51081e3e 100644 --- a/examples/Cargo.toml +++ b/examples/Cargo.toml @@ -22,6 +22,7 @@ ws = ["kube/ws"] latest = ["k8s-openapi/latest"] [dev-dependencies] +parking_lot.workspace = true tokio-util.workspace = true assert-json-diff.workspace = true garde = { version = "0.22.0", default-features = false, features = ["derive"] } diff --git a/examples/multi_reflector.rs b/examples/multi_reflector.rs index 9f3c4537d..e7703a465 100644 --- a/examples/multi_reflector.rs +++ b/examples/multi_reflector.rs @@ -1,74 +1,80 @@ -use futures::{future, StreamExt}; +use futures::{future, stream, StreamExt}; use k8s_openapi::api::{ apps::v1::Deployment, core::v1::{ConfigMap, Secret}, }; use kube::{ - runtime::{ - reflector, - reflector::{ObjectRef, Store}, - watcher, WatchStreamExt, - }, - Api, Client, + api::{ApiResource, DynamicObject, GroupVersionKind}, + core::TypedResource, + runtime::{reflector::store::CacheWriter, watcher, WatchStreamExt}, + Api, Client, Resource, }; +use parking_lot::RwLock; +use serde::de::DeserializeOwned; use std::sync::Arc; use tracing::*; -// This does not work because Resource trait is not dyn safe. -/* -use std::any::TypeId; use std::collections::HashMap; -use k8s_openapi::NamespaceResourceScope; -use kube::api::{Resource, ResourceExt}; -struct MultiStore { - stores: HashMap>>, - } -impl MultiStore { - fn get>(&self, name: &str, ns: &str) -> Option> { - let oref = ObjectRef::::new(name).within(ns); - if let Some(store) = self.stores.get(&TypeId::of::()) { - store.get(oref) - } else { - None - } - } -}*/ -// explicit store can work -struct MultiStore { - deploys: Store, - cms: Store, - secs: Store, +type Cache = Arc>>>; + +#[derive(Default, Clone, Hash, PartialEq, Eq, Debug)] +struct LookupKey { + gvk: GroupVersionKind, + name: Option, + namespace: Option, } -// but using generics to help out won't because the K needs to be concretised -/* -impl MultiStore { - fn get>(&self, name: &str, ns: &str) -> Option>> { - let oref = ObjectRef::::new(name).within(ns); - let kind = K::kind(&()).to_owned(); - match kind.as_ref() { - "Deployment" => self.deploys.get(&ObjectRef::new(name).within(ns)), - "ConfigMap" => self.cms.get(&ObjectRef::new(name).within(ns)), - "Secret" => self.secs.get(&ObjectRef::new(name).within(ns)), - _ => None, + +impl LookupKey { + fn new(resource: &R) -> LookupKey { + let meta = resource.meta(); + LookupKey { + gvk: resource.gvk(), + name: meta.name.clone(), + namespace: meta.namespace.clone(), } - None } } -*/ -// so left with this -impl MultiStore { - fn get_deploy(&self, name: &str, ns: &str) -> Option> { - self.deploys.get(&ObjectRef::::new(name).within(ns)) - } +#[derive(Default, Clone)] +struct MultiCache { + store: Cache, +} - fn get_secret(&self, name: &str, ns: &str) -> Option> { - self.secs.get(&ObjectRef::::new(name).within(ns)) +impl MultiCache { + fn get + DeserializeOwned + Clone>( + &self, + name: &str, + ns: &str, + ) -> Option> { + let obj = self + .store + .read() + .get(&LookupKey { + gvk: K::gvk(&Default::default()), + name: Some(name.into()), + namespace: if !ns.is_empty() { Some(ns.into()) } else { None }, + })? + .as_ref() + .clone(); + obj.try_parse().ok().map(Arc::new) } +} - fn get_cm(&self, name: &str, ns: &str) -> Option> { - self.cms.get(&ObjectRef::::new(name).within(ns)) +impl CacheWriter for MultiCache { + /// Applies a single watcher event to the store + fn apply_watcher_event(&mut self, event: &watcher::Event) { + match event { + watcher::Event::Init | watcher::Event::InitDone => {} + watcher::Event::Delete(obj) => { + self.store.write().remove(&LookupKey::new(obj)); + } + watcher::Event::InitApply(obj) | watcher::Event::Apply(obj) => { + self.store + .write() + .insert(LookupKey::new(obj), Arc::new(obj.clone())); + } + } } } @@ -77,60 +83,62 @@ async fn main() -> anyhow::Result<()> { tracing_subscriber::fmt::init(); let client = Client::try_default().await?; - let deploys: Api = Api::default_namespaced(client.clone()); - let cms: Api = Api::default_namespaced(client.clone()); - let secret: Api = Api::default_namespaced(client.clone()); + // multistore + let mut combo_stream = stream::select_all(vec![]); + combo_stream.push( + watcher::watcher( + Api::all_with(client.clone(), &ApiResource::erase::(&())), + Default::default(), + ) + .boxed(), + ); + combo_stream.push( + watcher::watcher( + Api::all_with(client.clone(), &ApiResource::erase::(&())), + Default::default(), + ) + .boxed(), + ); - let (dep_reader, dep_writer) = reflector::store::(); - let (cm_reader, cm_writer) = reflector::store::(); - let (sec_reader, sec_writer) = reflector::store::(); + // // Duplicate streams with narrowed down selection + combo_stream.push( + watcher::watcher( + Api::default_namespaced_with(client.clone(), &ApiResource::erase::(&())), + Default::default(), + ) + .boxed(), + ); + combo_stream.push( + watcher::watcher( + Api::all_with(client.clone(), &ApiResource::erase::(&())), + Default::default(), + ) + .boxed(), + ); - let cfg = watcher::Config::default(); - let dep_watcher = watcher(deploys, cfg.clone()) - .reflect(dep_writer) - .applied_objects() - .for_each(|_| future::ready(())); - let cm_watcher = watcher(cms, cfg.clone()) - .reflect(cm_writer) + let multi_writer = MultiCache::default(); + let watcher = combo_stream + .reflect(multi_writer.clone()) .applied_objects() .for_each(|_| future::ready(())); - let sec_watcher = watcher(secret, cfg) - .reflect(sec_writer) - .applied_objects() - .for_each(|_| future::ready(())); - // poll these forever - - // multistore - let stores = MultiStore { - deploys: dep_reader, - cms: cm_reader, - secs: sec_reader, - }; // simulate doing stuff with the stores from some other thread tokio::spawn(async move { - // Show state every 5 seconds of watching - info!("waiting for them to be ready"); - stores.deploys.wait_until_ready().await.unwrap(); - stores.cms.wait_until_ready().await.unwrap(); - stores.secs.wait_until_ready().await.unwrap(); - info!("stores initialised"); // can use helper accessors - info!( - "common cm: {:?}", - stores.get_cm("kube-root-ca.crt", "kube-system").unwrap() - ); loop { tokio::time::sleep(std::time::Duration::from_secs(5)).await; + info!("cache content: {:?}", multi_writer.store.read().keys()); + info!( + "common cm: {:?}", + multi_writer.get::("kube-root-ca.crt", "kube-system") + ); // access individual sub stores - info!("Current deploys count: {}", stores.deploys.state().len()); + info!("Current objects count: {}", multi_writer.store.read().len()); } }); - // info!("long watches starting"); + info!("long watches starting"); tokio::select! { - r = dep_watcher => println!("dep watcher exit: {r:?}"), - r = cm_watcher => println!("cm watcher exit: {r:?}"), - r = sec_watcher => println!("sec watcher exit: {r:?}"), + r = watcher => println!("watcher exit: {r:?}"), } Ok(()) diff --git a/kube-core/src/gvk.rs b/kube-core/src/gvk.rs index 91b986601..77d140eda 100644 --- a/kube-core/src/gvk.rs +++ b/kube-core/src/gvk.rs @@ -12,7 +12,7 @@ use thiserror::Error; pub struct ParseGroupVersionError(pub String); /// Core information about an API Resource. -#[derive(Deserialize, Serialize, Debug, Clone, PartialEq, Eq, Hash)] +#[derive(Deserialize, Serialize, Debug, Clone, PartialEq, Eq, Hash, Default)] pub struct GroupVersionKind { /// API group pub group: String, diff --git a/kube-core/src/lib.rs b/kube-core/src/lib.rs index 6ba9f81b6..58d1e4be6 100644 --- a/kube-core/src/lib.rs +++ b/kube-core/src/lib.rs @@ -35,7 +35,7 @@ pub mod gvk; pub use gvk::{GroupVersion, GroupVersionKind, GroupVersionResource}; pub mod metadata; -pub use metadata::{ListMeta, ObjectMeta, PartialObjectMeta, PartialObjectMetaExt, TypeMeta}; +pub use metadata::{ListMeta, ObjectMeta, PartialObjectMeta, PartialObjectMetaExt, TypeMeta, TypedResource}; pub mod labels; diff --git a/kube-core/src/metadata.rs b/kube-core/src/metadata.rs index 67edf6e16..d02188f28 100644 --- a/kube-core/src/metadata.rs +++ b/kube-core/src/metadata.rs @@ -4,7 +4,7 @@ use std::{borrow::Cow, marker::PhantomData}; pub use k8s_openapi::apimachinery::pkg::apis::meta::v1::{ListMeta, ObjectMeta}; use serde::{Deserialize, Serialize}; -use crate::{DynamicObject, Resource}; +use crate::{ApiResource, DynamicObject, GroupVersionKind, Resource}; /// Type information that is flattened into every kubernetes object #[derive(Deserialize, Serialize, Clone, Default, Debug, Eq, PartialEq, Hash)] @@ -51,6 +51,24 @@ impl TypeMeta { kind: K::kind(&()).into(), } } + + /// Construct a new `TypeMeta` for the object from the list `TypeMeta`. + /// + /// ``` + /// # use k8s_openapi::api::core::v1::Pod; + /// # use kube_core::TypeMeta; + /// + /// let mut type_meta = TypeMeta::resource::(); + /// type_meta.kind = "PodList".to_string(); + /// assert_eq!(type_meta.clone().singular().kind, "Pod"); + /// assert_eq!(type_meta.clone().singular().api_version, "v1"); + /// ``` + pub fn singular(self) -> Self { + Self { + kind: self.kind.strip_suffix("List").unwrap_or(&self.kind).to_string(), + ..self + } + } } /// A generic representation of any object with `ObjectMeta`. @@ -175,6 +193,123 @@ impl Resource for PartialObjectMeta { } } +/// +pub trait TypedResource: Resource + Sized { + /// + fn type_meta(&self) -> TypeMeta; + /// + fn gvk(&self) -> GroupVersionKind; + /// + fn kind(&self) -> Cow<'_, str>; + /// + fn group(&self) -> Cow<'_, str>; + /// + fn version(&self) -> Cow<'_, str>; + /// + fn plural(&self) -> Cow<'_, str>; +} + +impl TypedResource for K +where + K: Resource, + (K, K::DynamicType): TypedResourceImpl, +{ + fn type_meta(&self) -> TypeMeta { + <(K, K::DynamicType) as TypedResourceImpl>::type_meta(self) + } + + fn gvk(&self) -> GroupVersionKind { + <(K, K::DynamicType) as TypedResourceImpl>::gvk(self) + } + + fn kind(&self) -> Cow<'_, str> { + <(K, K::DynamicType) as TypedResourceImpl>::kind(self) + } + /// + fn group(&self) -> Cow<'_, str> { + <(K, K::DynamicType) as TypedResourceImpl>::group(self) + } + /// + fn version(&self) -> Cow<'_, str> { + <(K, K::DynamicType) as TypedResourceImpl>::version(self) + } + /// + fn plural(&self) -> Cow<'_, str> { + <(K, K::DynamicType) as TypedResourceImpl>::plural(self) + } +} + +#[doc(hidden)] +// Workaround for https://github.com/rust-lang/rust/issues/20400 +pub trait TypedResourceImpl { + type Resource: Resource; + fn type_meta(res: &Self::Resource) -> TypeMeta; + fn gvk(res: &Self::Resource) -> GroupVersionKind; + fn kind(res: &Self::Resource) -> Cow<'_, str>; + fn group(res: &Self::Resource) -> Cow<'_, str>; + fn version(res: &Self::Resource) -> Cow<'_, str>; + fn plural(res: &Self::Resource) -> Cow<'_, str>; +} + +impl TypedResourceImpl for (K, ()) +where + K: Resource, +{ + type Resource = K; + + fn type_meta(_: &Self::Resource) -> TypeMeta { + TypeMeta::resource::() + } + + fn gvk(res: &Self::Resource) -> GroupVersionKind { + GroupVersionKind::gvk(&res.group(), &res.version(), &res.kind()) + } + + fn kind(_: &Self::Resource) -> Cow<'_, str> { + K::kind(&()) + } + + fn group(_: &Self::Resource) -> Cow<'_, str> { + K::group(&()) + } + + fn version(_: &Self::Resource) -> Cow<'_, str> { + K::version(&()) + } + + fn plural(_: &Self::Resource) -> Cow<'_, str> { + K::plural(&()) + } +} + +impl TypedResourceImpl for (DynamicObject, ApiResource) { + type Resource = DynamicObject; + + fn type_meta(obj: &Self::Resource) -> TypeMeta { + obj.types.clone().unwrap_or_default() + } + + fn gvk(res: &Self::Resource) -> GroupVersionKind { + res.type_meta().try_into().unwrap_or_default() + } + + fn kind(res: &Self::Resource) -> Cow<'_, str> { + Cow::from(res.type_meta().kind) + } + + fn group(res: &Self::Resource) -> Cow<'_, str> { + Cow::from(res.gvk().group) + } + + fn version(res: &Self::Resource) -> Cow<'_, str> { + Cow::from(res.gvk().version) + } + + fn plural(res: &Self::Resource) -> Cow<'_, str> { + Cow::from(ApiResource::from_gvk(&res.gvk()).plural) + } +} + #[cfg(test)] mod test { use super::{ObjectMeta, PartialObjectMeta, PartialObjectMetaExt}; diff --git a/kube-core/src/object.rs b/kube-core/src/object.rs index 1e81a947b..9622060c5 100644 --- a/kube-core/src/object.rs +++ b/kube-core/src/object.rs @@ -3,8 +3,12 @@ use crate::{ discovery::ApiResource, metadata::{ListMeta, ObjectMeta, TypeMeta}, resource::{DynamicResourceScope, Resource}, + DynamicObject, +}; +use serde::{ + de::{self, DeserializeOwned}, + Deserialize, Deserializer, Serialize, }; -use serde::{Deserialize, Deserializer, Serialize}; use std::borrow::Cow; /// A generic Kubernetes object list @@ -16,7 +20,7 @@ use std::borrow::Cow; /// and is generally produced from list/watch/delete collection queries on an [`Resource`](super::Resource). /// /// This is almost equivalent to [`k8s_openapi::List`](k8s_openapi::List), but iterable. -#[derive(Serialize, Deserialize, Debug, Clone)] +#[derive(Serialize, Debug, Clone)] pub struct ObjectList where T: Clone, @@ -132,6 +136,42 @@ impl<'a, T: Clone> IntoIterator for &'a mut ObjectList { } } +#[derive(Deserialize)] +struct DynamicList { + #[serde(flatten, deserialize_with = "deserialize_v1_list_as_default")] + types: TypeMeta, + + #[serde(default)] + metadata: ListMeta, + + #[serde( + deserialize_with = "deserialize_null_as_default", + bound(deserialize = "Vec: Deserialize<'de>") + )] + items: Vec, +} + +impl<'de, T: DeserializeOwned + Clone> serde::Deserialize<'de> for ObjectList { + fn deserialize>(d: D) -> Result, D::Error> { + let DynamicList { + types, + metadata, + mut items, + } = DynamicList::deserialize(d)?; + let mut resources = vec![]; + for o in items.iter_mut() { + o.types = Some(types.clone().singular()); + let item = serde_json::to_value(o).map_err(de::Error::custom)?; + resources.push(serde_json::from_value(item).map_err(de::Error::custom)?) + } + Ok(ObjectList:: { + types, + metadata, + items: resources, + }) + } +} + /// A trait to access the `spec` of a Kubernetes resource. /// /// Some built-in Kubernetes resources and all custom resources do have a `spec` field. @@ -357,9 +397,10 @@ mod test { assert_eq!(mypod.namespace().unwrap(), "dev"); assert_eq!(mypod.name_unchecked(), "blog"); assert!(mypod.status().is_none()); - assert_eq!(mypod.spec().containers[0], ContainerSimple { - image: "blog".into() - }); + assert_eq!( + mypod.spec().containers[0], + ContainerSimple { image: "blog".into() } + ); assert_eq!(PodSimple::api_version(&ar), "v1"); assert_eq!(PodSimple::version(&ar), "v1"); diff --git a/kube-core/src/resource.rs b/kube-core/src/resource.rs index 3ab1d88df..6dffdeeb1 100644 --- a/kube-core/src/resource.rs +++ b/kube-core/src/resource.rs @@ -8,6 +8,8 @@ use std::{borrow::Cow, collections::BTreeMap}; pub use k8s_openapi::{ClusterResourceScope, NamespaceResourceScope, ResourceScope, SubResourceScope}; +use crate::GroupVersionKind; + /// Indicates that a [`Resource`] is of an indeterminate dynamic scope. pub struct DynamicResourceScope {} impl ResourceScope for DynamicResourceScope {} @@ -54,6 +56,11 @@ pub trait Resource { /// This is known as the resource in apimachinery, we rename it for disambiguation. fn plural(dt: &Self::DynamicType) -> Cow<'_, str>; + /// Generates an object reference for the resource + fn gvk(dt: &Self::DynamicType) -> GroupVersionKind { + GroupVersionKind::gvk(&Self::group(dt), &Self::version(dt), &Self::kind(dt)) + } + /// Creates a url path for http requests for this resource fn url_path(dt: &Self::DynamicType, namespace: Option<&str>) -> String { let n = if let Some(ns) = namespace { diff --git a/kube-runtime/src/controller/mod.rs b/kube-runtime/src/controller/mod.rs index 8a8f7fc3a..0fb7dcefc 100644 --- a/kube-runtime/src/controller/mod.rs +++ b/kube-runtime/src/controller/mod.rs @@ -1683,13 +1683,13 @@ mod tests { use super::{Action, APPLIER_REQUEUE_BUF_SIZE}; use crate::{ applier, - reflector::{self, ObjectRef}, + reflector::{self, store::CacheWriter as _, ObjectRef}, watcher::{self, metadata_watcher, watcher, Event}, Config, Controller, }; use futures::{Stream, StreamExt, TryStreamExt}; use k8s_openapi::api::core::v1::ConfigMap; - use kube_client::{core::ObjectMeta, Api, Resource}; + use kube_client::{api::TypeMeta, core::ObjectMeta, Api, Resource}; use serde::de::DeserializeOwned; use tokio::time::timeout; diff --git a/kube-runtime/src/reflector/mod.rs b/kube-runtime/src/reflector/mod.rs index 88f4f2910..95df28214 100644 --- a/kube-runtime/src/reflector/mod.rs +++ b/kube-runtime/src/reflector/mod.rs @@ -11,6 +11,7 @@ pub use self::{ use crate::watcher; use async_stream::stream; use futures::{Stream, StreamExt}; +use store::CacheWriter as _; use std::hash::Hash; #[cfg(feature = "unstable-runtime-subscribe")] pub use store::store_shared; pub use store::{store, Store}; diff --git a/kube-runtime/src/reflector/store.rs b/kube-runtime/src/reflector/store.rs index d6d264dea..11995fbd5 100644 --- a/kube-runtime/src/reflector/store.rs +++ b/kube-runtime/src/reflector/store.rs @@ -13,6 +13,12 @@ use thiserror::Error; type Cache = Arc, Arc>>>; +/// A writable `CacheWriter` trait +pub trait CacheWriter { + /// Applies a single watcher event to the store + fn apply_watcher_event(&mut self, event: &watcher::Event); +} + /// A writable Store handle /// /// This is exclusive since it's not safe to share a single `Store` between multiple reflectors. @@ -98,8 +104,40 @@ where .map(|dispatcher| dispatcher.subscribe(self.as_reader())) } + /// Broadcast an event to any downstream listeners subscribed on the store + pub(crate) async fn dispatch_event(&mut self, event: &watcher::Event) { + if let Some(ref mut dispatcher) = self.dispatcher { + match event { + watcher::Event::Apply(obj) => { + let obj_ref = obj.to_object_ref(self.dyntype.clone()); + // TODO (matei): should this take a timeout to log when backpressure has + // been applied for too long, e.g. 10s + dispatcher.broadcast(obj_ref).await; + } + + watcher::Event::InitDone => { + let obj_refs: Vec<_> = { + let store = self.store.read(); + store.keys().cloned().collect() + }; + + for obj_ref in obj_refs { + dispatcher.broadcast(obj_ref).await; + } + } + + _ => {} + } + } + } +} + +impl CacheWriter for Writer +where + K::DynamicType: Eq + Hash + Clone, + { /// Applies a single watcher event to the store - pub fn apply_watcher_event(&mut self, event: &watcher::Event) { + fn apply_watcher_event(&mut self, event: &watcher::Event) { match event { watcher::Event::Apply(obj) => { let key = obj.to_object_ref(self.dyntype.clone()); @@ -136,33 +174,6 @@ where } } } - - /// Broadcast an event to any downstream listeners subscribed on the store - pub(crate) async fn dispatch_event(&mut self, event: &watcher::Event) { - if let Some(ref mut dispatcher) = self.dispatcher { - match event { - watcher::Event::Apply(obj) => { - let obj_ref = obj.to_object_ref(self.dyntype.clone()); - // TODO (matei): should this take a timeout to log when backpressure has - // been applied for too long, e.g. 10s - dispatcher.broadcast(obj_ref).await; - } - - watcher::Event::InitDone => { - let obj_refs: Vec<_> = { - let store = self.store.read(); - store.keys().cloned().collect() - }; - - for obj_ref in obj_refs { - dispatcher.broadcast(obj_ref).await; - } - } - - _ => {} - } - } - } } impl Default for Writer @@ -309,7 +320,7 @@ where #[cfg(test)] mod tests { use super::{store, Writer}; - use crate::{reflector::ObjectRef, watcher}; + use crate::{reflector::{store::CacheWriter as _, ObjectRef}, watcher}; use k8s_openapi::api::core::v1::ConfigMap; use kube_client::api::ObjectMeta; diff --git a/kube-runtime/src/utils/reflect.rs b/kube-runtime/src/utils/reflect.rs index e93354202..fe126df32 100644 --- a/kube-runtime/src/utils/reflect.rs +++ b/kube-runtime/src/utils/reflect.rs @@ -2,44 +2,53 @@ use core::{ pin::Pin, task::{Context, Poll}, }; +use std::marker::PhantomData; use futures::{Stream, TryStream}; use pin_project::pin_project; use crate::{ - reflector::store::Writer, + reflector::store::CacheWriter, watcher::{Error, Event}, }; use kube_client::Resource; /// Stream returned by the [`reflect`](super::WatchStreamExt::reflect) method #[pin_project] -pub struct Reflect +pub struct Reflect where K: Resource + Clone + 'static, K::DynamicType: Eq + std::hash::Hash + Clone, + W: CacheWriter, { #[pin] stream: St, - writer: Writer, + writer: W, + _phantom: PhantomData, } -impl Reflect +impl Reflect where St: TryStream>, K: Resource + Clone, K::DynamicType: Eq + std::hash::Hash + Clone, + W: CacheWriter, { - pub(super) fn new(stream: St, writer: Writer) -> Reflect { - Self { stream, writer } + pub(super) fn new(stream: St, writer: W) -> Reflect { + Self { + stream, + writer, + _phantom: Default::default(), + } } } -impl Stream for Reflect +impl Stream for Reflect where K: Resource + Clone, K::DynamicType: Eq + std::hash::Hash + Clone, St: Stream, Error>>, + W: CacheWriter, { type Item = Result, Error>; diff --git a/kube-runtime/src/utils/watch_ext.rs b/kube-runtime/src/utils/watch_ext.rs index 241871837..8e67e4eac 100644 --- a/kube-runtime/src/utils/watch_ext.rs +++ b/kube-runtime/src/utils/watch_ext.rs @@ -1,4 +1,5 @@ use crate::{ + reflector::store::CacheWriter, utils::{ event_decode::EventDecode, event_modify::EventModify, @@ -9,10 +10,10 @@ use crate::{ }; use kube_client::Resource; -use crate::{ - reflector::store::Writer, - utils::{Backoff, Reflect}, -}; +#[cfg(feature = "unstable-runtime-subscribe")] +use crate::reflector::store::Writer; + +use crate::utils::{Backoff, Reflect}; use crate::watcher::DefaultBackoff; use futures::{Stream, TryStream}; @@ -174,7 +175,7 @@ pub trait WatchStreamExt: Stream { /// ``` /// /// [`Store`]: crate::reflector::Store - fn reflect(self, writer: Writer) -> Reflect + fn reflect(self, writer: impl CacheWriter) -> Reflect> where Self: Stream>> + Sized, K: Resource + Clone + 'static, From cf1e6463a2ef6fe91a395f2236b2c2f65fff306d Mon Sep 17 00:00:00 2001 From: Danil-Grigorev Date: Mon, 10 Feb 2025 16:53:59 +0100 Subject: [PATCH 03/16] Implement MultiDispatcher for a DynamicObject steam Signed-off-by: Danil-Grigorev --- examples/Cargo.toml | 4 +- examples/broadcast_reflector.rs | 108 +++++++++++++ examples/multi_reflector.rs | 145 ------------------ kube-core/src/dynamic.rs | 8 +- kube-core/src/lib.rs | 2 +- kube-core/src/metadata.rs | 127 +-------------- kube-core/src/object.rs | 4 +- kube-runtime/src/controller/mod.rs | 33 ++-- kube-runtime/src/lib.rs | 2 + kube-runtime/src/reflector/dispatcher.rs | 145 +++++++++++++++++- kube-runtime/src/reflector/mod.rs | 26 +++- .../src/reflector/multi_dispatcher.rs | 51 ++++++ kube-runtime/src/reflector/store.rs | 96 +++++++----- kube-runtime/src/utils/reflect.rs | 23 +-- kube-runtime/src/utils/watch_ext.rs | 21 ++- 15 files changed, 447 insertions(+), 348 deletions(-) create mode 100644 examples/broadcast_reflector.rs delete mode 100644 examples/multi_reflector.rs create mode 100644 kube-runtime/src/reflector/multi_dispatcher.rs diff --git a/examples/Cargo.toml b/examples/Cargo.toml index e51081e3e..01b20200b 100644 --- a/examples/Cargo.toml +++ b/examples/Cargo.toml @@ -132,8 +132,8 @@ name = "multi_watcher" path = "multi_watcher.rs" [[example]] -name = "multi_reflector" -path = "multi_reflector.rs" +name = "broadcast_reflector" +path = "broadcast_reflector.rs" [[example]] name = "pod_api" diff --git a/examples/broadcast_reflector.rs b/examples/broadcast_reflector.rs new file mode 100644 index 000000000..a00f9c133 --- /dev/null +++ b/examples/broadcast_reflector.rs @@ -0,0 +1,108 @@ +use futures::{future, stream, StreamExt}; +use k8s_openapi::api::{ + apps::v1::Deployment, + core::v1::{ConfigMap, Secret}, +}; +use kube::{ + api::ApiResource, + runtime::{controller::Action, reflector::multi_dispatcher::MultiDispatcher, watcher, Controller, WatchStreamExt as _}, + Api, Client, ResourceExt, +}; +use std::{fmt::Debug, sync::Arc, time::Duration}; +use thiserror::Error; +use tracing::*; + +#[derive(Debug, Error)] +enum Infallible {} + +// A generic reconciler that can be used with any object whose type is known at +// compile time. Will simply log its kind on reconciliation. +async fn reconcile(_obj: Arc, _ctx: Arc<()>) -> Result +where + K: ResourceExt, +{ + let kind = K::kind(&()); + info!("Reconciled {kind}"); + Ok(Action::await_change()) +} + +fn error_policy(_: Arc, _: &Infallible, _ctx: Arc<()>) -> Action { + info!("error"); + Action::requeue(Duration::from_secs(10)) +} + +#[tokio::main] +async fn main() -> anyhow::Result<()> { + tracing_subscriber::fmt::init(); + let client = Client::try_default().await?; + + let writer = MultiDispatcher::new(128); + + // multireflector stream + let mut combo_stream = stream::select_all(vec![]); + combo_stream.push( + watcher::watcher( + Api::all_with(client.clone(), &ApiResource::erase::(&())), + Default::default(), + ) + .boxed(), + ); + + // watching config maps, but ignoring in the final configuration + combo_stream.push( + watcher::watcher( + Api::all_with(client.clone(), &ApiResource::erase::(&())), + Default::default(), + ) + .boxed(), + ); + + // Combine duplicate type streams with narrowed down selection + combo_stream.push( + watcher::watcher( + Api::default_namespaced_with(client.clone(), &ApiResource::erase::(&())), + Default::default(), + ) + .boxed(), + ); + combo_stream.push( + watcher::watcher( + Api::namespaced_with(client.clone(), "kube-system", &ApiResource::erase::(&())), + Default::default(), + ) + .boxed(), + ); + + let watcher = combo_stream.broadcast_shared(writer.clone()); + + let (sub, reader) = writer.subscribe::(); + let deploy = Controller::for_shared_stream(sub, reader) + .shutdown_on_signal() + .run(reconcile, error_policy, Arc::new(())) + .for_each(|res| async move { + match res { + Ok(v) => info!("Reconciled deployment {v:?}"), + Err(error) => warn!(%error, "Failed to reconcile metadata"), + }; + }); + + let (sub, reader) = writer.subscribe::(); + let secret = Controller::for_shared_stream(sub, reader) + .shutdown_on_signal() + .run(reconcile, error_policy, Arc::new(())) + .for_each(|res| async move { + match res { + Ok(v) => info!("Reconciled secret {v:?}"), + Err(error) => warn!(%error, "Failed to reconcile metadata"), + }; + }); + + info!("long watches starting"); + tokio::select! { + r = watcher.for_each(|_| future::ready(())) => println!("watcher exit: {r:?}"), + x = deploy => println!("deployments exit: {x:?}"), + x = secret => println!("secrets exit: {x:?}"), + } + + Ok(()) +} diff --git a/examples/multi_reflector.rs b/examples/multi_reflector.rs deleted file mode 100644 index e7703a465..000000000 --- a/examples/multi_reflector.rs +++ /dev/null @@ -1,145 +0,0 @@ -use futures::{future, stream, StreamExt}; -use k8s_openapi::api::{ - apps::v1::Deployment, - core::v1::{ConfigMap, Secret}, -}; -use kube::{ - api::{ApiResource, DynamicObject, GroupVersionKind}, - core::TypedResource, - runtime::{reflector::store::CacheWriter, watcher, WatchStreamExt}, - Api, Client, Resource, -}; -use parking_lot::RwLock; -use serde::de::DeserializeOwned; -use std::sync::Arc; -use tracing::*; - -use std::collections::HashMap; - -type Cache = Arc>>>; - -#[derive(Default, Clone, Hash, PartialEq, Eq, Debug)] -struct LookupKey { - gvk: GroupVersionKind, - name: Option, - namespace: Option, -} - -impl LookupKey { - fn new(resource: &R) -> LookupKey { - let meta = resource.meta(); - LookupKey { - gvk: resource.gvk(), - name: meta.name.clone(), - namespace: meta.namespace.clone(), - } - } -} - -#[derive(Default, Clone)] -struct MultiCache { - store: Cache, -} - -impl MultiCache { - fn get + DeserializeOwned + Clone>( - &self, - name: &str, - ns: &str, - ) -> Option> { - let obj = self - .store - .read() - .get(&LookupKey { - gvk: K::gvk(&Default::default()), - name: Some(name.into()), - namespace: if !ns.is_empty() { Some(ns.into()) } else { None }, - })? - .as_ref() - .clone(); - obj.try_parse().ok().map(Arc::new) - } -} - -impl CacheWriter for MultiCache { - /// Applies a single watcher event to the store - fn apply_watcher_event(&mut self, event: &watcher::Event) { - match event { - watcher::Event::Init | watcher::Event::InitDone => {} - watcher::Event::Delete(obj) => { - self.store.write().remove(&LookupKey::new(obj)); - } - watcher::Event::InitApply(obj) | watcher::Event::Apply(obj) => { - self.store - .write() - .insert(LookupKey::new(obj), Arc::new(obj.clone())); - } - } - } -} - -#[tokio::main] -async fn main() -> anyhow::Result<()> { - tracing_subscriber::fmt::init(); - let client = Client::try_default().await?; - - // multistore - let mut combo_stream = stream::select_all(vec![]); - combo_stream.push( - watcher::watcher( - Api::all_with(client.clone(), &ApiResource::erase::(&())), - Default::default(), - ) - .boxed(), - ); - combo_stream.push( - watcher::watcher( - Api::all_with(client.clone(), &ApiResource::erase::(&())), - Default::default(), - ) - .boxed(), - ); - - // // Duplicate streams with narrowed down selection - combo_stream.push( - watcher::watcher( - Api::default_namespaced_with(client.clone(), &ApiResource::erase::(&())), - Default::default(), - ) - .boxed(), - ); - combo_stream.push( - watcher::watcher( - Api::all_with(client.clone(), &ApiResource::erase::(&())), - Default::default(), - ) - .boxed(), - ); - - let multi_writer = MultiCache::default(); - let watcher = combo_stream - .reflect(multi_writer.clone()) - .applied_objects() - .for_each(|_| future::ready(())); - - // simulate doing stuff with the stores from some other thread - tokio::spawn(async move { - // can use helper accessors - loop { - tokio::time::sleep(std::time::Duration::from_secs(5)).await; - info!("cache content: {:?}", multi_writer.store.read().keys()); - info!( - "common cm: {:?}", - multi_writer.get::("kube-root-ca.crt", "kube-system") - ); - // access individual sub stores - info!("Current objects count: {}", multi_writer.store.read().len()); - } - }); - info!("long watches starting"); - tokio::select! { - r = watcher => println!("watcher exit: {r:?}"), - } - - Ok(()) -} diff --git a/kube-core/src/dynamic.rs b/kube-core/src/dynamic.rs index ebaa7e3a6..f07c5e709 100644 --- a/kube-core/src/dynamic.rs +++ b/kube-core/src/dynamic.rs @@ -4,7 +4,7 @@ pub use crate::discovery::ApiResource; use crate::{ metadata::TypeMeta, - resource::{DynamicResourceScope, Resource}, + resource::{DynamicResourceScope, Resource}, GroupVersionKind, }; use k8s_openapi::apimachinery::pkg::apis::meta::v1::ObjectMeta; @@ -73,6 +73,12 @@ impl DynamicObject { ) -> Result { Ok(serde_json::from_value(serde_json::to_value(self)?)?) } + + /// Returns the group, version, and kind (GVK) of this resource. + pub fn gvk(&self) -> Option { + let gvk = self.types.clone()?; + gvk.try_into().ok() + } } impl Resource for DynamicObject { diff --git a/kube-core/src/lib.rs b/kube-core/src/lib.rs index 58d1e4be6..6ba9f81b6 100644 --- a/kube-core/src/lib.rs +++ b/kube-core/src/lib.rs @@ -35,7 +35,7 @@ pub mod gvk; pub use gvk::{GroupVersion, GroupVersionKind, GroupVersionResource}; pub mod metadata; -pub use metadata::{ListMeta, ObjectMeta, PartialObjectMeta, PartialObjectMetaExt, TypeMeta, TypedResource}; +pub use metadata::{ListMeta, ObjectMeta, PartialObjectMeta, PartialObjectMetaExt, TypeMeta}; pub mod labels; diff --git a/kube-core/src/metadata.rs b/kube-core/src/metadata.rs index d02188f28..44e0a750d 100644 --- a/kube-core/src/metadata.rs +++ b/kube-core/src/metadata.rs @@ -4,7 +4,7 @@ use std::{borrow::Cow, marker::PhantomData}; pub use k8s_openapi::apimachinery::pkg::apis::meta::v1::{ListMeta, ObjectMeta}; use serde::{Deserialize, Serialize}; -use crate::{ApiResource, DynamicObject, GroupVersionKind, Resource}; +use crate::{DynamicObject, Resource}; /// Type information that is flattened into every kubernetes object #[derive(Deserialize, Serialize, Clone, Default, Debug, Eq, PartialEq, Hash)] @@ -63,11 +63,9 @@ impl TypeMeta { /// assert_eq!(type_meta.clone().singular().kind, "Pod"); /// assert_eq!(type_meta.clone().singular().api_version, "v1"); /// ``` - pub fn singular(self) -> Self { - Self { - kind: self.kind.strip_suffix("List").unwrap_or(&self.kind).to_string(), - ..self - } + pub fn singular(self) -> Option { + let kind = self.kind.strip_suffix("List")?.to_string(); + (!kind.is_empty()).then_some(Self { kind, ..self }) } } @@ -193,123 +191,6 @@ impl Resource for PartialObjectMeta { } } -/// -pub trait TypedResource: Resource + Sized { - /// - fn type_meta(&self) -> TypeMeta; - /// - fn gvk(&self) -> GroupVersionKind; - /// - fn kind(&self) -> Cow<'_, str>; - /// - fn group(&self) -> Cow<'_, str>; - /// - fn version(&self) -> Cow<'_, str>; - /// - fn plural(&self) -> Cow<'_, str>; -} - -impl TypedResource for K -where - K: Resource, - (K, K::DynamicType): TypedResourceImpl, -{ - fn type_meta(&self) -> TypeMeta { - <(K, K::DynamicType) as TypedResourceImpl>::type_meta(self) - } - - fn gvk(&self) -> GroupVersionKind { - <(K, K::DynamicType) as TypedResourceImpl>::gvk(self) - } - - fn kind(&self) -> Cow<'_, str> { - <(K, K::DynamicType) as TypedResourceImpl>::kind(self) - } - /// - fn group(&self) -> Cow<'_, str> { - <(K, K::DynamicType) as TypedResourceImpl>::group(self) - } - /// - fn version(&self) -> Cow<'_, str> { - <(K, K::DynamicType) as TypedResourceImpl>::version(self) - } - /// - fn plural(&self) -> Cow<'_, str> { - <(K, K::DynamicType) as TypedResourceImpl>::plural(self) - } -} - -#[doc(hidden)] -// Workaround for https://github.com/rust-lang/rust/issues/20400 -pub trait TypedResourceImpl { - type Resource: Resource; - fn type_meta(res: &Self::Resource) -> TypeMeta; - fn gvk(res: &Self::Resource) -> GroupVersionKind; - fn kind(res: &Self::Resource) -> Cow<'_, str>; - fn group(res: &Self::Resource) -> Cow<'_, str>; - fn version(res: &Self::Resource) -> Cow<'_, str>; - fn plural(res: &Self::Resource) -> Cow<'_, str>; -} - -impl TypedResourceImpl for (K, ()) -where - K: Resource, -{ - type Resource = K; - - fn type_meta(_: &Self::Resource) -> TypeMeta { - TypeMeta::resource::() - } - - fn gvk(res: &Self::Resource) -> GroupVersionKind { - GroupVersionKind::gvk(&res.group(), &res.version(), &res.kind()) - } - - fn kind(_: &Self::Resource) -> Cow<'_, str> { - K::kind(&()) - } - - fn group(_: &Self::Resource) -> Cow<'_, str> { - K::group(&()) - } - - fn version(_: &Self::Resource) -> Cow<'_, str> { - K::version(&()) - } - - fn plural(_: &Self::Resource) -> Cow<'_, str> { - K::plural(&()) - } -} - -impl TypedResourceImpl for (DynamicObject, ApiResource) { - type Resource = DynamicObject; - - fn type_meta(obj: &Self::Resource) -> TypeMeta { - obj.types.clone().unwrap_or_default() - } - - fn gvk(res: &Self::Resource) -> GroupVersionKind { - res.type_meta().try_into().unwrap_or_default() - } - - fn kind(res: &Self::Resource) -> Cow<'_, str> { - Cow::from(res.type_meta().kind) - } - - fn group(res: &Self::Resource) -> Cow<'_, str> { - Cow::from(res.gvk().group) - } - - fn version(res: &Self::Resource) -> Cow<'_, str> { - Cow::from(res.gvk().version) - } - - fn plural(res: &Self::Resource) -> Cow<'_, str> { - Cow::from(ApiResource::from_gvk(&res.gvk()).plural) - } -} - #[cfg(test)] mod test { use super::{ObjectMeta, PartialObjectMeta, PartialObjectMetaExt}; diff --git a/kube-core/src/object.rs b/kube-core/src/object.rs index 9622060c5..a7d62d07c 100644 --- a/kube-core/src/object.rs +++ b/kube-core/src/object.rs @@ -160,7 +160,9 @@ impl<'de, T: DeserializeOwned + Clone> serde::Deserialize<'de> for ObjectList } = DynamicList::deserialize(d)?; let mut resources = vec![]; for o in items.iter_mut() { - o.types = Some(types.clone().singular()); + if o.types.is_none() { + o.types = types.clone().singular(); + } let item = serde_json::to_value(o).map_err(de::Error::custom)?; resources.push(serde_json::from_value(item).map_err(de::Error::custom)?) } diff --git a/kube-runtime/src/controller/mod.rs b/kube-runtime/src/controller/mod.rs index 0fb7dcefc..68eb2d8c5 100644 --- a/kube-runtime/src/controller/mod.rs +++ b/kube-runtime/src/controller/mod.rs @@ -851,7 +851,10 @@ where /// } /// # } #[cfg(feature = "unstable-runtime-subscribe")] - pub fn for_shared_stream(trigger: impl Stream> + Send + 'static, reader: Store) -> Self + pub fn for_shared_stream( + trigger: impl Stream>> + Send + 'static> + Send + 'static, + reader: Store, + ) -> Self where K::DynamicType: Default, { @@ -878,12 +881,16 @@ where /// [`dynamic`]: kube_client::core::dynamic #[cfg(feature = "unstable-runtime-subscribe")] pub fn for_shared_stream_with( - trigger: impl Stream> + Send + 'static, + trigger: impl Stream>> + Send + 'static> + Send + 'static, reader: Store, dyntype: K::DynamicType, ) -> Self { let mut trigger_selector = stream::SelectAll::new(); - let self_watcher = trigger_self_shared(trigger.map(Ok), dyntype.clone()).boxed(); + let self_watcher = trigger_self_shared( + trigger.filter_map(|r| async move { r.into() }).map(Ok), + dyntype.clone(), + ) + .boxed(); trigger_selector.push(self_watcher); Self { trigger_selector, @@ -1112,7 +1119,7 @@ where #[must_use] pub fn owns_shared_stream + Send + 'static>( self, - trigger: impl Stream> + Send + 'static, + trigger: impl Stream>> + Send + 'static> + Send + 'static, ) -> Self { self.owns_shared_stream_with(trigger, ()) } @@ -1130,13 +1137,17 @@ where #[must_use] pub fn owns_shared_stream_with + Send + 'static>( mut self, - trigger: impl Stream> + Send + 'static, + trigger: impl Stream>> + Send + 'static> + Send + 'static, dyntype: Child::DynamicType, ) -> Self where Child::DynamicType: Debug + Eq + Hash + Clone, { - let child_watcher = trigger_owners_shared(trigger.map(Ok), self.dyntype.clone(), dyntype); + let child_watcher = trigger_owners_shared( + trigger.filter_map(|r| async move { r.into() }).map(Ok), + self.dyntype.clone(), + dyntype, + ); self.trigger_selector.push(child_watcher.boxed()); self } @@ -1383,7 +1394,7 @@ where #[must_use] pub fn watches_shared_stream( self, - trigger: impl Stream> + Send + 'static, + trigger: impl Stream>> + Send + 'static> + Send + 'static, mapper: impl Fn(Arc) -> I + Sync + Send + 'static, ) -> Self where @@ -1408,7 +1419,7 @@ where #[must_use] pub fn watches_shared_stream_with( mut self, - trigger: impl Stream> + Send + 'static, + trigger: impl Stream>> + Send + 'static> + Send + 'static, mapper: impl Fn(Arc) -> I + Sync + Send + 'static, dyntype: Other::DynamicType, ) -> Self @@ -1418,7 +1429,11 @@ where I: 'static + IntoIterator>, I::IntoIter: Send, { - let other_watcher = trigger_others_shared(trigger.map(Ok), mapper, dyntype); + let other_watcher = trigger_others_shared( + trigger.filter_map(|r| async move { r.into() }).map(Ok), + mapper, + dyntype, + ); self.trigger_selector.push(other_watcher.boxed()); self } diff --git a/kube-runtime/src/lib.rs b/kube-runtime/src/lib.rs index 7d0d8512c..1ba4b091d 100644 --- a/kube-runtime/src/lib.rs +++ b/kube-runtime/src/lib.rs @@ -33,6 +33,8 @@ pub mod watcher; pub use controller::{applier, Config, Controller}; pub use finalizer::finalizer; pub use reflector::reflector; +#[cfg(feature = "unstable-runtime-subscribe")] +pub use reflector::broadcaster; pub use scheduler::scheduler; pub use utils::WatchStreamExt; pub use watcher::{metadata_watcher, watcher}; diff --git a/kube-runtime/src/reflector/dispatcher.rs b/kube-runtime/src/reflector/dispatcher.rs index 1060dab2b..80e32514d 100644 --- a/kube-runtime/src/reflector/dispatcher.rs +++ b/kube-runtime/src/reflector/dispatcher.rs @@ -6,13 +6,18 @@ use std::{fmt::Debug, sync::Arc}; use educe::Educe; use futures::Stream; +use kube_client::{api::DynamicObject, Resource}; use pin_project::pin_project; +use serde::de::DeserializeOwned; use std::task::ready; -use crate::reflector::{ObjectRef, Store}; +use crate::{ + reflector::{ObjectRef, Store}, + watcher::{self, Event}, +}; use async_broadcast::{InactiveReceiver, Receiver, Sender}; -use super::Lookup; +use super::{store::Writer, Lookup}; #[derive(Educe)] #[educe(Debug(bound("K: Debug, K::DynamicType: Debug")), Clone)] @@ -72,6 +77,59 @@ where } } +#[derive(Clone)] +// A helper type that holds a broadcast transmitter and a broadcast receiver, +// used to fan-out events from a root stream to multiple listeners. +pub(crate) struct TypedDispatcher { + dispatch_tx: Sender>, + // An inactive reader that prevents the channel from closing until the + // writer is dropped. + _dispatch_rx: InactiveReceiver>, +} + +impl TypedDispatcher { + /// Creates and returns a new self that wraps a broadcast sender and an + /// inactive broadcast receiver + /// + /// A buffer size is required to create the underlying broadcast channel. + /// Messages will be buffered until all active readers have received a copy + /// of the message. When the channel is full, senders will apply + /// backpressure by waiting for space to free up. + // + // N.B messages are eagerly broadcasted, meaning no active receivers are + // required for a message to be broadcasted. + pub(crate) fn new(buf_size: usize) -> TypedDispatcher { + // Create a broadcast (tx, rx) pair + let (mut dispatch_tx, dispatch_rx) = async_broadcast::broadcast(buf_size); + // The tx half will not wait for any receivers to be active before + // broadcasting events. If no receivers are active, events will be + // buffered. + dispatch_tx.set_await_active(false); + Self { + dispatch_tx, + _dispatch_rx: dispatch_rx.deactivate(), + } + } + + // Calls broadcast on the channel. Will return when the channel has enough + // space to send an event. + pub(crate) async fn broadcast(&mut self, evt: Event) { + let _ = self.dispatch_tx.broadcast_direct(evt).await; + } + + // Creates a `TypedReflectHandle` by creating a receiver from the tx half. + // N.B: the new receiver will be fast-forwarded to the _latest_ event. + // The receiver won't have access to any events that are currently waiting + // to be acked by listeners. + pub(crate) fn subscribe(&self) -> TypedReflectHandle + where + K: Resource + DeserializeOwned + Clone, + K::DynamicType: Eq + std::hash::Hash + Clone + Default, + { + TypedReflectHandle::new(self.dispatch_tx.new_receiver()) + } +} + /// A handle to a shared stream reader /// /// [`ReflectHandle`]s are created by calling [`subscribe()`] on a [`Writer`], @@ -125,7 +183,7 @@ where impl Stream for ReflectHandle where K: Lookup + Clone, - K::DynamicType: Eq + std::hash::Hash + Clone + Default, + K::DynamicType: Eq + std::hash::Hash + Clone, { type Item = Arc; @@ -141,6 +199,87 @@ where } } +/// A handle to a shared dynamic object stream +/// +/// [`TypedReflectHandle`]s are created by calling [`subscribe()`] on a [`TypedDispatcher`], +/// Each shared stream reader should be polled independently and driven to readiness +/// to avoid deadlocks. When the [`TypedDispatcher`]'s buffer is filled, backpressure +/// will be applied on the root stream side. +/// +/// When the root stream is dropped, or it ends, all [`TypedReflectHandle`]s +/// subscribed to the shared stream will also terminate after all events yielded by +/// the root stream have been observed. This means [`TypedReflectHandle`] streams +/// can still be polled after the root stream has been dropped. +#[pin_project] +pub struct TypedReflectHandle +where + K: Lookup + Clone + 'static, + K::DynamicType: Eq + std::hash::Hash + Clone, + K: DeserializeOwned, +{ + #[pin] + rx: Receiver>, + store: Writer, +} + +impl TypedReflectHandle +where + K: Lookup + Clone + 'static, + K::DynamicType: Eq + std::hash::Hash + Clone + Default, + K: DeserializeOwned, +{ + pub(super) fn new(rx: Receiver>) -> TypedReflectHandle { + Self { + rx, + // Initialize a ready store by default + store: { + let mut store: Writer = Default::default(); + store.apply_shared_watcher_event(&watcher::Event::InitDone); + store + }, + } + } + + pub fn reader(&self) -> Store { + self.store.as_reader() + } +} + +impl Stream for TypedReflectHandle +where + K: Resource + Clone + 'static, + K::DynamicType: Eq + std::hash::Hash + Clone + Default, + K: DeserializeOwned, +{ + type Item = Option>; + + fn poll_next(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + let mut this = self.project(); + match ready!(this.rx.as_mut().poll_next(cx)) { + Some(event) => { + let obj = match event { + Event::InitApply(obj) | Event::Apply(obj) + if obj.gvk() == Some(K::gvk(&Default::default())) => + { + obj.try_parse::().ok().map(Arc::new).inspect(|o| { + this.store.apply_shared_watcher_event(&Event::Apply(o.clone())); + }) + } + Event::Delete(obj) if obj.gvk() == Some(K::gvk(&Default::default())) => { + obj.try_parse::().ok().map(Arc::new).inspect(|o| { + this.store.apply_shared_watcher_event(&Event::Delete(o.clone())); + }) + } + _ => None, + }; + + Poll::Ready(Some(obj)) + } + None => Poll::Ready(None), + } + } +} + #[cfg(feature = "unstable-runtime-subscribe")] #[cfg(test)] pub(crate) mod test { diff --git a/kube-runtime/src/reflector/mod.rs b/kube-runtime/src/reflector/mod.rs index 95df28214..479516829 100644 --- a/kube-runtime/src/reflector/mod.rs +++ b/kube-runtime/src/reflector/mod.rs @@ -2,6 +2,8 @@ mod dispatcher; mod object_ref; +#[cfg(feature = "unstable-runtime-subscribe")] +pub mod multi_dispatcher; pub mod store; pub use self::{ @@ -11,7 +13,9 @@ pub use self::{ use crate::watcher; use async_stream::stream; use futures::{Stream, StreamExt}; -use store::CacheWriter as _; +use kube_client::api::DynamicObject; +#[cfg(feature = "unstable-runtime-subscribe")] +use multi_dispatcher::MultiDispatcher; use std::hash::Hash; #[cfg(feature = "unstable-runtime-subscribe")] pub use store::store_shared; pub use store::{store, Store}; @@ -135,6 +139,26 @@ where } } +// broadcaster uses a common stream of DynamicObject events to distribute to any subscribed typed watcher. +#[cfg(feature = "unstable-runtime-subscribe")] +pub fn broadcaster(mut writer: MultiDispatcher, stream: W) -> impl Stream +where + W: Stream>>, +{ + let mut stream = Box::pin(stream); + stream! { + while let Some(event) = stream.next().await { + match event { + Ok(ev) => { + writer.broadcast_event(&ev).await; + yield Ok(ev); + }, + Err(ev) => yield Err(ev) + } + } + } +} + #[cfg(test)] mod tests { use super::{reflector, store, ObjectRef}; diff --git a/kube-runtime/src/reflector/multi_dispatcher.rs b/kube-runtime/src/reflector/multi_dispatcher.rs new file mode 100644 index 000000000..f679e16d8 --- /dev/null +++ b/kube-runtime/src/reflector/multi_dispatcher.rs @@ -0,0 +1,51 @@ +use std::hash::Hash; + +use kube_client::{ + api::DynamicObject, + Resource, +}; +use serde::de::DeserializeOwned; + +use crate::watcher; + +use super::{ + dispatcher::{TypedDispatcher, TypedReflectHandle}, Store, +}; + +#[derive(Clone)] +pub struct MultiDispatcher { + dispatcher: TypedDispatcher, +} + +impl MultiDispatcher { + #[must_use] + pub fn new(buf_size: usize) -> Self { + Self { + // store: Default::default(), + dispatcher: TypedDispatcher::new(buf_size), + } + } + + /// Return a handle to a typed subscriber + /// + /// Multiple subscribe handles may be obtained, by either calling + /// `subscribe` multiple times, or by calling `clone()` + /// + /// This function returns a `Some` when the [`Writer`] is constructed through + /// [`Writer::new_shared`] or [`store_shared`], and a `None` otherwise. + #[must_use] + pub fn subscribe(&self) -> (TypedReflectHandle, Store) + where + K: Resource + Clone + DeserializeOwned, + K::DynamicType: Eq + Clone + Hash + Default, + { + let sub = self.dispatcher.subscribe(); + let reader = sub.reader(); + (sub, reader) + } + + /// Broadcast an event to any downstream listeners subscribed on the store + pub(crate) async fn broadcast_event(&mut self, event: &watcher::Event) { + self.dispatcher.broadcast(event.clone()).await + } +} diff --git a/kube-runtime/src/reflector/store.rs b/kube-runtime/src/reflector/store.rs index 11995fbd5..a55837732 100644 --- a/kube-runtime/src/reflector/store.rs +++ b/kube-runtime/src/reflector/store.rs @@ -13,12 +13,6 @@ use thiserror::Error; type Cache = Arc, Arc>>>; -/// A writable `CacheWriter` trait -pub trait CacheWriter { - /// Applies a single watcher event to the store - fn apply_watcher_event(&mut self, event: &watcher::Event); -} - /// A writable Store handle /// /// This is exclusive since it's not safe to share a single `Store` between multiple reflectors. @@ -104,45 +98,36 @@ where .map(|dispatcher| dispatcher.subscribe(self.as_reader())) } - /// Broadcast an event to any downstream listeners subscribed on the store - pub(crate) async fn dispatch_event(&mut self, event: &watcher::Event) { - if let Some(ref mut dispatcher) = self.dispatcher { - match event { - watcher::Event::Apply(obj) => { - let obj_ref = obj.to_object_ref(self.dyntype.clone()); - // TODO (matei): should this take a timeout to log when backpressure has - // been applied for too long, e.g. 10s - dispatcher.broadcast(obj_ref).await; - } - - watcher::Event::InitDone => { - let obj_refs: Vec<_> = { - let store = self.store.read(); - store.keys().cloned().collect() - }; - - for obj_ref in obj_refs { - dispatcher.broadcast(obj_ref).await; - } - } - - _ => {} + /// Applies a single watcher event to the store + pub(crate) fn apply_watcher_event(&mut self, event: &watcher::Event) { + match event { + watcher::Event::Apply(obj) => { + let obj = Arc::new(obj.clone()); + self.apply_shared_watcher_event(&watcher::Event::Apply(obj)); + } + watcher::Event::Delete(obj) => { + let obj = Arc::new(obj.clone()); + self.apply_shared_watcher_event(&watcher::Event::Delete(obj)); + } + watcher::Event::InitApply(obj) => { + let obj = Arc::new(obj.clone()); + self.apply_shared_watcher_event(&watcher::Event::InitApply(obj)); + } + watcher::Event::Init => { + self.apply_shared_watcher_event(&watcher::Event::Init); + } + watcher::Event::InitDone => { + self.apply_shared_watcher_event(&watcher::Event::InitDone); } } } -} -impl CacheWriter for Writer -where - K::DynamicType: Eq + Hash + Clone, - { - /// Applies a single watcher event to the store - fn apply_watcher_event(&mut self, event: &watcher::Event) { + /// Applies a single shared watcher event to the store + pub(crate) fn apply_shared_watcher_event(&mut self, event: &watcher::Event>) { match event { watcher::Event::Apply(obj) => { let key = obj.to_object_ref(self.dyntype.clone()); - let obj = Arc::new(obj.clone()); - self.store.write().insert(key, obj); + self.store.write().insert(key, obj.clone()); } watcher::Event::Delete(obj) => { let key = obj.to_object_ref(self.dyntype.clone()); @@ -153,8 +138,7 @@ where } watcher::Event::InitApply(obj) => { let key = obj.to_object_ref(self.dyntype.clone()); - let obj = Arc::new(obj.clone()); - self.buffer.insert(key, obj); + self.buffer.insert(key, obj.clone()); } watcher::Event::InitDone => { let mut store = self.store.write(); @@ -174,6 +158,33 @@ where } } } + + /// Broadcast an event to any downstream listeners subscribed on the store + pub(crate) async fn dispatch_event(&mut self, event: &watcher::Event) { + if let Some(ref mut dispatcher) = self.dispatcher { + match event { + watcher::Event::Apply(obj) => { + let obj_ref = obj.to_object_ref(self.dyntype.clone()); + // TODO (matei): should this take a timeout to log when backpressure has + // been applied for too long, e.g. 10s + dispatcher.broadcast(obj_ref).await; + } + + watcher::Event::InitDone => { + let obj_refs: Vec<_> = { + let store = self.store.read(); + store.keys().cloned().collect() + }; + + for obj_ref in obj_refs { + dispatcher.broadcast(obj_ref).await; + } + } + + _ => {} + } + } + } } impl Default for Writer @@ -320,7 +331,10 @@ where #[cfg(test)] mod tests { use super::{store, Writer}; - use crate::{reflector::{store::CacheWriter as _, ObjectRef}, watcher}; + use crate::{ + reflector::{store::CacheWriter as _, ObjectRef}, + watcher, + }; use k8s_openapi::api::core::v1::ConfigMap; use kube_client::api::ObjectMeta; diff --git a/kube-runtime/src/utils/reflect.rs b/kube-runtime/src/utils/reflect.rs index fe126df32..e93354202 100644 --- a/kube-runtime/src/utils/reflect.rs +++ b/kube-runtime/src/utils/reflect.rs @@ -2,53 +2,44 @@ use core::{ pin::Pin, task::{Context, Poll}, }; -use std::marker::PhantomData; use futures::{Stream, TryStream}; use pin_project::pin_project; use crate::{ - reflector::store::CacheWriter, + reflector::store::Writer, watcher::{Error, Event}, }; use kube_client::Resource; /// Stream returned by the [`reflect`](super::WatchStreamExt::reflect) method #[pin_project] -pub struct Reflect +pub struct Reflect where K: Resource + Clone + 'static, K::DynamicType: Eq + std::hash::Hash + Clone, - W: CacheWriter, { #[pin] stream: St, - writer: W, - _phantom: PhantomData, + writer: Writer, } -impl Reflect +impl Reflect where St: TryStream>, K: Resource + Clone, K::DynamicType: Eq + std::hash::Hash + Clone, - W: CacheWriter, { - pub(super) fn new(stream: St, writer: W) -> Reflect { - Self { - stream, - writer, - _phantom: Default::default(), - } + pub(super) fn new(stream: St, writer: Writer) -> Reflect { + Self { stream, writer } } } -impl Stream for Reflect +impl Stream for Reflect where K: Resource + Clone, K::DynamicType: Eq + std::hash::Hash + Clone, St: Stream, Error>>, - W: CacheWriter, { type Item = Result, Error>; diff --git a/kube-runtime/src/utils/watch_ext.rs b/kube-runtime/src/utils/watch_ext.rs index 8e67e4eac..76a1f73ce 100644 --- a/kube-runtime/src/utils/watch_ext.rs +++ b/kube-runtime/src/utils/watch_ext.rs @@ -1,5 +1,4 @@ use crate::{ - reflector::store::CacheWriter, utils::{ event_decode::EventDecode, event_modify::EventModify, @@ -11,9 +10,13 @@ use crate::{ use kube_client::Resource; #[cfg(feature = "unstable-runtime-subscribe")] -use crate::reflector::store::Writer; - -use crate::utils::{Backoff, Reflect}; +use crate::reflector::multi_dispatcher::MultiDispatcher; +use crate::{ + reflector::store::Writer, + utils::{Backoff, Reflect}, +}; +#[cfg(feature = "unstable-runtime-subscribe")] +use kube_client::api::DynamicObject; use crate::watcher::DefaultBackoff; use futures::{Stream, TryStream}; @@ -175,7 +178,7 @@ pub trait WatchStreamExt: Stream { /// ``` /// /// [`Store`]: crate::reflector::Store - fn reflect(self, writer: impl CacheWriter) -> Reflect> + fn reflect(self, writer: Writer) -> Reflect where Self: Stream>> + Sized, K: Resource + Clone + 'static, @@ -272,6 +275,14 @@ pub trait WatchStreamExt: Stream { { crate::reflector(writer, self) } + + #[cfg(feature = "unstable-runtime-subscribe")] + fn broadcast_shared(self, writer: MultiDispatcher) -> impl Stream + where + Self: Stream>> + Sized, + { + crate::broadcaster(writer, self) + } } impl WatchStreamExt for St where St: Stream {} From 6bf37fab5d3309f867ececf66d31762ff787493e Mon Sep 17 00:00:00 2001 From: Danil-Grigorev Date: Wed, 26 Feb 2025 13:47:51 +0100 Subject: [PATCH 04/16] Gate behind unstable-runtime-subscribe feature Signed-off-by: Danil-Grigorev --- kube-runtime/src/controller/mod.rs | 4 +-- kube-runtime/src/reflector/dispatcher.rs | 28 ++++++++++++------- kube-runtime/src/reflector/mod.rs | 6 ++-- .../src/reflector/multi_dispatcher.rs | 6 ++-- kube-runtime/src/reflector/store.rs | 2 +- 5 files changed, 28 insertions(+), 18 deletions(-) diff --git a/kube-runtime/src/controller/mod.rs b/kube-runtime/src/controller/mod.rs index 68eb2d8c5..7d1a32bca 100644 --- a/kube-runtime/src/controller/mod.rs +++ b/kube-runtime/src/controller/mod.rs @@ -1698,13 +1698,13 @@ mod tests { use super::{Action, APPLIER_REQUEUE_BUF_SIZE}; use crate::{ applier, - reflector::{self, store::CacheWriter as _, ObjectRef}, + reflector::{self, ObjectRef}, watcher::{self, metadata_watcher, watcher, Event}, Config, Controller, }; use futures::{Stream, StreamExt, TryStreamExt}; use k8s_openapi::api::core::v1::ConfigMap; - use kube_client::{api::TypeMeta, core::ObjectMeta, Api, Resource}; + use kube_client::{core::ObjectMeta, Api, Resource}; use serde::de::DeserializeOwned; use tokio::time::timeout; diff --git a/kube-runtime/src/reflector/dispatcher.rs b/kube-runtime/src/reflector/dispatcher.rs index 80e32514d..43b3641be 100644 --- a/kube-runtime/src/reflector/dispatcher.rs +++ b/kube-runtime/src/reflector/dispatcher.rs @@ -6,18 +6,21 @@ use std::{fmt::Debug, sync::Arc}; use educe::Educe; use futures::Stream; +#[cfg(feature = "unstable-runtime-subscribe")] use kube_client::{api::DynamicObject, Resource}; use pin_project::pin_project; +#[cfg(feature = "unstable-runtime-subscribe")] use serde::de::DeserializeOwned; use std::task::ready; -use crate::{ - reflector::{ObjectRef, Store}, - watcher::{self, Event}, -}; +use crate::reflector::{ObjectRef, Store}; +#[cfg(feature = "unstable-runtime-subscribe")] +use crate::watcher::Event; use async_broadcast::{InactiveReceiver, Receiver, Sender}; -use super::{store::Writer, Lookup}; +#[cfg(feature = "unstable-runtime-subscribe")] +use super::store::Writer; +use super::Lookup; #[derive(Educe)] #[educe(Debug(bound("K: Debug, K::DynamicType: Debug")), Clone)] @@ -77,17 +80,19 @@ where } } -#[derive(Clone)] // A helper type that holds a broadcast transmitter and a broadcast receiver, // used to fan-out events from a root stream to multiple listeners. -pub(crate) struct TypedDispatcher { +#[cfg(feature = "unstable-runtime-subscribe")] +#[derive(Clone)] +pub(crate) struct DynamicDispatcher { dispatch_tx: Sender>, // An inactive reader that prevents the channel from closing until the // writer is dropped. _dispatch_rx: InactiveReceiver>, } -impl TypedDispatcher { +#[cfg(feature = "unstable-runtime-subscribe")] +impl DynamicDispatcher { /// Creates and returns a new self that wraps a broadcast sender and an /// inactive broadcast receiver /// @@ -98,7 +103,7 @@ impl TypedDispatcher { // // N.B messages are eagerly broadcasted, meaning no active receivers are // required for a message to be broadcasted. - pub(crate) fn new(buf_size: usize) -> TypedDispatcher { + pub(crate) fn new(buf_size: usize) -> DynamicDispatcher { // Create a broadcast (tx, rx) pair let (mut dispatch_tx, dispatch_rx) = async_broadcast::broadcast(buf_size); // The tx half will not wait for any receivers to be active before @@ -210,6 +215,7 @@ where /// subscribed to the shared stream will also terminate after all events yielded by /// the root stream have been observed. This means [`TypedReflectHandle`] streams /// can still be polled after the root stream has been dropped. +#[cfg(feature = "unstable-runtime-subscribe")] #[pin_project] pub struct TypedReflectHandle where @@ -222,6 +228,7 @@ where store: Writer, } +#[cfg(feature = "unstable-runtime-subscribe")] impl TypedReflectHandle where K: Lookup + Clone + 'static, @@ -234,7 +241,7 @@ where // Initialize a ready store by default store: { let mut store: Writer = Default::default(); - store.apply_shared_watcher_event(&watcher::Event::InitDone); + store.apply_shared_watcher_event(&Event::InitDone); store }, } @@ -245,6 +252,7 @@ where } } +#[cfg(feature = "unstable-runtime-subscribe")] impl Stream for TypedReflectHandle where K: Resource + Clone + 'static, diff --git a/kube-runtime/src/reflector/mod.rs b/kube-runtime/src/reflector/mod.rs index 479516829..7296ebe40 100644 --- a/kube-runtime/src/reflector/mod.rs +++ b/kube-runtime/src/reflector/mod.rs @@ -1,9 +1,9 @@ //! Caches objects in memory mod dispatcher; -mod object_ref; #[cfg(feature = "unstable-runtime-subscribe")] pub mod multi_dispatcher; +mod object_ref; pub mod store; pub use self::{ @@ -13,11 +13,13 @@ pub use self::{ use crate::watcher; use async_stream::stream; use futures::{Stream, StreamExt}; +#[cfg(feature = "unstable-runtime-subscribe")] use kube_client::api::DynamicObject; #[cfg(feature = "unstable-runtime-subscribe")] use multi_dispatcher::MultiDispatcher; use std::hash::Hash; -#[cfg(feature = "unstable-runtime-subscribe")] pub use store::store_shared; +#[cfg(feature = "unstable-runtime-subscribe")] +pub use store::store_shared; pub use store::{store, Store}; /// Cache objects from a [`watcher()`] stream into a local [`Store`] diff --git a/kube-runtime/src/reflector/multi_dispatcher.rs b/kube-runtime/src/reflector/multi_dispatcher.rs index f679e16d8..22a7d57e6 100644 --- a/kube-runtime/src/reflector/multi_dispatcher.rs +++ b/kube-runtime/src/reflector/multi_dispatcher.rs @@ -9,12 +9,12 @@ use serde::de::DeserializeOwned; use crate::watcher; use super::{ - dispatcher::{TypedDispatcher, TypedReflectHandle}, Store, + dispatcher::{DynamicDispatcher, TypedReflectHandle}, Store, }; #[derive(Clone)] pub struct MultiDispatcher { - dispatcher: TypedDispatcher, + dispatcher: DynamicDispatcher, } impl MultiDispatcher { @@ -22,7 +22,7 @@ impl MultiDispatcher { pub fn new(buf_size: usize) -> Self { Self { // store: Default::default(), - dispatcher: TypedDispatcher::new(buf_size), + dispatcher: DynamicDispatcher::new(buf_size), } } diff --git a/kube-runtime/src/reflector/store.rs b/kube-runtime/src/reflector/store.rs index a55837732..7b5a11b2e 100644 --- a/kube-runtime/src/reflector/store.rs +++ b/kube-runtime/src/reflector/store.rs @@ -332,7 +332,7 @@ where mod tests { use super::{store, Writer}; use crate::{ - reflector::{store::CacheWriter as _, ObjectRef}, + reflector::ObjectRef, watcher, }; use k8s_openapi::api::core::v1::ConfigMap; From 6a1cc7ed396e18c9b7e36592bd554e67046b08ac Mon Sep 17 00:00:00 2001 From: Danil-Grigorev Date: Wed, 26 Feb 2025 13:50:20 +0100 Subject: [PATCH 05/16] Lint fixes Signed-off-by: Danil-Grigorev --- examples/broadcast_reflector.rs | 5 ++++- kube-core/src/dynamic.rs | 3 ++- kube-core/src/object.rs | 7 +++---- kube-runtime/src/lib.rs | 3 +-- kube-runtime/src/reflector/dispatcher.rs | 9 +++------ kube-runtime/src/reflector/mod.rs | 6 ++---- kube-runtime/src/reflector/multi_dispatcher.rs | 8 +++----- kube-runtime/src/reflector/store.rs | 5 +---- 8 files changed, 19 insertions(+), 27 deletions(-) diff --git a/examples/broadcast_reflector.rs b/examples/broadcast_reflector.rs index a00f9c133..09abd704f 100644 --- a/examples/broadcast_reflector.rs +++ b/examples/broadcast_reflector.rs @@ -5,7 +5,10 @@ use k8s_openapi::api::{ }; use kube::{ api::ApiResource, - runtime::{controller::Action, reflector::multi_dispatcher::MultiDispatcher, watcher, Controller, WatchStreamExt as _}, + runtime::{ + controller::Action, reflector::multi_dispatcher::MultiDispatcher, watcher, Controller, + WatchStreamExt as _, + }, Api, Client, ResourceExt, }; use std::{fmt::Debug, sync::Arc, time::Duration}; diff --git a/kube-core/src/dynamic.rs b/kube-core/src/dynamic.rs index f07c5e709..9f82ff248 100644 --- a/kube-core/src/dynamic.rs +++ b/kube-core/src/dynamic.rs @@ -4,7 +4,8 @@ pub use crate::discovery::ApiResource; use crate::{ metadata::TypeMeta, - resource::{DynamicResourceScope, Resource}, GroupVersionKind, + resource::{DynamicResourceScope, Resource}, + GroupVersionKind, }; use k8s_openapi::apimachinery::pkg::apis::meta::v1::ObjectMeta; diff --git a/kube-core/src/object.rs b/kube-core/src/object.rs index a7d62d07c..0a8d10e28 100644 --- a/kube-core/src/object.rs +++ b/kube-core/src/object.rs @@ -399,10 +399,9 @@ mod test { assert_eq!(mypod.namespace().unwrap(), "dev"); assert_eq!(mypod.name_unchecked(), "blog"); assert!(mypod.status().is_none()); - assert_eq!( - mypod.spec().containers[0], - ContainerSimple { image: "blog".into() } - ); + assert_eq!(mypod.spec().containers[0], ContainerSimple { + image: "blog".into() + }); assert_eq!(PodSimple::api_version(&ar), "v1"); assert_eq!(PodSimple::version(&ar), "v1"); diff --git a/kube-runtime/src/lib.rs b/kube-runtime/src/lib.rs index 1ba4b091d..8c6669656 100644 --- a/kube-runtime/src/lib.rs +++ b/kube-runtime/src/lib.rs @@ -32,9 +32,8 @@ pub mod watcher; pub use controller::{applier, Config, Controller}; pub use finalizer::finalizer; +#[cfg(feature = "unstable-runtime-subscribe")] pub use reflector::broadcaster; pub use reflector::reflector; -#[cfg(feature = "unstable-runtime-subscribe")] -pub use reflector::broadcaster; pub use scheduler::scheduler; pub use utils::WatchStreamExt; pub use watcher::{metadata_watcher, watcher}; diff --git a/kube-runtime/src/reflector/dispatcher.rs b/kube-runtime/src/reflector/dispatcher.rs index 43b3641be..7a0be3fec 100644 --- a/kube-runtime/src/reflector/dispatcher.rs +++ b/kube-runtime/src/reflector/dispatcher.rs @@ -9,17 +9,14 @@ use futures::Stream; #[cfg(feature = "unstable-runtime-subscribe")] use kube_client::{api::DynamicObject, Resource}; use pin_project::pin_project; -#[cfg(feature = "unstable-runtime-subscribe")] -use serde::de::DeserializeOwned; +#[cfg(feature = "unstable-runtime-subscribe")] use serde::de::DeserializeOwned; use std::task::ready; use crate::reflector::{ObjectRef, Store}; -#[cfg(feature = "unstable-runtime-subscribe")] -use crate::watcher::Event; +#[cfg(feature = "unstable-runtime-subscribe")] use crate::watcher::Event; use async_broadcast::{InactiveReceiver, Receiver, Sender}; -#[cfg(feature = "unstable-runtime-subscribe")] -use super::store::Writer; +#[cfg(feature = "unstable-runtime-subscribe")] use super::store::Writer; use super::Lookup; #[derive(Educe)] diff --git a/kube-runtime/src/reflector/mod.rs b/kube-runtime/src/reflector/mod.rs index 7296ebe40..a02204369 100644 --- a/kube-runtime/src/reflector/mod.rs +++ b/kube-runtime/src/reflector/mod.rs @@ -1,8 +1,7 @@ //! Caches objects in memory mod dispatcher; -#[cfg(feature = "unstable-runtime-subscribe")] -pub mod multi_dispatcher; +#[cfg(feature = "unstable-runtime-subscribe")] pub mod multi_dispatcher; mod object_ref; pub mod store; @@ -18,8 +17,7 @@ use kube_client::api::DynamicObject; #[cfg(feature = "unstable-runtime-subscribe")] use multi_dispatcher::MultiDispatcher; use std::hash::Hash; -#[cfg(feature = "unstable-runtime-subscribe")] -pub use store::store_shared; +#[cfg(feature = "unstable-runtime-subscribe")] pub use store::store_shared; pub use store::{store, Store}; /// Cache objects from a [`watcher()`] stream into a local [`Store`] diff --git a/kube-runtime/src/reflector/multi_dispatcher.rs b/kube-runtime/src/reflector/multi_dispatcher.rs index 22a7d57e6..abfe33ac6 100644 --- a/kube-runtime/src/reflector/multi_dispatcher.rs +++ b/kube-runtime/src/reflector/multi_dispatcher.rs @@ -1,15 +1,13 @@ use std::hash::Hash; -use kube_client::{ - api::DynamicObject, - Resource, -}; +use kube_client::{api::DynamicObject, Resource}; use serde::de::DeserializeOwned; use crate::watcher; use super::{ - dispatcher::{DynamicDispatcher, TypedReflectHandle}, Store, + dispatcher::{DynamicDispatcher, TypedReflectHandle}, + Store, }; #[derive(Clone)] diff --git a/kube-runtime/src/reflector/store.rs b/kube-runtime/src/reflector/store.rs index 7b5a11b2e..f88662063 100644 --- a/kube-runtime/src/reflector/store.rs +++ b/kube-runtime/src/reflector/store.rs @@ -331,10 +331,7 @@ where #[cfg(test)] mod tests { use super::{store, Writer}; - use crate::{ - reflector::ObjectRef, - watcher, - }; + use crate::{reflector::ObjectRef, watcher}; use k8s_openapi::api::core::v1::ConfigMap; use kube_client::api::ObjectMeta; From 144eb080c885fe49a06ca60dde9cbb8d3a6b62be Mon Sep 17 00:00:00 2001 From: Danil-Grigorev Date: Wed, 26 Feb 2025 13:57:32 +0100 Subject: [PATCH 06/16] Addressing review comments Signed-off-by: Danil-Grigorev --- examples/Cargo.toml | 1 - kube-core/src/metadata.rs | 6 +++--- kube-core/src/object.rs | 2 +- 3 files changed, 4 insertions(+), 5 deletions(-) diff --git a/examples/Cargo.toml b/examples/Cargo.toml index 01b20200b..6022f44e7 100644 --- a/examples/Cargo.toml +++ b/examples/Cargo.toml @@ -22,7 +22,6 @@ ws = ["kube/ws"] latest = ["k8s-openapi/latest"] [dev-dependencies] -parking_lot.workspace = true tokio-util.workspace = true assert-json-diff.workspace = true garde = { version = "0.22.0", default-features = false, features = ["derive"] } diff --git a/kube-core/src/metadata.rs b/kube-core/src/metadata.rs index 44e0a750d..0c12b9f5b 100644 --- a/kube-core/src/metadata.rs +++ b/kube-core/src/metadata.rs @@ -60,10 +60,10 @@ impl TypeMeta { /// /// let mut type_meta = TypeMeta::resource::(); /// type_meta.kind = "PodList".to_string(); - /// assert_eq!(type_meta.clone().singular().kind, "Pod"); - /// assert_eq!(type_meta.clone().singular().api_version, "v1"); + /// assert_eq!(type_meta.clone().singular_list().unwrap().kind, "Pod"); + /// assert_eq!(type_meta.clone().singular_list().unwrap().api_version, "v1"); /// ``` - pub fn singular(self) -> Option { + pub fn singular_list(self) -> Option { let kind = self.kind.strip_suffix("List")?.to_string(); (!kind.is_empty()).then_some(Self { kind, ..self }) } diff --git a/kube-core/src/object.rs b/kube-core/src/object.rs index 0a8d10e28..c9e2a25f8 100644 --- a/kube-core/src/object.rs +++ b/kube-core/src/object.rs @@ -161,7 +161,7 @@ impl<'de, T: DeserializeOwned + Clone> serde::Deserialize<'de> for ObjectList let mut resources = vec![]; for o in items.iter_mut() { if o.types.is_none() { - o.types = types.clone().singular(); + o.types = types.clone().singular_list(); } let item = serde_json::to_value(o).map_err(de::Error::custom)?; resources.push(serde_json::from_value(item).map_err(de::Error::custom)?) From 7e46bd7ad028c2eaaa4efc655d836aeb4e377319 Mon Sep 17 00:00:00 2001 From: Danil-Grigorev Date: Wed, 26 Feb 2025 19:06:38 +0100 Subject: [PATCH 07/16] Wrap stream in Mutex for shared use Signed-off-by: Danil-Grigorev --- examples/broadcast_reflector.rs | 22 +++++++++++----------- kube-runtime/src/reflector/mod.rs | 28 ++++++++++++++++++++++------ kube-runtime/src/utils/watch_ext.rs | 12 ++++-------- 3 files changed, 37 insertions(+), 25 deletions(-) diff --git a/examples/broadcast_reflector.rs b/examples/broadcast_reflector.rs index 09abd704f..98281b7cb 100644 --- a/examples/broadcast_reflector.rs +++ b/examples/broadcast_reflector.rs @@ -1,4 +1,4 @@ -use futures::{future, stream, StreamExt}; +use futures::{future, pin_mut, stream, StreamExt}; use k8s_openapi::api::{ apps::v1::Deployment, core::v1::{ConfigMap, Secret}, @@ -6,13 +6,13 @@ use k8s_openapi::api::{ use kube::{ api::ApiResource, runtime::{ - controller::Action, reflector::multi_dispatcher::MultiDispatcher, watcher, Controller, - WatchStreamExt as _, + broadcaster, controller::Action, reflector::multi_dispatcher::MultiDispatcher, watcher, Controller, }, Api, Client, ResourceExt, }; -use std::{fmt::Debug, sync::Arc, time::Duration}; +use std::{fmt::Debug, pin::pin, sync::Arc, time::Duration}; use thiserror::Error; +use tokio::sync::Mutex; use tracing::*; #[derive(Debug, Error)] @@ -42,8 +42,10 @@ async fn main() -> anyhow::Result<()> { let writer = MultiDispatcher::new(128); // multireflector stream - let mut combo_stream = stream::select_all(vec![]); - combo_stream.push( + let combo_stream = Arc::new(Mutex::new(stream::select_all(vec![]))); + let watcher = broadcaster(writer.clone(), combo_stream.clone()); + + combo_stream.lock().await.push( watcher::watcher( Api::all_with(client.clone(), &ApiResource::erase::(&())), Default::default(), @@ -52,7 +54,7 @@ async fn main() -> anyhow::Result<()> { ); // watching config maps, but ignoring in the final configuration - combo_stream.push( + combo_stream.lock().await.push( watcher::watcher( Api::all_with(client.clone(), &ApiResource::erase::(&())), Default::default(), @@ -61,14 +63,14 @@ async fn main() -> anyhow::Result<()> { ); // Combine duplicate type streams with narrowed down selection - combo_stream.push( + combo_stream.lock().await.push( watcher::watcher( Api::default_namespaced_with(client.clone(), &ApiResource::erase::(&())), Default::default(), ) .boxed(), ); - combo_stream.push( + combo_stream.lock().await.push( watcher::watcher( Api::namespaced_with(client.clone(), "kube-system", &ApiResource::erase::(&())), Default::default(), @@ -76,8 +78,6 @@ async fn main() -> anyhow::Result<()> { .boxed(), ); - let watcher = combo_stream.broadcast_shared(writer.clone()); - let (sub, reader) = writer.subscribe::(); let deploy = Controller::for_shared_stream(sub, reader) .shutdown_on_signal() diff --git a/kube-runtime/src/reflector/mod.rs b/kube-runtime/src/reflector/mod.rs index a02204369..d9c6bdb6b 100644 --- a/kube-runtime/src/reflector/mod.rs +++ b/kube-runtime/src/reflector/mod.rs @@ -1,9 +1,14 @@ //! Caches objects in memory mod dispatcher; -#[cfg(feature = "unstable-runtime-subscribe")] pub mod multi_dispatcher; +#[cfg(feature = "unstable-runtime-subscribe")] +pub mod multi_dispatcher; mod object_ref; pub mod store; +#[cfg(feature = "unstable-runtime-subscribe")] +use std::sync::Arc; +#[cfg(feature = "unstable-runtime-subscribe")] +use tokio::sync::Mutex; pub use self::{ dispatcher::ReflectHandle, @@ -11,15 +16,27 @@ pub use self::{ }; use crate::watcher; use async_stream::stream; +#[cfg(feature = "unstable-runtime-subscribe")] +use futures::stream::SelectAll; use futures::{Stream, StreamExt}; #[cfg(feature = "unstable-runtime-subscribe")] use kube_client::api::DynamicObject; #[cfg(feature = "unstable-runtime-subscribe")] use multi_dispatcher::MultiDispatcher; use std::hash::Hash; -#[cfg(feature = "unstable-runtime-subscribe")] pub use store::store_shared; +#[cfg(feature = "unstable-runtime-subscribe")] +use std::pin::Pin; +#[cfg(feature = "unstable-runtime-subscribe")] +pub use store::store_shared; pub use store::{store, Store}; +#[cfg(feature = "unstable-runtime-subscribe")] +/// Type for a shared stream of dynamic objects, which can be provided to [`broadcaster`] +/// wrapped as [`Arc>`], this type can be stored in context and provided +/// to controllers, which can arbitrary modify existing event streams at runtime. +pub type DynamicStream = + SelectAll, watcher::Error>> + Send>>>; + /// Cache objects from a [`watcher()`] stream into a local [`Store`] /// /// Observes the raw `Stream` of [`watcher::Event`] objects, and modifies the cache. @@ -141,13 +158,12 @@ where // broadcaster uses a common stream of DynamicObject events to distribute to any subscribed typed watcher. #[cfg(feature = "unstable-runtime-subscribe")] -pub fn broadcaster(mut writer: MultiDispatcher, stream: W) -> impl Stream +pub fn broadcaster(mut writer: MultiDispatcher, stream: Arc>) -> impl Stream where - W: Stream>>, + W: Stream>> + Unpin, { - let mut stream = Box::pin(stream); stream! { - while let Some(event) = stream.next().await { + while let Some(event) = stream.lock().await.next().await { match event { Ok(ev) => { writer.broadcast_event(&ev).await; diff --git a/kube-runtime/src/utils/watch_ext.rs b/kube-runtime/src/utils/watch_ext.rs index 76a1f73ce..05b8e2c8b 100644 --- a/kube-runtime/src/utils/watch_ext.rs +++ b/kube-runtime/src/utils/watch_ext.rs @@ -17,6 +17,10 @@ use crate::{ }; #[cfg(feature = "unstable-runtime-subscribe")] use kube_client::api::DynamicObject; +#[cfg(feature = "unstable-runtime-subscribe")] +use std::sync::Arc; +#[cfg(feature = "unstable-runtime-subscribe")] +use tokio::sync::Mutex; use crate::watcher::DefaultBackoff; use futures::{Stream, TryStream}; @@ -275,14 +279,6 @@ pub trait WatchStreamExt: Stream { { crate::reflector(writer, self) } - - #[cfg(feature = "unstable-runtime-subscribe")] - fn broadcast_shared(self, writer: MultiDispatcher) -> impl Stream - where - Self: Stream>> + Sized, - { - crate::broadcaster(writer, self) - } } impl WatchStreamExt for St where St: Stream {} From 86a4636fd4317991775324dc12242d41342bec36 Mon Sep 17 00:00:00 2001 From: Danil-Grigorev Date: Thu, 27 Feb 2025 12:51:39 +0100 Subject: [PATCH 08/16] Fmt fixes Signed-off-by: Danil-Grigorev --- kube-runtime/src/reflector/mod.rs | 18 ++++++------------ kube-runtime/src/reflector/multi_dispatcher.rs | 1 - kube-runtime/src/utils/watch_ext.rs | 6 ++---- 3 files changed, 8 insertions(+), 17 deletions(-) diff --git a/kube-runtime/src/reflector/mod.rs b/kube-runtime/src/reflector/mod.rs index d9c6bdb6b..52f657c04 100644 --- a/kube-runtime/src/reflector/mod.rs +++ b/kube-runtime/src/reflector/mod.rs @@ -1,14 +1,11 @@ //! Caches objects in memory mod dispatcher; -#[cfg(feature = "unstable-runtime-subscribe")] -pub mod multi_dispatcher; +#[cfg(feature = "unstable-runtime-subscribe")] pub mod multi_dispatcher; mod object_ref; pub mod store; -#[cfg(feature = "unstable-runtime-subscribe")] -use std::sync::Arc; -#[cfg(feature = "unstable-runtime-subscribe")] -use tokio::sync::Mutex; +#[cfg(feature = "unstable-runtime-subscribe")] use std::sync::Arc; +#[cfg(feature = "unstable-runtime-subscribe")] use tokio::sync::Mutex; pub use self::{ dispatcher::ReflectHandle, @@ -16,18 +13,15 @@ pub use self::{ }; use crate::watcher; use async_stream::stream; -#[cfg(feature = "unstable-runtime-subscribe")] -use futures::stream::SelectAll; +#[cfg(feature = "unstable-runtime-subscribe")] use futures::stream::SelectAll; use futures::{Stream, StreamExt}; #[cfg(feature = "unstable-runtime-subscribe")] use kube_client::api::DynamicObject; #[cfg(feature = "unstable-runtime-subscribe")] use multi_dispatcher::MultiDispatcher; use std::hash::Hash; -#[cfg(feature = "unstable-runtime-subscribe")] -use std::pin::Pin; -#[cfg(feature = "unstable-runtime-subscribe")] -pub use store::store_shared; +#[cfg(feature = "unstable-runtime-subscribe")] use std::pin::Pin; +#[cfg(feature = "unstable-runtime-subscribe")] pub use store::store_shared; pub use store::{store, Store}; #[cfg(feature = "unstable-runtime-subscribe")] diff --git a/kube-runtime/src/reflector/multi_dispatcher.rs b/kube-runtime/src/reflector/multi_dispatcher.rs index abfe33ac6..99bebd605 100644 --- a/kube-runtime/src/reflector/multi_dispatcher.rs +++ b/kube-runtime/src/reflector/multi_dispatcher.rs @@ -19,7 +19,6 @@ impl MultiDispatcher { #[must_use] pub fn new(buf_size: usize) -> Self { Self { - // store: Default::default(), dispatcher: DynamicDispatcher::new(buf_size), } } diff --git a/kube-runtime/src/utils/watch_ext.rs b/kube-runtime/src/utils/watch_ext.rs index 05b8e2c8b..6ac1d133a 100644 --- a/kube-runtime/src/utils/watch_ext.rs +++ b/kube-runtime/src/utils/watch_ext.rs @@ -17,10 +17,8 @@ use crate::{ }; #[cfg(feature = "unstable-runtime-subscribe")] use kube_client::api::DynamicObject; -#[cfg(feature = "unstable-runtime-subscribe")] -use std::sync::Arc; -#[cfg(feature = "unstable-runtime-subscribe")] -use tokio::sync::Mutex; +#[cfg(feature = "unstable-runtime-subscribe")] use std::sync::Arc; +#[cfg(feature = "unstable-runtime-subscribe")] use tokio::sync::Mutex; use crate::watcher::DefaultBackoff; use futures::{Stream, TryStream}; From e31e42f045efdce64a6fc2519d7ba01446025f1f Mon Sep 17 00:00:00 2001 From: Danil-Grigorev Date: Thu, 27 Feb 2025 13:36:16 +0100 Subject: [PATCH 09/16] Refactor blocking loop Signed-off-by: Danil-Grigorev --- examples/broadcast_reflector.rs | 10 ++-- kube-runtime/src/reflector/mod.rs | 16 +++--- .../src/reflector/multi_dispatcher.rs | 50 ++++++++++++++++++- 3 files changed, 61 insertions(+), 15 deletions(-) diff --git a/examples/broadcast_reflector.rs b/examples/broadcast_reflector.rs index 98281b7cb..f21cbb1e8 100644 --- a/examples/broadcast_reflector.rs +++ b/examples/broadcast_reflector.rs @@ -1,4 +1,4 @@ -use futures::{future, pin_mut, stream, StreamExt}; +use futures::{future, lock::Mutex, pin_mut, stream, StreamExt}; use k8s_openapi::api::{ apps::v1::Deployment, core::v1::{ConfigMap, Secret}, @@ -6,13 +6,15 @@ use k8s_openapi::api::{ use kube::{ api::ApiResource, runtime::{ - broadcaster, controller::Action, reflector::multi_dispatcher::MultiDispatcher, watcher, Controller, + broadcaster, + controller::Action, + reflector::multi_dispatcher::{BroadcastStream, MultiDispatcher}, + watcher, Controller, }, Api, Client, ResourceExt, }; use std::{fmt::Debug, pin::pin, sync::Arc, time::Duration}; use thiserror::Error; -use tokio::sync::Mutex; use tracing::*; #[derive(Debug, Error)] @@ -43,7 +45,7 @@ async fn main() -> anyhow::Result<()> { // multireflector stream let combo_stream = Arc::new(Mutex::new(stream::select_all(vec![]))); - let watcher = broadcaster(writer.clone(), combo_stream.clone()); + let watcher = broadcaster(writer.clone(), BroadcastStream::new(combo_stream.clone())); combo_stream.lock().await.push( watcher::watcher( diff --git a/kube-runtime/src/reflector/mod.rs b/kube-runtime/src/reflector/mod.rs index 52f657c04..be0288956 100644 --- a/kube-runtime/src/reflector/mod.rs +++ b/kube-runtime/src/reflector/mod.rs @@ -18,19 +18,14 @@ use futures::{Stream, StreamExt}; #[cfg(feature = "unstable-runtime-subscribe")] use kube_client::api::DynamicObject; #[cfg(feature = "unstable-runtime-subscribe")] +use multi_dispatcher::BroadcastStream; +#[cfg(feature = "unstable-runtime-subscribe")] use multi_dispatcher::MultiDispatcher; use std::hash::Hash; #[cfg(feature = "unstable-runtime-subscribe")] use std::pin::Pin; #[cfg(feature = "unstable-runtime-subscribe")] pub use store::store_shared; pub use store::{store, Store}; -#[cfg(feature = "unstable-runtime-subscribe")] -/// Type for a shared stream of dynamic objects, which can be provided to [`broadcaster`] -/// wrapped as [`Arc>`], this type can be stored in context and provided -/// to controllers, which can arbitrary modify existing event streams at runtime. -pub type DynamicStream = - SelectAll, watcher::Error>> + Send>>>; - /// Cache objects from a [`watcher()`] stream into a local [`Store`] /// /// Observes the raw `Stream` of [`watcher::Event`] objects, and modifies the cache. @@ -152,12 +147,15 @@ where // broadcaster uses a common stream of DynamicObject events to distribute to any subscribed typed watcher. #[cfg(feature = "unstable-runtime-subscribe")] -pub fn broadcaster(mut writer: MultiDispatcher, stream: Arc>) -> impl Stream +pub fn broadcaster( + mut writer: MultiDispatcher, + mut stream: BroadcastStream, +) -> impl Stream where W: Stream>> + Unpin, { stream! { - while let Some(event) = stream.lock().await.next().await { + while let Some(event) = stream.next().await { match event { Ok(ev) => { writer.broadcast_event(&ev).await; diff --git a/kube-runtime/src/reflector/multi_dispatcher.rs b/kube-runtime/src/reflector/multi_dispatcher.rs index 99bebd605..f5a4538f6 100644 --- a/kube-runtime/src/reflector/multi_dispatcher.rs +++ b/kube-runtime/src/reflector/multi_dispatcher.rs @@ -1,5 +1,11 @@ -use std::hash::Hash; +use std::{ + hash::Hash, + pin::Pin, + sync::Arc, + task::{Context, Poll}, +}; +use futures::{lock::Mutex, FutureExt, Stream, StreamExt as _}; use kube_client::{api::DynamicObject, Resource}; use serde::de::DeserializeOwned; @@ -43,6 +49,46 @@ impl MultiDispatcher { /// Broadcast an event to any downstream listeners subscribed on the store pub(crate) async fn broadcast_event(&mut self, event: &watcher::Event) { - self.dispatcher.broadcast(event.clone()).await + match event { + watcher::Event::InitDone => {}, + ev => self.dispatcher.broadcast(ev.clone()).await, + } + } +} + +/// See [`Scheduler::hold`] +pub struct BroadcastStream { + pub stream: Arc>, +} + +impl Clone for BroadcastStream { + fn clone(&self) -> Self { + Self { + stream: self.stream.clone(), + } + } +} + +impl BroadcastStream +where + W: Stream>> + Unpin, +{ + pub fn new(stream: Arc>) -> Self { + Self { stream } + } +} + +impl Stream for BroadcastStream +where + W: Stream>> + Unpin, +{ + type Item = W::Item; + + fn poll_next(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + if let Some(mut stream) = self.stream.try_lock() { + return stream.poll_next_unpin(cx); + } + + Poll::Pending } } From 8fa74349073c93cbb205b134f06d1e46293ee50f Mon Sep 17 00:00:00 2001 From: Danil-Grigorev Date: Sun, 2 Mar 2025 00:43:48 +0100 Subject: [PATCH 10/16] Implement restart on watch configuration change Signed-off-by: Danil-Grigorev --- kube-runtime/src/watcher.rs | 359 ++++++++++++++++++++++++++---------- 1 file changed, 260 insertions(+), 99 deletions(-) diff --git a/kube-runtime/src/watcher.rs b/kube-runtime/src/watcher.rs index 755320b38..98195c333 100644 --- a/kube-runtime/src/watcher.rs +++ b/kube-runtime/src/watcher.rs @@ -2,20 +2,31 @@ //! //! See [`watcher`] for the primary entry point. -use crate::utils::{Backoff, ResetTimerBackoff}; +use crate::{ + reflector::multi_dispatcher::BroadcastStream, + utils::{Backoff, ResetTimerBackoff}, +}; +use async_stream::stream; use async_trait::async_trait; use backon::BackoffBuilder; use educe::Educe; -use futures::{stream::BoxStream, Stream, StreamExt}; +use futures::{ + channel::mpsc::Receiver, + future::BoxFuture, + lock::Mutex, + stream::{self, empty, select_all, select_with_strategy, BoxStream, PollNext}, + Stream, StreamExt, +}; use kube_client::{ api::{ListParams, Resource, ResourceExt, VersionMatch, WatchEvent, WatchParams}, core::{metadata::PartialObjectMeta, ObjectList, Selector}, error::ErrorResponse, Api, Error as ClientErr, }; +use pin_project::pin_project; use serde::de::DeserializeOwned; -use std::{clone::Clone, collections::VecDeque, fmt::Debug, future, time::Duration}; +use std::{clone::Clone, collections::VecDeque, fmt::Debug, future, pin::Pin, sync::Arc, time::Duration}; use thiserror::Error; use tracing::{debug, error, warn}; @@ -146,6 +157,7 @@ enum State { InitialWatch { #[educe(Debug(ignore))] stream: BoxStream<'static, kube_client::Result>>, + params: WatchParams, }, /// The initial LIST was successful, so we should move on to starting the actual watch. InitListed { resource_version: String }, @@ -159,6 +171,7 @@ enum State { resource_version: String, #[educe(Debug(ignore))] stream: BoxStream<'static, kube_client::Result>>, + params: WatchParams, }, } @@ -175,7 +188,6 @@ trait ApiMode { version: &str, ) -> kube_client::Result>>>; } - /// A wrapper around the `Api` of a `Resource` type that when used by the /// watcher will return the entire (full) object struct FullObject<'a, K> { @@ -452,6 +464,7 @@ where wp: &WatchParams, version: &str, ) -> kube_client::Result>>> { + let x = || async { self.api.watch(wp, version).await }; self.api.watch(wp, version).await.map(StreamExt::boxed) } } @@ -482,6 +495,59 @@ where } } +/// A wrapper around the `Api` of a `Resource` type that when used by the +/// watcher will return only the metadata associated with an object +#[pin_project] +struct Interruptable<'a, K> { + api: &'a Api, + #[pin] + interrupt: Receiver<()>, +} + +// /// Type to tell [`SelectWithStrategy`] which stream to poll next. +// #[derive(Debug, PartialEq, Eq, Copy, Clone, Hash, Default)] +// pub enum Interrupt { +// /// Poll the first stream. +// #[default] +// Regular, +// /// Poll the second stream once. +// Restart, +// } + +// impl Interrupt { +// fn propagate(&mut self) -> PollNext { +// match self { +// Interrupt::Regular => PollNext::Left, +// Interrupt::Restart => { +// *self = Interrupt::Regular; +// PollNext::Right +// } +// } +// } +// } + +// #[async_trait] +// impl ApiMode for Interruptable<'_, K> +// where +// K: Clone + Debug + DeserializeOwned + Send + 'static, +// { +// type Value = K; + +// async fn list(&self, lp: &ListParams) -> kube_client::Result> { +// self.api.list(lp).await +// } + +// async fn watch( +// &self, +// wp: &WatchParams, +// version: &str, +// ) -> kube_client::Result>>> { +// let stream = self.api.watch(wp, version).await.map(StreamExt::boxed)?; + +// Ok(Box::pin(stream.restart_on(Box::pin(self.interrupt)))) +// } +// } + /// Progresses the watcher a single step, returning (event, state) /// /// This function should be trampolined: if event == `None` @@ -494,26 +560,32 @@ async fn step_trampolined( ) -> (Option>>, State) where A: ApiMode, - A::Value: Resource + 'static, + A::Value: Resource + 'static + Send, { match state { State::Empty => match wc.initial_list_strategy { - InitialListStrategy::ListWatch => (Some(Ok(Event::Init)), State::InitPage { - continue_token: None, - objects: VecDeque::default(), - last_bookmark: None, - }), - InitialListStrategy::StreamingList => match api.watch(&wc.to_watch_params(), "0").await { - Ok(stream) => (None, State::InitialWatch { stream }), - Err(err) => { - if std::matches!(err, ClientErr::Api(ErrorResponse { code: 403, .. })) { - warn!("watch initlist error with 403: {err:?}"); - } else { - debug!("watch initlist error: {err:?}"); + InitialListStrategy::ListWatch => ( + Some(Ok(Event::Init)), + State::InitPage { + continue_token: None, + objects: VecDeque::default(), + last_bookmark: None, + }, + ), + InitialListStrategy::StreamingList => { + let params = wc.to_watch_params(); + match api.watch(¶ms, "0").await { + Ok(stream) => (None, State::InitialWatch { stream, params }), + Err(err) => { + if std::matches!(err, ClientErr::Api(ErrorResponse { code: 403, .. })) { + warn!("watch initlist error with 403: {err:?}"); + } else { + debug!("watch initlist error: {err:?}"); + } + (Some(Err(Error::WatchStartFailed(err))), State::default()) } - (Some(Err(Error::WatchStartFailed(err))), State::default()) } - }, + } }, State::InitPage { continue_token, @@ -521,11 +593,14 @@ where last_bookmark, } => { if let Some(next) = objects.pop_front() { - return (Some(Ok(Event::InitApply(next))), State::InitPage { - continue_token, - objects, - last_bookmark, - }); + return ( + Some(Ok(Event::InitApply(next))), + State::InitPage { + continue_token, + objects, + last_bookmark, + }, + ); } // check if we need to perform more pages if continue_token.is_none() { @@ -545,11 +620,14 @@ where } // Buffer page here, causing us to return to this enum branch (State::InitPage) // until the objects buffer has drained - (None, State::InitPage { - continue_token, - objects: list.items.into_iter().collect(), - last_bookmark, - }) + ( + None, + State::InitPage { + continue_token, + objects: list.items.into_iter().collect(), + last_bookmark, + }, + ) } Err(err) => { if std::matches!(err, ClientErr::Api(ErrorResponse { code: 403, .. })) { @@ -561,26 +639,42 @@ where } } } - State::InitialWatch { mut stream } => { + State::InitialWatch { mut stream, params } => { + if params != wc.to_watch_params() { + return ( + Some(Err(Error::WatchError(kube_client::core::ErrorResponse { + status: "Restarting".into(), + message: "Watches are restarting due to selector change".into(), + reason: "Restart".into(), + code: 410, + }))), + State::default(), + ); + } match stream.next().await { - Some(Ok(WatchEvent::Added(obj) | WatchEvent::Modified(obj))) => { - (Some(Ok(Event::InitApply(obj))), State::InitialWatch { stream }) - } + Some(Ok(WatchEvent::Added(obj) | WatchEvent::Modified(obj))) => ( + Some(Ok(Event::InitApply(obj))), + State::InitialWatch { stream, params }, + ), Some(Ok(WatchEvent::Deleted(_obj))) => { // Kubernetes claims these events are impossible // https://kubernetes.io/docs/reference/using-api/api-concepts/#streaming-lists error!("got deleted event during initial watch. this is a bug"); - (None, State::InitialWatch { stream }) + (None, State::InitialWatch { stream, params }) } Some(Ok(WatchEvent::Bookmark(bm))) => { let marks_initial_end = bm.metadata.annotations.contains_key("k8s.io/initial-events-end"); if marks_initial_end { - (Some(Ok(Event::InitDone)), State::Watching { - resource_version: bm.metadata.resource_version, - stream, - }) + ( + Some(Ok(Event::InitDone)), + State::Watching { + resource_version: bm.metadata.resource_version, + stream, + params, + }, + ) } else { - (None, State::InitialWatch { stream }) + (None, State::InitialWatch { stream, params }) } } Some(Ok(WatchEvent::Error(err))) => { @@ -588,7 +682,7 @@ where let new_state = if err.code == 410 { State::default() } else { - State::InitialWatch { stream } + State::InitialWatch { stream, params } }; if err.code == 403 { warn!("watcher watchevent error 403: {err:?}"); @@ -603,89 +697,129 @@ where } else { debug!("watcher error: {err:?}"); } - (Some(Err(Error::WatchFailed(err))), State::InitialWatch { stream }) + ( + Some(Err(Error::WatchFailed(err))), + State::InitialWatch { stream, params }, + ) } None => (None, State::default()), } } State::InitListed { resource_version } => { - match api.watch(&wc.to_watch_params(), &resource_version).await { - Ok(stream) => (None, State::Watching { - resource_version, - stream, - }), + let params = wc.to_watch_params(); + match api.watch(¶ms, &resource_version).await { + Ok(stream) => ( + None, + State::Watching { + resource_version, + stream, + params, + }, + ), Err(err) => { if std::matches!(err, ClientErr::Api(ErrorResponse { code: 403, .. })) { warn!("watch initlist error with 403: {err:?}"); } else { debug!("watch initlist error: {err:?}"); } - (Some(Err(Error::WatchStartFailed(err))), State::InitListed { - resource_version, - }) + ( + Some(Err(Error::WatchStartFailed(err))), + State::InitListed { resource_version }, + ) } } } State::Watching { resource_version, mut stream, - } => match stream.next().await { - Some(Ok(WatchEvent::Added(obj) | WatchEvent::Modified(obj))) => { - let resource_version = obj.resource_version().unwrap_or_default(); - if resource_version.is_empty() { - (Some(Err(Error::NoResourceVersion)), State::default()) - } else { - (Some(Ok(Event::Apply(obj))), State::Watching { - resource_version, - stream, - }) - } + params, + } => { + if params != wc.to_watch_params() { + return ( + Some(Err(Error::WatchError(kube_client::core::ErrorResponse { + status: "Restarting".into(), + message: "Watches are restarting due to selector change".into(), + reason: "Restart".into(), + code: 410, + }))), + State::default(), + ); } - Some(Ok(WatchEvent::Deleted(obj))) => { - let resource_version = obj.resource_version().unwrap_or_default(); - if resource_version.is_empty() { - (Some(Err(Error::NoResourceVersion)), State::default()) - } else { - (Some(Ok(Event::Delete(obj))), State::Watching { - resource_version, - stream, - }) + match stream.next().await { + Some(Ok(WatchEvent::Added(obj) | WatchEvent::Modified(obj))) => { + let resource_version = obj.resource_version().unwrap_or_default(); + if resource_version.is_empty() { + (Some(Err(Error::NoResourceVersion)), State::default()) + } else { + ( + Some(Ok(Event::Apply(obj))), + State::Watching { + resource_version, + stream, + params, + }, + ) + } } - } - Some(Ok(WatchEvent::Bookmark(bm))) => (None, State::Watching { - resource_version: bm.metadata.resource_version, - stream, - }), - Some(Ok(WatchEvent::Error(err))) => { - // HTTP GONE, means we have desynced and need to start over and re-list :( - let new_state = if err.code == 410 { - State::default() - } else { + Some(Ok(WatchEvent::Deleted(obj))) => { + let resource_version = obj.resource_version().unwrap_or_default(); + if resource_version.is_empty() { + (Some(Err(Error::NoResourceVersion)), State::default()) + } else { + ( + Some(Ok(Event::Delete(obj))), + State::Watching { + resource_version, + stream, + params, + }, + ) + } + } + Some(Ok(WatchEvent::Bookmark(bm))) => ( + None, State::Watching { - resource_version, + resource_version: bm.metadata.resource_version, stream, + params, + }, + ), + Some(Ok(WatchEvent::Error(err))) => { + // HTTP GONE, means we have desynced and need to start over and re-list :( + let new_state = if err.code == 410 { + State::default() + } else { + State::Watching { + resource_version, + stream, + params, + } + }; + if err.code == 403 { + warn!("watcher watchevent error 403: {err:?}"); + } else { + debug!("error watchevent error: {err:?}"); } - }; - if err.code == 403 { - warn!("watcher watchevent error 403: {err:?}"); - } else { - debug!("error watchevent error: {err:?}"); + (Some(Err(Error::WatchError(err))), new_state) } - (Some(Err(Error::WatchError(err))), new_state) - } - Some(Err(err)) => { - if std::matches!(err, ClientErr::Api(ErrorResponse { code: 403, .. })) { - warn!("watcher error 403: {err:?}"); - } else { - debug!("watcher error: {err:?}"); + Some(Err(err)) => { + if std::matches!(err, ClientErr::Api(ErrorResponse { code: 403, .. })) { + warn!("watcher error 403: {err:?}"); + } else { + debug!("watcher error: {err:?}"); + } + ( + Some(Err(Error::WatchFailed(err))), + State::Watching { + resource_version, + stream, + params, + }, + ) } - (Some(Err(Error::WatchFailed(err))), State::Watching { - resource_version, - stream, - }) + None => (None, State::InitListed { resource_version }), } - None => (None, State::InitListed { resource_version }), - }, + } } } @@ -697,7 +831,7 @@ async fn step( ) -> (Result>, State) where A: ApiMode, - A::Value: Resource + 'static, + A::Value: Resource + 'static + Send, { loop { match step_trampolined(api, config, state).await { @@ -769,6 +903,33 @@ pub fn watcher( ) } +// pub fn watcher_restart( +// api: Api, +// watcher_config: Config, +// trigger: impl Stream + Send + Sync + 'static + Clone +// ) -> impl Stream>> + Send { +// futures::stream::unfold( +// (api, watcher_config, State::default(), trigger), +// |(api, watcher_config, state, trigger)| async move { +// let (event, state) = { +// let api = FullObject { api: &api }; +// let config: &Config = &watcher_config; +// let mut state = state; +// let trigger = trigger.clone(); +// async move { +// loop { +// match step_trampolined(&api, config, state, trigger).await { +// (Some(result), new_state) => return (result, new_state), +// (None, new_state) => state = new_state, +// } +// } +// } +// }.await; +// Some((event, (api, watcher_config, state, trigger))) +// }, +// ) +// } + /// Watches a Kubernetes Resource for changes continuously and receives only the /// metadata /// From 8770710c629254fe5ebea6e8576991ffae8bcc50 Mon Sep 17 00:00:00 2001 From: Danil-Grigorev Date: Mon, 3 Mar 2025 21:55:46 +0100 Subject: [PATCH 11/16] Perform typed event filtering inside stream impl Signed-off-by: Danil-Grigorev --- kube-runtime/src/controller/mod.rs | 33 +- kube-runtime/src/reflector/dispatcher.rs | 47 ++- .../src/reflector/multi_dispatcher.rs | 3 +- kube-runtime/src/watcher.rs | 359 +++++------------- 4 files changed, 137 insertions(+), 305 deletions(-) diff --git a/kube-runtime/src/controller/mod.rs b/kube-runtime/src/controller/mod.rs index 7d1a32bca..8a8f7fc3a 100644 --- a/kube-runtime/src/controller/mod.rs +++ b/kube-runtime/src/controller/mod.rs @@ -851,10 +851,7 @@ where /// } /// # } #[cfg(feature = "unstable-runtime-subscribe")] - pub fn for_shared_stream( - trigger: impl Stream>> + Send + 'static> + Send + 'static, - reader: Store, - ) -> Self + pub fn for_shared_stream(trigger: impl Stream> + Send + 'static, reader: Store) -> Self where K::DynamicType: Default, { @@ -881,16 +878,12 @@ where /// [`dynamic`]: kube_client::core::dynamic #[cfg(feature = "unstable-runtime-subscribe")] pub fn for_shared_stream_with( - trigger: impl Stream>> + Send + 'static> + Send + 'static, + trigger: impl Stream> + Send + 'static, reader: Store, dyntype: K::DynamicType, ) -> Self { let mut trigger_selector = stream::SelectAll::new(); - let self_watcher = trigger_self_shared( - trigger.filter_map(|r| async move { r.into() }).map(Ok), - dyntype.clone(), - ) - .boxed(); + let self_watcher = trigger_self_shared(trigger.map(Ok), dyntype.clone()).boxed(); trigger_selector.push(self_watcher); Self { trigger_selector, @@ -1119,7 +1112,7 @@ where #[must_use] pub fn owns_shared_stream + Send + 'static>( self, - trigger: impl Stream>> + Send + 'static> + Send + 'static, + trigger: impl Stream> + Send + 'static, ) -> Self { self.owns_shared_stream_with(trigger, ()) } @@ -1137,17 +1130,13 @@ where #[must_use] pub fn owns_shared_stream_with + Send + 'static>( mut self, - trigger: impl Stream>> + Send + 'static> + Send + 'static, + trigger: impl Stream> + Send + 'static, dyntype: Child::DynamicType, ) -> Self where Child::DynamicType: Debug + Eq + Hash + Clone, { - let child_watcher = trigger_owners_shared( - trigger.filter_map(|r| async move { r.into() }).map(Ok), - self.dyntype.clone(), - dyntype, - ); + let child_watcher = trigger_owners_shared(trigger.map(Ok), self.dyntype.clone(), dyntype); self.trigger_selector.push(child_watcher.boxed()); self } @@ -1394,7 +1383,7 @@ where #[must_use] pub fn watches_shared_stream( self, - trigger: impl Stream>> + Send + 'static> + Send + 'static, + trigger: impl Stream> + Send + 'static, mapper: impl Fn(Arc) -> I + Sync + Send + 'static, ) -> Self where @@ -1419,7 +1408,7 @@ where #[must_use] pub fn watches_shared_stream_with( mut self, - trigger: impl Stream>> + Send + 'static> + Send + 'static, + trigger: impl Stream> + Send + 'static, mapper: impl Fn(Arc) -> I + Sync + Send + 'static, dyntype: Other::DynamicType, ) -> Self @@ -1429,11 +1418,7 @@ where I: 'static + IntoIterator>, I::IntoIter: Send, { - let other_watcher = trigger_others_shared( - trigger.filter_map(|r| async move { r.into() }).map(Ok), - mapper, - dyntype, - ); + let other_watcher = trigger_others_shared(trigger.map(Ok), mapper, dyntype); self.trigger_selector.push(other_watcher.boxed()); self } diff --git a/kube-runtime/src/reflector/dispatcher.rs b/kube-runtime/src/reflector/dispatcher.rs index 7a0be3fec..ce7617e4f 100644 --- a/kube-runtime/src/reflector/dispatcher.rs +++ b/kube-runtime/src/reflector/dispatcher.rs @@ -256,31 +256,38 @@ where K::DynamicType: Eq + std::hash::Hash + Clone + Default, K: DeserializeOwned, { - type Item = Option>; + type Item = Arc; fn poll_next(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { let mut this = self.project(); - match ready!(this.rx.as_mut().poll_next(cx)) { - Some(event) => { - let obj = match event { - Event::InitApply(obj) | Event::Apply(obj) - if obj.gvk() == Some(K::gvk(&Default::default())) => - { - obj.try_parse::().ok().map(Arc::new).inspect(|o| { - this.store.apply_shared_watcher_event(&Event::Apply(o.clone())); - }) + loop { + return match ready!(this.rx.as_mut().poll_next(cx)) { + Some(event) => { + let obj = match event { + Event::InitApply(obj) | Event::Apply(obj) + if obj.gvk() == Some(K::gvk(&Default::default())) => + { + obj.try_parse::().ok().map(Arc::new).inspect(|o| { + this.store.apply_shared_watcher_event(&Event::Apply(o.clone())); + }) + } + Event::Delete(obj) if obj.gvk() == Some(K::gvk(&Default::default())) => { + obj.try_parse::().ok().map(Arc::new).inspect(|o| { + this.store.apply_shared_watcher_event(&Event::Delete(o.clone())); + }) + } + _ => None, + }; + + // Skip propagating all objects which do not belong to the cache + if obj.is_none() { + continue; } - Event::Delete(obj) if obj.gvk() == Some(K::gvk(&Default::default())) => { - obj.try_parse::().ok().map(Arc::new).inspect(|o| { - this.store.apply_shared_watcher_event(&Event::Delete(o.clone())); - }) - } - _ => None, - }; - Poll::Ready(Some(obj)) - } - None => Poll::Ready(None), + Poll::Ready(obj) + } + None => Poll::Ready(None), + }; } } } diff --git a/kube-runtime/src/reflector/multi_dispatcher.rs b/kube-runtime/src/reflector/multi_dispatcher.rs index f5a4538f6..085b97df1 100644 --- a/kube-runtime/src/reflector/multi_dispatcher.rs +++ b/kube-runtime/src/reflector/multi_dispatcher.rs @@ -50,7 +50,8 @@ impl MultiDispatcher { /// Broadcast an event to any downstream listeners subscribed on the store pub(crate) async fn broadcast_event(&mut self, event: &watcher::Event) { match event { - watcher::Event::InitDone => {}, + // Broadcast stores are pre-initialized + watcher::Event::InitDone => {} ev => self.dispatcher.broadcast(ev.clone()).await, } } diff --git a/kube-runtime/src/watcher.rs b/kube-runtime/src/watcher.rs index 98195c333..755320b38 100644 --- a/kube-runtime/src/watcher.rs +++ b/kube-runtime/src/watcher.rs @@ -2,31 +2,20 @@ //! //! See [`watcher`] for the primary entry point. -use crate::{ - reflector::multi_dispatcher::BroadcastStream, - utils::{Backoff, ResetTimerBackoff}, -}; +use crate::utils::{Backoff, ResetTimerBackoff}; -use async_stream::stream; use async_trait::async_trait; use backon::BackoffBuilder; use educe::Educe; -use futures::{ - channel::mpsc::Receiver, - future::BoxFuture, - lock::Mutex, - stream::{self, empty, select_all, select_with_strategy, BoxStream, PollNext}, - Stream, StreamExt, -}; +use futures::{stream::BoxStream, Stream, StreamExt}; use kube_client::{ api::{ListParams, Resource, ResourceExt, VersionMatch, WatchEvent, WatchParams}, core::{metadata::PartialObjectMeta, ObjectList, Selector}, error::ErrorResponse, Api, Error as ClientErr, }; -use pin_project::pin_project; use serde::de::DeserializeOwned; -use std::{clone::Clone, collections::VecDeque, fmt::Debug, future, pin::Pin, sync::Arc, time::Duration}; +use std::{clone::Clone, collections::VecDeque, fmt::Debug, future, time::Duration}; use thiserror::Error; use tracing::{debug, error, warn}; @@ -157,7 +146,6 @@ enum State { InitialWatch { #[educe(Debug(ignore))] stream: BoxStream<'static, kube_client::Result>>, - params: WatchParams, }, /// The initial LIST was successful, so we should move on to starting the actual watch. InitListed { resource_version: String }, @@ -171,7 +159,6 @@ enum State { resource_version: String, #[educe(Debug(ignore))] stream: BoxStream<'static, kube_client::Result>>, - params: WatchParams, }, } @@ -188,6 +175,7 @@ trait ApiMode { version: &str, ) -> kube_client::Result>>>; } + /// A wrapper around the `Api` of a `Resource` type that when used by the /// watcher will return the entire (full) object struct FullObject<'a, K> { @@ -464,7 +452,6 @@ where wp: &WatchParams, version: &str, ) -> kube_client::Result>>> { - let x = || async { self.api.watch(wp, version).await }; self.api.watch(wp, version).await.map(StreamExt::boxed) } } @@ -495,59 +482,6 @@ where } } -/// A wrapper around the `Api` of a `Resource` type that when used by the -/// watcher will return only the metadata associated with an object -#[pin_project] -struct Interruptable<'a, K> { - api: &'a Api, - #[pin] - interrupt: Receiver<()>, -} - -// /// Type to tell [`SelectWithStrategy`] which stream to poll next. -// #[derive(Debug, PartialEq, Eq, Copy, Clone, Hash, Default)] -// pub enum Interrupt { -// /// Poll the first stream. -// #[default] -// Regular, -// /// Poll the second stream once. -// Restart, -// } - -// impl Interrupt { -// fn propagate(&mut self) -> PollNext { -// match self { -// Interrupt::Regular => PollNext::Left, -// Interrupt::Restart => { -// *self = Interrupt::Regular; -// PollNext::Right -// } -// } -// } -// } - -// #[async_trait] -// impl ApiMode for Interruptable<'_, K> -// where -// K: Clone + Debug + DeserializeOwned + Send + 'static, -// { -// type Value = K; - -// async fn list(&self, lp: &ListParams) -> kube_client::Result> { -// self.api.list(lp).await -// } - -// async fn watch( -// &self, -// wp: &WatchParams, -// version: &str, -// ) -> kube_client::Result>>> { -// let stream = self.api.watch(wp, version).await.map(StreamExt::boxed)?; - -// Ok(Box::pin(stream.restart_on(Box::pin(self.interrupt)))) -// } -// } - /// Progresses the watcher a single step, returning (event, state) /// /// This function should be trampolined: if event == `None` @@ -560,32 +494,26 @@ async fn step_trampolined( ) -> (Option>>, State) where A: ApiMode, - A::Value: Resource + 'static + Send, + A::Value: Resource + 'static, { match state { State::Empty => match wc.initial_list_strategy { - InitialListStrategy::ListWatch => ( - Some(Ok(Event::Init)), - State::InitPage { - continue_token: None, - objects: VecDeque::default(), - last_bookmark: None, - }, - ), - InitialListStrategy::StreamingList => { - let params = wc.to_watch_params(); - match api.watch(¶ms, "0").await { - Ok(stream) => (None, State::InitialWatch { stream, params }), - Err(err) => { - if std::matches!(err, ClientErr::Api(ErrorResponse { code: 403, .. })) { - warn!("watch initlist error with 403: {err:?}"); - } else { - debug!("watch initlist error: {err:?}"); - } - (Some(Err(Error::WatchStartFailed(err))), State::default()) + InitialListStrategy::ListWatch => (Some(Ok(Event::Init)), State::InitPage { + continue_token: None, + objects: VecDeque::default(), + last_bookmark: None, + }), + InitialListStrategy::StreamingList => match api.watch(&wc.to_watch_params(), "0").await { + Ok(stream) => (None, State::InitialWatch { stream }), + Err(err) => { + if std::matches!(err, ClientErr::Api(ErrorResponse { code: 403, .. })) { + warn!("watch initlist error with 403: {err:?}"); + } else { + debug!("watch initlist error: {err:?}"); } + (Some(Err(Error::WatchStartFailed(err))), State::default()) } - } + }, }, State::InitPage { continue_token, @@ -593,14 +521,11 @@ where last_bookmark, } => { if let Some(next) = objects.pop_front() { - return ( - Some(Ok(Event::InitApply(next))), - State::InitPage { - continue_token, - objects, - last_bookmark, - }, - ); + return (Some(Ok(Event::InitApply(next))), State::InitPage { + continue_token, + objects, + last_bookmark, + }); } // check if we need to perform more pages if continue_token.is_none() { @@ -620,14 +545,11 @@ where } // Buffer page here, causing us to return to this enum branch (State::InitPage) // until the objects buffer has drained - ( - None, - State::InitPage { - continue_token, - objects: list.items.into_iter().collect(), - last_bookmark, - }, - ) + (None, State::InitPage { + continue_token, + objects: list.items.into_iter().collect(), + last_bookmark, + }) } Err(err) => { if std::matches!(err, ClientErr::Api(ErrorResponse { code: 403, .. })) { @@ -639,42 +561,26 @@ where } } } - State::InitialWatch { mut stream, params } => { - if params != wc.to_watch_params() { - return ( - Some(Err(Error::WatchError(kube_client::core::ErrorResponse { - status: "Restarting".into(), - message: "Watches are restarting due to selector change".into(), - reason: "Restart".into(), - code: 410, - }))), - State::default(), - ); - } + State::InitialWatch { mut stream } => { match stream.next().await { - Some(Ok(WatchEvent::Added(obj) | WatchEvent::Modified(obj))) => ( - Some(Ok(Event::InitApply(obj))), - State::InitialWatch { stream, params }, - ), + Some(Ok(WatchEvent::Added(obj) | WatchEvent::Modified(obj))) => { + (Some(Ok(Event::InitApply(obj))), State::InitialWatch { stream }) + } Some(Ok(WatchEvent::Deleted(_obj))) => { // Kubernetes claims these events are impossible // https://kubernetes.io/docs/reference/using-api/api-concepts/#streaming-lists error!("got deleted event during initial watch. this is a bug"); - (None, State::InitialWatch { stream, params }) + (None, State::InitialWatch { stream }) } Some(Ok(WatchEvent::Bookmark(bm))) => { let marks_initial_end = bm.metadata.annotations.contains_key("k8s.io/initial-events-end"); if marks_initial_end { - ( - Some(Ok(Event::InitDone)), - State::Watching { - resource_version: bm.metadata.resource_version, - stream, - params, - }, - ) + (Some(Ok(Event::InitDone)), State::Watching { + resource_version: bm.metadata.resource_version, + stream, + }) } else { - (None, State::InitialWatch { stream, params }) + (None, State::InitialWatch { stream }) } } Some(Ok(WatchEvent::Error(err))) => { @@ -682,7 +588,7 @@ where let new_state = if err.code == 410 { State::default() } else { - State::InitialWatch { stream, params } + State::InitialWatch { stream } }; if err.code == 403 { warn!("watcher watchevent error 403: {err:?}"); @@ -697,129 +603,89 @@ where } else { debug!("watcher error: {err:?}"); } - ( - Some(Err(Error::WatchFailed(err))), - State::InitialWatch { stream, params }, - ) + (Some(Err(Error::WatchFailed(err))), State::InitialWatch { stream }) } None => (None, State::default()), } } State::InitListed { resource_version } => { - let params = wc.to_watch_params(); - match api.watch(¶ms, &resource_version).await { - Ok(stream) => ( - None, - State::Watching { - resource_version, - stream, - params, - }, - ), + match api.watch(&wc.to_watch_params(), &resource_version).await { + Ok(stream) => (None, State::Watching { + resource_version, + stream, + }), Err(err) => { if std::matches!(err, ClientErr::Api(ErrorResponse { code: 403, .. })) { warn!("watch initlist error with 403: {err:?}"); } else { debug!("watch initlist error: {err:?}"); } - ( - Some(Err(Error::WatchStartFailed(err))), - State::InitListed { resource_version }, - ) + (Some(Err(Error::WatchStartFailed(err))), State::InitListed { + resource_version, + }) } } } State::Watching { resource_version, mut stream, - params, - } => { - if params != wc.to_watch_params() { - return ( - Some(Err(Error::WatchError(kube_client::core::ErrorResponse { - status: "Restarting".into(), - message: "Watches are restarting due to selector change".into(), - reason: "Restart".into(), - code: 410, - }))), - State::default(), - ); - } - match stream.next().await { - Some(Ok(WatchEvent::Added(obj) | WatchEvent::Modified(obj))) => { - let resource_version = obj.resource_version().unwrap_or_default(); - if resource_version.is_empty() { - (Some(Err(Error::NoResourceVersion)), State::default()) - } else { - ( - Some(Ok(Event::Apply(obj))), - State::Watching { - resource_version, - stream, - params, - }, - ) - } + } => match stream.next().await { + Some(Ok(WatchEvent::Added(obj) | WatchEvent::Modified(obj))) => { + let resource_version = obj.resource_version().unwrap_or_default(); + if resource_version.is_empty() { + (Some(Err(Error::NoResourceVersion)), State::default()) + } else { + (Some(Ok(Event::Apply(obj))), State::Watching { + resource_version, + stream, + }) } - Some(Ok(WatchEvent::Deleted(obj))) => { - let resource_version = obj.resource_version().unwrap_or_default(); - if resource_version.is_empty() { - (Some(Err(Error::NoResourceVersion)), State::default()) - } else { - ( - Some(Ok(Event::Delete(obj))), - State::Watching { - resource_version, - stream, - params, - }, - ) - } + } + Some(Ok(WatchEvent::Deleted(obj))) => { + let resource_version = obj.resource_version().unwrap_or_default(); + if resource_version.is_empty() { + (Some(Err(Error::NoResourceVersion)), State::default()) + } else { + (Some(Ok(Event::Delete(obj))), State::Watching { + resource_version, + stream, + }) } - Some(Ok(WatchEvent::Bookmark(bm))) => ( - None, + } + Some(Ok(WatchEvent::Bookmark(bm))) => (None, State::Watching { + resource_version: bm.metadata.resource_version, + stream, + }), + Some(Ok(WatchEvent::Error(err))) => { + // HTTP GONE, means we have desynced and need to start over and re-list :( + let new_state = if err.code == 410 { + State::default() + } else { State::Watching { - resource_version: bm.metadata.resource_version, + resource_version, stream, - params, - }, - ), - Some(Ok(WatchEvent::Error(err))) => { - // HTTP GONE, means we have desynced and need to start over and re-list :( - let new_state = if err.code == 410 { - State::default() - } else { - State::Watching { - resource_version, - stream, - params, - } - }; - if err.code == 403 { - warn!("watcher watchevent error 403: {err:?}"); - } else { - debug!("error watchevent error: {err:?}"); } - (Some(Err(Error::WatchError(err))), new_state) + }; + if err.code == 403 { + warn!("watcher watchevent error 403: {err:?}"); + } else { + debug!("error watchevent error: {err:?}"); } - Some(Err(err)) => { - if std::matches!(err, ClientErr::Api(ErrorResponse { code: 403, .. })) { - warn!("watcher error 403: {err:?}"); - } else { - debug!("watcher error: {err:?}"); - } - ( - Some(Err(Error::WatchFailed(err))), - State::Watching { - resource_version, - stream, - params, - }, - ) + (Some(Err(Error::WatchError(err))), new_state) + } + Some(Err(err)) => { + if std::matches!(err, ClientErr::Api(ErrorResponse { code: 403, .. })) { + warn!("watcher error 403: {err:?}"); + } else { + debug!("watcher error: {err:?}"); } - None => (None, State::InitListed { resource_version }), + (Some(Err(Error::WatchFailed(err))), State::Watching { + resource_version, + stream, + }) } - } + None => (None, State::InitListed { resource_version }), + }, } } @@ -831,7 +697,7 @@ async fn step( ) -> (Result>, State) where A: ApiMode, - A::Value: Resource + 'static + Send, + A::Value: Resource + 'static, { loop { match step_trampolined(api, config, state).await { @@ -903,33 +769,6 @@ pub fn watcher( ) } -// pub fn watcher_restart( -// api: Api, -// watcher_config: Config, -// trigger: impl Stream + Send + Sync + 'static + Clone -// ) -> impl Stream>> + Send { -// futures::stream::unfold( -// (api, watcher_config, State::default(), trigger), -// |(api, watcher_config, state, trigger)| async move { -// let (event, state) = { -// let api = FullObject { api: &api }; -// let config: &Config = &watcher_config; -// let mut state = state; -// let trigger = trigger.clone(); -// async move { -// loop { -// match step_trampolined(&api, config, state, trigger).await { -// (Some(result), new_state) => return (result, new_state), -// (None, new_state) => state = new_state, -// } -// } -// } -// }.await; -// Some((event, (api, watcher_config, state, trigger))) -// }, -// ) -// } - /// Watches a Kubernetes Resource for changes continuously and receives only the /// metadata /// From ea8fd2bbf64e17dea971db0f5eb8cd5b551daeb1 Mon Sep 17 00:00:00 2001 From: Danil-Grigorev Date: Mon, 3 Mar 2025 23:03:33 +0100 Subject: [PATCH 12/16] Use default deserialize Signed-off-by: Danil-Grigorev --- kube-core/src/object.rs | 46 ++--------------------------------------- 1 file changed, 2 insertions(+), 44 deletions(-) diff --git a/kube-core/src/object.rs b/kube-core/src/object.rs index c9e2a25f8..1e81a947b 100644 --- a/kube-core/src/object.rs +++ b/kube-core/src/object.rs @@ -3,12 +3,8 @@ use crate::{ discovery::ApiResource, metadata::{ListMeta, ObjectMeta, TypeMeta}, resource::{DynamicResourceScope, Resource}, - DynamicObject, -}; -use serde::{ - de::{self, DeserializeOwned}, - Deserialize, Deserializer, Serialize, }; +use serde::{Deserialize, Deserializer, Serialize}; use std::borrow::Cow; /// A generic Kubernetes object list @@ -20,7 +16,7 @@ use std::borrow::Cow; /// and is generally produced from list/watch/delete collection queries on an [`Resource`](super::Resource). /// /// This is almost equivalent to [`k8s_openapi::List`](k8s_openapi::List), but iterable. -#[derive(Serialize, Debug, Clone)] +#[derive(Serialize, Deserialize, Debug, Clone)] pub struct ObjectList where T: Clone, @@ -136,44 +132,6 @@ impl<'a, T: Clone> IntoIterator for &'a mut ObjectList { } } -#[derive(Deserialize)] -struct DynamicList { - #[serde(flatten, deserialize_with = "deserialize_v1_list_as_default")] - types: TypeMeta, - - #[serde(default)] - metadata: ListMeta, - - #[serde( - deserialize_with = "deserialize_null_as_default", - bound(deserialize = "Vec: Deserialize<'de>") - )] - items: Vec, -} - -impl<'de, T: DeserializeOwned + Clone> serde::Deserialize<'de> for ObjectList { - fn deserialize>(d: D) -> Result, D::Error> { - let DynamicList { - types, - metadata, - mut items, - } = DynamicList::deserialize(d)?; - let mut resources = vec![]; - for o in items.iter_mut() { - if o.types.is_none() { - o.types = types.clone().singular_list(); - } - let item = serde_json::to_value(o).map_err(de::Error::custom)?; - resources.push(serde_json::from_value(item).map_err(de::Error::custom)?) - } - Ok(ObjectList:: { - types, - metadata, - items: resources, - }) - } -} - /// A trait to access the `spec` of a Kubernetes resource. /// /// Some built-in Kubernetes resources and all custom resources do have a `spec` field. From 27357d085ec142a12dcf95d411d26dbb9936cc63 Mon Sep 17 00:00:00 2001 From: Danil-Grigorev Date: Mon, 3 Mar 2025 23:24:12 +0100 Subject: [PATCH 13/16] Drop DynamicDispatcher in favor of MultiDispatcher Signed-off-by: Danil-Grigorev --- kube-runtime/src/reflector/dispatcher.rs | 55 ------------------- .../src/reflector/multi_dispatcher.rs | 29 +++++++--- kube-runtime/src/reflector/store.rs | 4 +- 3 files changed, 22 insertions(+), 66 deletions(-) diff --git a/kube-runtime/src/reflector/dispatcher.rs b/kube-runtime/src/reflector/dispatcher.rs index ce7617e4f..09608ed92 100644 --- a/kube-runtime/src/reflector/dispatcher.rs +++ b/kube-runtime/src/reflector/dispatcher.rs @@ -77,61 +77,6 @@ where } } -// A helper type that holds a broadcast transmitter and a broadcast receiver, -// used to fan-out events from a root stream to multiple listeners. -#[cfg(feature = "unstable-runtime-subscribe")] -#[derive(Clone)] -pub(crate) struct DynamicDispatcher { - dispatch_tx: Sender>, - // An inactive reader that prevents the channel from closing until the - // writer is dropped. - _dispatch_rx: InactiveReceiver>, -} - -#[cfg(feature = "unstable-runtime-subscribe")] -impl DynamicDispatcher { - /// Creates and returns a new self that wraps a broadcast sender and an - /// inactive broadcast receiver - /// - /// A buffer size is required to create the underlying broadcast channel. - /// Messages will be buffered until all active readers have received a copy - /// of the message. When the channel is full, senders will apply - /// backpressure by waiting for space to free up. - // - // N.B messages are eagerly broadcasted, meaning no active receivers are - // required for a message to be broadcasted. - pub(crate) fn new(buf_size: usize) -> DynamicDispatcher { - // Create a broadcast (tx, rx) pair - let (mut dispatch_tx, dispatch_rx) = async_broadcast::broadcast(buf_size); - // The tx half will not wait for any receivers to be active before - // broadcasting events. If no receivers are active, events will be - // buffered. - dispatch_tx.set_await_active(false); - Self { - dispatch_tx, - _dispatch_rx: dispatch_rx.deactivate(), - } - } - - // Calls broadcast on the channel. Will return when the channel has enough - // space to send an event. - pub(crate) async fn broadcast(&mut self, evt: Event) { - let _ = self.dispatch_tx.broadcast_direct(evt).await; - } - - // Creates a `TypedReflectHandle` by creating a receiver from the tx half. - // N.B: the new receiver will be fast-forwarded to the _latest_ event. - // The receiver won't have access to any events that are currently waiting - // to be acked by listeners. - pub(crate) fn subscribe(&self) -> TypedReflectHandle - where - K: Resource + DeserializeOwned + Clone, - K::DynamicType: Eq + std::hash::Hash + Clone + Default, - { - TypedReflectHandle::new(self.dispatch_tx.new_receiver()) - } -} - /// A handle to a shared stream reader /// /// [`ReflectHandle`]s are created by calling [`subscribe()`] on a [`Writer`], diff --git a/kube-runtime/src/reflector/multi_dispatcher.rs b/kube-runtime/src/reflector/multi_dispatcher.rs index 085b97df1..8fa903e36 100644 --- a/kube-runtime/src/reflector/multi_dispatcher.rs +++ b/kube-runtime/src/reflector/multi_dispatcher.rs @@ -5,27 +5,36 @@ use std::{ task::{Context, Poll}, }; -use futures::{lock::Mutex, FutureExt, Stream, StreamExt as _}; +use crate::watcher::Event; +use async_broadcast::{InactiveReceiver, Sender}; +use futures::{lock::Mutex, Stream, StreamExt as _}; use kube_client::{api::DynamicObject, Resource}; use serde::de::DeserializeOwned; use crate::watcher; -use super::{ - dispatcher::{DynamicDispatcher, TypedReflectHandle}, - Store, -}; +use super::{dispatcher::TypedReflectHandle, Store}; #[derive(Clone)] pub struct MultiDispatcher { - dispatcher: DynamicDispatcher, + dispatch_tx: Sender>, + // An inactive reader that prevents the channel from closing until the + // writer is dropped. + _dispatch_rx: InactiveReceiver>, } impl MultiDispatcher { #[must_use] pub fn new(buf_size: usize) -> Self { + // Create a broadcast (tx, rx) pair + let (mut dispatch_tx, dispatch_rx) = async_broadcast::broadcast(buf_size); + // The tx half will not wait for any receivers to be active before + // broadcasting events. If no receivers are active, events will be + // buffered. + dispatch_tx.set_await_active(false); Self { - dispatcher: DynamicDispatcher::new(buf_size), + dispatch_tx, + _dispatch_rx: dispatch_rx.deactivate(), } } @@ -42,7 +51,7 @@ impl MultiDispatcher { K: Resource + Clone + DeserializeOwned, K::DynamicType: Eq + Clone + Hash + Default, { - let sub = self.dispatcher.subscribe(); + let sub = TypedReflectHandle::new(self.dispatch_tx.new_receiver()); let reader = sub.reader(); (sub, reader) } @@ -52,7 +61,9 @@ impl MultiDispatcher { match event { // Broadcast stores are pre-initialized watcher::Event::InitDone => {} - ev => self.dispatcher.broadcast(ev.clone()).await, + ev => { + let _ = self.dispatch_tx.broadcast_direct(ev.clone()).await; + } } } } diff --git a/kube-runtime/src/reflector/store.rs b/kube-runtime/src/reflector/store.rs index f88662063..f4158d610 100644 --- a/kube-runtime/src/reflector/store.rs +++ b/kube-runtime/src/reflector/store.rs @@ -99,7 +99,7 @@ where } /// Applies a single watcher event to the store - pub(crate) fn apply_watcher_event(&mut self, event: &watcher::Event) { + pub fn apply_watcher_event(&mut self, event: &watcher::Event) { match event { watcher::Event::Apply(obj) => { let obj = Arc::new(obj.clone()); @@ -123,7 +123,7 @@ where } /// Applies a single shared watcher event to the store - pub(crate) fn apply_shared_watcher_event(&mut self, event: &watcher::Event>) { + pub fn apply_shared_watcher_event(&mut self, event: &watcher::Event>) { match event { watcher::Event::Apply(obj) => { let key = obj.to_object_ref(self.dyntype.clone()); From 8bf3ed9637ec15d04821beae8acad091786cda90 Mon Sep 17 00:00:00 2001 From: Danil-Grigorev Date: Mon, 3 Mar 2025 23:32:19 +0100 Subject: [PATCH 14/16] Remove unused import Signed-off-by: Danil-Grigorev --- kube-runtime/src/reflector/mod.rs | 4 ---- kube-runtime/src/reflector/multi_dispatcher.rs | 3 ++- kube-runtime/src/utils/watch_ext.rs | 6 ------ 3 files changed, 2 insertions(+), 11 deletions(-) diff --git a/kube-runtime/src/reflector/mod.rs b/kube-runtime/src/reflector/mod.rs index be0288956..f0354c337 100644 --- a/kube-runtime/src/reflector/mod.rs +++ b/kube-runtime/src/reflector/mod.rs @@ -4,8 +4,6 @@ mod dispatcher; #[cfg(feature = "unstable-runtime-subscribe")] pub mod multi_dispatcher; mod object_ref; pub mod store; -#[cfg(feature = "unstable-runtime-subscribe")] use std::sync::Arc; -#[cfg(feature = "unstable-runtime-subscribe")] use tokio::sync::Mutex; pub use self::{ dispatcher::ReflectHandle, @@ -13,7 +11,6 @@ pub use self::{ }; use crate::watcher; use async_stream::stream; -#[cfg(feature = "unstable-runtime-subscribe")] use futures::stream::SelectAll; use futures::{Stream, StreamExt}; #[cfg(feature = "unstable-runtime-subscribe")] use kube_client::api::DynamicObject; @@ -22,7 +19,6 @@ use multi_dispatcher::BroadcastStream; #[cfg(feature = "unstable-runtime-subscribe")] use multi_dispatcher::MultiDispatcher; use std::hash::Hash; -#[cfg(feature = "unstable-runtime-subscribe")] use std::pin::Pin; #[cfg(feature = "unstable-runtime-subscribe")] pub use store::store_shared; pub use store::{store, Store}; diff --git a/kube-runtime/src/reflector/multi_dispatcher.rs b/kube-runtime/src/reflector/multi_dispatcher.rs index 8fa903e36..f301bfb31 100644 --- a/kube-runtime/src/reflector/multi_dispatcher.rs +++ b/kube-runtime/src/reflector/multi_dispatcher.rs @@ -68,7 +68,8 @@ impl MultiDispatcher { } } -/// See [`Scheduler::hold`] +/// BroadcastStream allows to stream shared list of dynamic objects, +/// sources of which can be changed at any moment. pub struct BroadcastStream { pub stream: Arc>, } diff --git a/kube-runtime/src/utils/watch_ext.rs b/kube-runtime/src/utils/watch_ext.rs index 6ac1d133a..241871837 100644 --- a/kube-runtime/src/utils/watch_ext.rs +++ b/kube-runtime/src/utils/watch_ext.rs @@ -9,16 +9,10 @@ use crate::{ }; use kube_client::Resource; -#[cfg(feature = "unstable-runtime-subscribe")] -use crate::reflector::multi_dispatcher::MultiDispatcher; use crate::{ reflector::store::Writer, utils::{Backoff, Reflect}, }; -#[cfg(feature = "unstable-runtime-subscribe")] -use kube_client::api::DynamicObject; -#[cfg(feature = "unstable-runtime-subscribe")] use std::sync::Arc; -#[cfg(feature = "unstable-runtime-subscribe")] use tokio::sync::Mutex; use crate::watcher::DefaultBackoff; use futures::{Stream, TryStream}; From f860c39dbaf1ad0c7612818898ab57d321f361d2 Mon Sep 17 00:00:00 2001 From: Danil-Grigorev Date: Mon, 3 Mar 2025 23:34:39 +0100 Subject: [PATCH 15/16] Clippy fixes Signed-off-by: Danil-Grigorev --- kube-runtime/src/reflector/multi_dispatcher.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kube-runtime/src/reflector/multi_dispatcher.rs b/kube-runtime/src/reflector/multi_dispatcher.rs index f301bfb31..22129e18f 100644 --- a/kube-runtime/src/reflector/multi_dispatcher.rs +++ b/kube-runtime/src/reflector/multi_dispatcher.rs @@ -68,7 +68,7 @@ impl MultiDispatcher { } } -/// BroadcastStream allows to stream shared list of dynamic objects, +/// `BroadcastStream` allows to stream shared list of dynamic objects, /// sources of which can be changed at any moment. pub struct BroadcastStream { pub stream: Arc>, From 905456ffe8231877a4baca71698ae51b55a3af4a Mon Sep 17 00:00:00 2001 From: Danil-Grigorev Date: Sun, 20 Apr 2025 19:17:50 +0200 Subject: [PATCH 16/16] Add optional id to watch events Signed-off-by: Danil-Grigorev --- kube-core/src/params.rs | 6 +- kube-runtime/src/controller/mod.rs | 4 +- kube-runtime/src/reflector/dispatcher.rs | 86 +++++----- kube-runtime/src/reflector/mod.rs | 22 +-- .../src/reflector/multi_dispatcher.rs | 2 +- kube-runtime/src/reflector/store.rs | 46 ++--- kube-runtime/src/utils/event_decode.rs | 20 +-- kube-runtime/src/utils/event_modify.rs | 8 +- kube-runtime/src/utils/reflect.rs | 20 +-- kube-runtime/src/watcher.rs | 161 +++++++++++------- 10 files changed, 207 insertions(+), 168 deletions(-) diff --git a/kube-core/src/params.rs b/kube-core/src/params.rs index fa508dc4e..4abe942c9 100644 --- a/kube-core/src/params.rs +++ b/kube-core/src/params.rs @@ -8,7 +8,7 @@ use serde::Serialize; /// depending on what `resource_version`, `limit`, `continue_token` you include with the list request. /// /// See for details. -#[derive(Clone, Debug, PartialEq)] +#[derive(Clone, Debug, PartialEq, Hash)] pub enum VersionMatch { /// Returns data at least as new as the provided resource version. /// @@ -36,7 +36,7 @@ pub enum VersionMatch { } /// Common query parameters used in list/delete calls on collections -#[derive(Clone, Debug, Default, PartialEq)] +#[derive(Clone, Debug, Default, PartialEq, Hash)] pub struct ListParams { /// A selector to restrict the list of returned objects by their labels. /// @@ -305,7 +305,7 @@ impl ValidationDirective { } /// Common query parameters used in watch calls on collections -#[derive(Clone, Debug, PartialEq)] +#[derive(Clone, Debug, PartialEq, Hash)] pub struct WatchParams { /// A selector to restrict returned objects by their labels. /// diff --git a/kube-runtime/src/controller/mod.rs b/kube-runtime/src/controller/mod.rs index 8a8f7fc3a..2816129b8 100644 --- a/kube-runtime/src/controller/mod.rs +++ b/kube-runtime/src/controller/mod.rs @@ -1766,7 +1766,7 @@ mod tests { queue_rx.map(Result::<_, Infallible>::Ok), Config::default(), )); - store_tx.apply_watcher_event(&watcher::Event::InitDone); + store_tx.apply_watcher_event(&watcher::Event::InitDone(None)); for i in 0..items { let obj = ConfigMap { metadata: ObjectMeta { @@ -1776,7 +1776,7 @@ mod tests { }, ..Default::default() }; - store_tx.apply_watcher_event(&watcher::Event::Apply(obj.clone())); + store_tx.apply_watcher_event(&watcher::Event::Apply(obj.clone(), None)); queue_tx.unbounded_send(ObjectRef::from_obj(&obj)).unwrap(); } diff --git a/kube-runtime/src/reflector/dispatcher.rs b/kube-runtime/src/reflector/dispatcher.rs index 09608ed92..31c4bbbac 100644 --- a/kube-runtime/src/reflector/dispatcher.rs +++ b/kube-runtime/src/reflector/dispatcher.rs @@ -183,7 +183,7 @@ where // Initialize a ready store by default store: { let mut store: Writer = Default::default(); - store.apply_shared_watcher_event(&Event::InitDone); + store.apply_shared_watcher_event(&Event::InitDone(None)); store }, } @@ -209,16 +209,16 @@ where return match ready!(this.rx.as_mut().poll_next(cx)) { Some(event) => { let obj = match event { - Event::InitApply(obj) | Event::Apply(obj) + Event::InitApply(obj, id) | Event::Apply(obj, id) if obj.gvk() == Some(K::gvk(&Default::default())) => { obj.try_parse::().ok().map(Arc::new).inspect(|o| { - this.store.apply_shared_watcher_event(&Event::Apply(o.clone())); + this.store.apply_shared_watcher_event(&Event::Apply(o.clone(), id)); }) } - Event::Delete(obj) if obj.gvk() == Some(K::gvk(&Default::default())) => { + Event::Delete(obj, id) if obj.gvk() == Some(K::gvk(&Default::default())) => { obj.try_parse::().ok().map(Arc::new).inspect(|o| { - this.store.apply_shared_watcher_event(&Event::Delete(o.clone())); + this.store.apply_shared_watcher_event(&Event::Delete(o.clone(), id)); }) } _ => None, @@ -261,12 +261,12 @@ pub(crate) mod test { let foo = testpod("foo"); let bar = testpod("bar"); let st = stream::iter([ - Ok(Event::Apply(foo.clone())), + Ok(Event::Apply(foo.clone(), None)), Err(Error::NoResourceVersion), - Ok(Event::Init), - Ok(Event::InitApply(foo)), - Ok(Event::InitApply(bar)), - Ok(Event::InitDone), + Ok(Event::Init(None)), + Ok(Event::InitApply(foo, None)), + Ok(Event::InitApply(bar, None)), + Ok(Event::InitDone(None)), ]); let (reader, writer) = reflector::store_shared(10); @@ -276,7 +276,7 @@ pub(crate) mod test { assert_eq!(reader.len(), 0); assert!(matches!( poll!(reflect.next()), - Poll::Ready(Some(Ok(Event::Apply(_)))) + Poll::Ready(Some(Ok(Event::Apply(..)))) )); // Make progress and assert all events are seen @@ -288,19 +288,19 @@ pub(crate) mod test { assert_eq!(reader.len(), 1); let restarted = poll!(reflect.next()); - assert!(matches!(restarted, Poll::Ready(Some(Ok(Event::Init))))); + assert!(matches!(restarted, Poll::Ready(Some(Ok(Event::Init(_)))))); assert_eq!(reader.len(), 1); let restarted = poll!(reflect.next()); - assert!(matches!(restarted, Poll::Ready(Some(Ok(Event::InitApply(_)))))); + assert!(matches!(restarted, Poll::Ready(Some(Ok(Event::InitApply(..)))))); assert_eq!(reader.len(), 1); let restarted = poll!(reflect.next()); - assert!(matches!(restarted, Poll::Ready(Some(Ok(Event::InitApply(_)))))); + assert!(matches!(restarted, Poll::Ready(Some(Ok(Event::InitApply(..)))))); assert_eq!(reader.len(), 1); let restarted = poll!(reflect.next()); - assert!(matches!(restarted, Poll::Ready(Some(Ok(Event::InitDone))))); + assert!(matches!(restarted, Poll::Ready(Some(Ok(Event::InitDone(_)))))); assert_eq!(reader.len(), 2); assert!(matches!(poll!(reflect.next()), Poll::Ready(None))); @@ -316,13 +316,13 @@ pub(crate) mod test { let foo = testpod("foo"); let bar = testpod("bar"); let st = stream::iter([ - Ok(Event::Delete(foo.clone())), - Ok(Event::Apply(foo.clone())), + Ok(Event::Delete(foo.clone(), None)), + Ok(Event::Apply(foo.clone(), None)), Err(Error::NoResourceVersion), - Ok(Event::Init), - Ok(Event::InitApply(foo.clone())), - Ok(Event::InitApply(bar.clone())), - Ok(Event::InitDone), + Ok(Event::Init(None)), + Ok(Event::InitApply(foo.clone(), None)), + Ok(Event::InitApply(bar.clone(), None)), + Ok(Event::InitDone(None)), ]); let foo = Arc::new(foo); @@ -335,13 +335,13 @@ pub(crate) mod test { // Deleted events should be skipped by subscriber. assert!(matches!( poll!(reflect.next()), - Poll::Ready(Some(Ok(Event::Delete(_)))) + Poll::Ready(Some(Ok(Event::Delete(..)))) )); assert_eq!(poll!(subscriber.next()), Poll::Pending); assert!(matches!( poll!(reflect.next()), - Poll::Ready(Some(Ok(Event::Apply(_)))) + Poll::Ready(Some(Ok(Event::Apply(..)))) )); assert_eq!(poll!(subscriber.next()), Poll::Ready(Some(foo.clone()))); @@ -356,21 +356,21 @@ pub(crate) mod test { assert!(matches!( poll!(reflect.next()), - Poll::Ready(Some(Ok(Event::Init))) + Poll::Ready(Some(Ok(Event::Init(_)))) )); assert!(matches!( poll!(reflect.next()), - Poll::Ready(Some(Ok(Event::InitApply(_)))) + Poll::Ready(Some(Ok(Event::InitApply(..)))) )); assert!(matches!( poll!(reflect.next()), - Poll::Ready(Some(Ok(Event::InitApply(_)))) + Poll::Ready(Some(Ok(Event::InitApply(..)))) )); assert!(matches!( poll!(reflect.next()), - Poll::Ready(Some(Ok(Event::InitDone))) + Poll::Ready(Some(Ok(Event::InitDone(_)))) )); // these don't come back in order atm: @@ -389,11 +389,11 @@ pub(crate) mod test { let foo = testpod("foo"); let bar = testpod("bar"); let st = stream::iter([ - Ok(Event::Apply(foo.clone())), - Ok(Event::Init), - Ok(Event::InitApply(foo.clone())), - Ok(Event::InitApply(bar.clone())), - Ok(Event::InitDone), + Ok(Event::Apply(foo.clone(), None)), + Ok(Event::Init(None)), + Ok(Event::InitApply(foo.clone(), None)), + Ok(Event::InitApply(bar.clone(), None)), + Ok(Event::InitDone(None)), ]); let foo = Arc::new(foo); @@ -405,7 +405,7 @@ pub(crate) mod test { assert!(matches!( poll!(reflect.next()), - Poll::Ready(Some(Ok(Event::Apply(_)))) + Poll::Ready(Some(Ok(Event::Apply(..)))) )); assert_eq!(poll!(subscriber.next()), Poll::Ready(Some(foo.clone()))); @@ -417,25 +417,25 @@ pub(crate) mod test { assert!(matches!( poll!(reflect.next()), - Poll::Ready(Some(Ok(Event::Init))) + Poll::Ready(Some(Ok(Event::Init(_)))) )); assert_eq!(poll!(subscriber.next()), Poll::Pending); assert!(matches!( poll!(reflect.next()), - Poll::Ready(Some(Ok(Event::InitApply(_)))) + Poll::Ready(Some(Ok(Event::InitApply(..)))) )); assert_eq!(poll!(subscriber.next()), Poll::Pending); assert!(matches!( poll!(reflect.next()), - Poll::Ready(Some(Ok(Event::InitApply(_)))) + Poll::Ready(Some(Ok(Event::InitApply(..)))) )); assert_eq!(poll!(subscriber.next()), Poll::Pending); assert!(matches!( poll!(reflect.next()), - Poll::Ready(Some(Ok(Event::InitDone))) + Poll::Ready(Some(Ok(Event::InitDone(_)))) )); drop(reflect); @@ -456,9 +456,9 @@ pub(crate) mod test { let bar = testpod("bar"); let st = stream::iter([ //TODO: include a ready event here to avoid dealing with Init? - Ok(Event::Apply(foo.clone())), - Ok(Event::Apply(bar.clone())), - Ok(Event::Apply(foo.clone())), + Ok(Event::Apply(foo.clone(), None)), + Ok(Event::Apply(bar.clone(), None)), + Ok(Event::Apply(foo.clone(), None)), ]); let foo = Arc::new(foo); @@ -478,7 +478,7 @@ pub(crate) mod test { // we will still get an event from the root. assert!(matches!( poll!(reflect.next()), - Poll::Ready(Some(Ok(Event::Apply(_)))) + Poll::Ready(Some(Ok(Event::Apply(..)))) )); assert_eq!(poll!(subscriber.next()), Poll::Ready(Some(foo.clone()))); @@ -500,14 +500,14 @@ pub(crate) mod test { // had two. We repeat the same pattern. assert!(matches!( poll!(reflect.next()), - Poll::Ready(Some(Ok(Event::Apply(_)))) + Poll::Ready(Some(Ok(Event::Apply(..)))) )); assert_eq!(poll!(subscriber.next()), Poll::Ready(Some(bar.clone()))); assert!(matches!(poll!(reflect.next()), Poll::Pending)); assert_eq!(poll!(subscriber_slow.next()), Poll::Ready(Some(bar.clone()))); assert!(matches!( poll!(reflect.next()), - Poll::Ready(Some(Ok(Event::Apply(_)))) + Poll::Ready(Some(Ok(Event::Apply(..)))) )); // Poll again to drain the queue. assert!(matches!(poll!(reflect.next()), Poll::Ready(None))); diff --git a/kube-runtime/src/reflector/mod.rs b/kube-runtime/src/reflector/mod.rs index f0354c337..802d3932d 100644 --- a/kube-runtime/src/reflector/mod.rs +++ b/kube-runtime/src/reflector/mod.rs @@ -186,7 +186,7 @@ mod tests { }, ..ConfigMap::default() }; - reflector(store_w, stream::iter(vec![Ok(watcher::Event::Apply(cm.clone()))])) + reflector(store_w, stream::iter(vec![Ok(watcher::Event::Apply(cm.clone(), None))])) .map(|_| ()) .collect::<()>() .await; @@ -215,8 +215,8 @@ mod tests { reflector( store_w, stream::iter(vec![ - Ok(watcher::Event::Apply(cm.clone())), - Ok(watcher::Event::Apply(updated_cm.clone())), + Ok(watcher::Event::Apply(cm.clone(), None)), + Ok(watcher::Event::Apply(updated_cm.clone(), None)), ]), ) .map(|_| ()) @@ -239,8 +239,8 @@ mod tests { reflector( store_w, stream::iter(vec![ - Ok(watcher::Event::Apply(cm.clone())), - Ok(watcher::Event::Delete(cm.clone())), + Ok(watcher::Event::Apply(cm.clone(), None)), + Ok(watcher::Event::Delete(cm.clone(), None)), ]), ) .map(|_| ()) @@ -270,10 +270,10 @@ mod tests { reflector( store_w, stream::iter(vec![ - Ok(watcher::Event::Apply(cm_a.clone())), - Ok(watcher::Event::Init), - Ok(watcher::Event::InitApply(cm_b.clone())), - Ok(watcher::Event::InitDone), + Ok(watcher::Event::Apply(cm_a.clone(), None)), + Ok(watcher::Event::Init(None)), + Ok(watcher::Event::InitApply(cm_b.clone(), None)), + Ok(watcher::Event::InitDone(None)), ]), ) .map(|_| ()) @@ -304,9 +304,9 @@ mod tests { ..ConfigMap::default() }; Ok(if deleted { - watcher::Event::Delete(obj) + watcher::Event::Delete(obj, None) } else { - watcher::Event::Apply(obj) + watcher::Event::Apply(obj, None) }) })), ) diff --git a/kube-runtime/src/reflector/multi_dispatcher.rs b/kube-runtime/src/reflector/multi_dispatcher.rs index 22129e18f..3eb993df6 100644 --- a/kube-runtime/src/reflector/multi_dispatcher.rs +++ b/kube-runtime/src/reflector/multi_dispatcher.rs @@ -60,7 +60,7 @@ impl MultiDispatcher { pub(crate) async fn broadcast_event(&mut self, event: &watcher::Event) { match event { // Broadcast stores are pre-initialized - watcher::Event::InitDone => {} + watcher::Event::InitDone(_) => {} ev => { let _ = self.dispatch_tx.broadcast_direct(ev.clone()).await; } diff --git a/kube-runtime/src/reflector/store.rs b/kube-runtime/src/reflector/store.rs index f4158d610..00f4eff1e 100644 --- a/kube-runtime/src/reflector/store.rs +++ b/kube-runtime/src/reflector/store.rs @@ -101,23 +101,23 @@ where /// Applies a single watcher event to the store pub fn apply_watcher_event(&mut self, event: &watcher::Event) { match event { - watcher::Event::Apply(obj) => { + watcher::Event::Apply(obj, id) => { let obj = Arc::new(obj.clone()); - self.apply_shared_watcher_event(&watcher::Event::Apply(obj)); + self.apply_shared_watcher_event(&watcher::Event::Apply(obj, *id)); } - watcher::Event::Delete(obj) => { + watcher::Event::Delete(obj, id) => { let obj = Arc::new(obj.clone()); - self.apply_shared_watcher_event(&watcher::Event::Delete(obj)); + self.apply_shared_watcher_event(&watcher::Event::Delete(obj, *id)); } - watcher::Event::InitApply(obj) => { + watcher::Event::InitApply(obj, id) => { let obj = Arc::new(obj.clone()); - self.apply_shared_watcher_event(&watcher::Event::InitApply(obj)); + self.apply_shared_watcher_event(&watcher::Event::InitApply(obj, *id)); } - watcher::Event::Init => { - self.apply_shared_watcher_event(&watcher::Event::Init); + watcher::Event::Init(id) => { + self.apply_shared_watcher_event(&watcher::Event::Init(*id)); } - watcher::Event::InitDone => { - self.apply_shared_watcher_event(&watcher::Event::InitDone); + watcher::Event::InitDone(id) => { + self.apply_shared_watcher_event(&watcher::Event::InitDone(*id)); } } } @@ -125,22 +125,22 @@ where /// Applies a single shared watcher event to the store pub fn apply_shared_watcher_event(&mut self, event: &watcher::Event>) { match event { - watcher::Event::Apply(obj) => { + watcher::Event::Apply(obj, ..) => { let key = obj.to_object_ref(self.dyntype.clone()); self.store.write().insert(key, obj.clone()); } - watcher::Event::Delete(obj) => { + watcher::Event::Delete(obj, ..) => { let key = obj.to_object_ref(self.dyntype.clone()); self.store.write().remove(&key); } - watcher::Event::Init => { + watcher::Event::Init(_) => { self.buffer = AHashMap::new(); } - watcher::Event::InitApply(obj) => { + watcher::Event::InitApply(obj, ..) => { let key = obj.to_object_ref(self.dyntype.clone()); self.buffer.insert(key, obj.clone()); } - watcher::Event::InitDone => { + watcher::Event::InitDone(_) => { let mut store = self.store.write(); // Swap the buffer into the store @@ -163,14 +163,14 @@ where pub(crate) async fn dispatch_event(&mut self, event: &watcher::Event) { if let Some(ref mut dispatcher) = self.dispatcher { match event { - watcher::Event::Apply(obj) => { + watcher::Event::Apply(obj, id) => { let obj_ref = obj.to_object_ref(self.dyntype.clone()); // TODO (matei): should this take a timeout to log when backpressure has // been applied for too long, e.g. 10s dispatcher.broadcast(obj_ref).await; } - watcher::Event::InitDone => { + watcher::Event::InitDone(_) => { let obj_refs: Vec<_> = { let store = self.store.read(); store.keys().cloned().collect() @@ -346,7 +346,7 @@ mod tests { ..ConfigMap::default() }; let mut store_w = Writer::default(); - store_w.apply_watcher_event(&watcher::Event::Apply(cm.clone())); + store_w.apply_watcher_event(&watcher::Event::Apply(cm.clone(), None)); let store = store_w.as_reader(); assert_eq!(store.get(&ObjectRef::from_obj(&cm)).as_deref(), Some(&cm)); } @@ -364,7 +364,7 @@ mod tests { let mut cluster_cm = cm.clone(); cluster_cm.metadata.namespace = None; let mut store_w = Writer::default(); - store_w.apply_watcher_event(&watcher::Event::Apply(cm)); + store_w.apply_watcher_event(&watcher::Event::Apply(cm, None)); let store = store_w.as_reader(); assert_eq!(store.get(&ObjectRef::from_obj(&cluster_cm)), None); } @@ -380,7 +380,7 @@ mod tests { ..ConfigMap::default() }; let (store, mut writer) = store(); - writer.apply_watcher_event(&watcher::Event::Apply(cm.clone())); + writer.apply_watcher_event(&watcher::Event::Apply(cm.clone(), None)); assert_eq!(store.get(&ObjectRef::from_obj(&cm)).as_deref(), Some(&cm)); } @@ -398,7 +398,7 @@ mod tests { let mut nsed_cm = cm.clone(); nsed_cm.metadata.namespace = Some("ns".to_string()); let mut store_w = Writer::default(); - store_w.apply_watcher_event(&watcher::Event::Apply(cm.clone())); + store_w.apply_watcher_event(&watcher::Event::Apply(cm.clone(), None)); let store = store_w.as_reader(); assert_eq!(store.get(&ObjectRef::from_obj(&nsed_cm)).as_deref(), Some(&cm)); } @@ -417,14 +417,14 @@ mod tests { let (reader, mut writer) = store::(); assert!(reader.is_empty()); - writer.apply_watcher_event(&watcher::Event::Apply(cm)); + writer.apply_watcher_event(&watcher::Event::Apply(cm, None)); assert_eq!(reader.len(), 1); assert!(reader.find(|k| k.metadata.generation == Some(1234)).is_none()); target_cm.metadata.name = Some("obj1".to_string()); target_cm.metadata.generation = Some(1234); - writer.apply_watcher_event(&watcher::Event::Apply(target_cm.clone())); + writer.apply_watcher_event(&watcher::Event::Apply(target_cm.clone(), None)); assert!(!reader.is_empty()); assert_eq!(reader.len(), 2); let found = reader.find(|k| k.metadata.generation == Some(1234)); diff --git a/kube-runtime/src/utils/event_decode.rs b/kube-runtime/src/utils/event_decode.rs index 2a0085120..b2991e220 100644 --- a/kube-runtime/src/utils/event_decode.rs +++ b/kube-runtime/src/utils/event_decode.rs @@ -29,15 +29,15 @@ where let mut me = self.project(); Poll::Ready(loop { let var_name = match ready!(me.stream.as_mut().poll_next(cx)) { - Some(Ok(Event::Apply(obj) | Event::InitApply(obj))) => Some(Ok(obj)), - Some(Ok(Event::Delete(obj))) => { + Some(Ok(Event::Apply(obj, ..) | Event::InitApply(obj, ..))) => Some(Ok(obj)), + Some(Ok(Event::Delete(obj, ..))) => { if *me.emit_deleted { Some(Ok(obj)) } else { continue; } } - Some(Ok(Event::Init | Event::InitDone)) => continue, + Some(Ok(Event::Init(_) | Event::InitDone(_))) => continue, Some(Err(err)) => Some(Err(err)), None => return Poll::Ready(None), }; @@ -56,14 +56,14 @@ pub(crate) mod tests { #[tokio::test] async fn watches_applies_uses_correct_stream() { let data = stream::iter([ - Ok(Event::Apply(0)), - Ok(Event::Apply(1)), - Ok(Event::Delete(0)), - Ok(Event::Apply(2)), - Ok(Event::InitApply(1)), - Ok(Event::InitApply(2)), + Ok(Event::Apply(0, None)), + Ok(Event::Apply(1, None)), + Ok(Event::Delete(0, None)), + Ok(Event::Apply(2, None)), + Ok(Event::InitApply(1, None)), + Ok(Event::InitApply(2, None)), Err(Error::NoResourceVersion), - Ok(Event::Apply(2)), + Ok(Event::Apply(2, None)), ]); let mut rx = pin!(EventDecode::new(data, false)); assert!(matches!(poll!(rx.next()), Poll::Ready(Some(Ok(0))))); diff --git a/kube-runtime/src/utils/event_modify.rs b/kube-runtime/src/utils/event_modify.rs index 39a2ec909..c04f5c8da 100644 --- a/kube-runtime/src/utils/event_modify.rs +++ b/kube-runtime/src/utils/event_modify.rs @@ -54,9 +54,9 @@ pub(crate) mod test { #[tokio::test] async fn eventmodify_modifies_innner_value_of_event() { let st = stream::iter([ - Ok(Event::Apply(0)), + Ok(Event::Apply(0, None)), Err(Error::NoResourceVersion), - Ok(Event::InitApply(10)), + Ok(Event::InitApply(10, None)), ]); let mut ev_modify = pin!(EventModify::new(st, |x| { *x += 1; @@ -64,7 +64,7 @@ pub(crate) mod test { assert!(matches!( poll!(ev_modify.next()), - Poll::Ready(Some(Ok(Event::Apply(1)))) + Poll::Ready(Some(Ok(Event::Apply(1, None)))) )); assert!(matches!( @@ -75,7 +75,7 @@ pub(crate) mod test { let restarted = poll!(ev_modify.next()); assert!(matches!( restarted, - Poll::Ready(Some(Ok(Event::InitApply(x)))) if x == 11 + Poll::Ready(Some(Ok(Event::InitApply(x, _)))) if x == 11 )); assert!(matches!(poll!(ev_modify.next()), Poll::Ready(None))); diff --git a/kube-runtime/src/utils/reflect.rs b/kube-runtime/src/utils/reflect.rs index e93354202..035f570e0 100644 --- a/kube-runtime/src/utils/reflect.rs +++ b/kube-runtime/src/utils/reflect.rs @@ -72,12 +72,12 @@ pub(crate) mod test { let foo = testpod("foo"); let bar = testpod("bar"); let st = stream::iter([ - Ok(Event::Apply(foo.clone())), + Ok(Event::Apply(foo.clone(), None)), Err(Error::NoResourceVersion), - Ok(Event::Init), - Ok(Event::InitApply(foo)), - Ok(Event::InitApply(bar)), - Ok(Event::InitDone), + Ok(Event::Init(None)), + Ok(Event::InitApply(foo, None)), + Ok(Event::InitApply(bar, None)), + Ok(Event::InitDone(None)), ]); let (reader, writer) = reflector::store(); @@ -86,7 +86,7 @@ pub(crate) mod test { assert!(matches!( poll!(reflect.next()), - Poll::Ready(Some(Ok(Event::Apply(_)))) + Poll::Ready(Some(Ok(Event::Apply(..)))) )); assert_eq!(reader.len(), 1); @@ -98,20 +98,20 @@ pub(crate) mod test { assert!(matches!( poll!(reflect.next()), - Poll::Ready(Some(Ok(Event::Init))) + Poll::Ready(Some(Ok(Event::Init(None)))) )); assert_eq!(reader.len(), 1); let restarted = poll!(reflect.next()); - assert!(matches!(restarted, Poll::Ready(Some(Ok(Event::InitApply(_)))))); + assert!(matches!(restarted, Poll::Ready(Some(Ok(Event::InitApply(..)))))); assert_eq!(reader.len(), 1); let restarted = poll!(reflect.next()); - assert!(matches!(restarted, Poll::Ready(Some(Ok(Event::InitApply(_)))))); + assert!(matches!(restarted, Poll::Ready(Some(Ok(Event::InitApply(..)))))); assert_eq!(reader.len(), 1); assert!(matches!( poll!(reflect.next()), - Poll::Ready(Some(Ok(Event::InitDone))) + Poll::Ready(Some(Ok(Event::InitDone(None)))) )); assert_eq!(reader.len(), 2); diff --git a/kube-runtime/src/watcher.rs b/kube-runtime/src/watcher.rs index 755320b38..f136b74b5 100644 --- a/kube-runtime/src/watcher.rs +++ b/kube-runtime/src/watcher.rs @@ -38,17 +38,17 @@ pub type Result = std::result::Result; /// Watch events returned from the [`watcher`] pub enum Event { /// An object was added or modified - Apply(K), + Apply(K, Option), /// An object was deleted /// /// NOTE: This should not be used for managing persistent state elsewhere, since /// events may be lost if the watcher is unavailable. Use Finalizers instead. - Delete(K), + Delete(K, Option), /// The watch stream was restarted. /// /// A series of `InitApply` events are expected to follow until all matching objects /// have been listed. This event can be used to prepare a buffer for `InitApply` events. - Init, + Init(Option), /// Received an object during `Init`. /// /// Objects returned here are either from the initial stream using the `StreamingList` strategy, @@ -56,7 +56,7 @@ pub enum Event { /// /// These events can be passed up if having a complete set of objects is not a concern. /// If you need to wait for a complete set, please buffer these events until an `InitDone`. - InitApply(K), + InitApply(K, Option), /// The initialisation is complete. /// /// This can be used as a signal to replace buffered store contents atomically. @@ -64,7 +64,7 @@ pub enum Event { /// /// Any objects that were previously [`Applied`](Event::Applied) but are not listed in any of /// the `InitApply` events should be assumed to have been [`Deleted`](Event::Deleted). - InitDone, + InitDone(Option), } impl Event { @@ -78,8 +78,8 @@ impl Event { )] pub fn into_iter_applied(self) -> impl Iterator { match self { - Self::Apply(obj) | Self::InitApply(obj) => Some(obj), - Self::Delete(_) | Self::Init | Self::InitDone => None, + Self::Apply(obj, ..) | Self::InitApply(obj, ..) => Some(obj), + Self::Delete(..) | Self::Init(..) | Self::InitDone(..) => None, } .into_iter() } @@ -95,8 +95,8 @@ impl Event { )] pub fn into_iter_touched(self) -> impl Iterator { match self { - Self::Apply(obj) | Self::Delete(obj) | Self::InitApply(obj) => Some(obj), - Self::Init | Self::InitDone => None, + Self::Apply(obj, ..) | Self::Delete(obj, ..) | Self::InitApply(obj, ..) => Some(obj), + Self::Init(_) | Self::InitDone(_) => None, } .into_iter() } @@ -121,8 +121,8 @@ impl Event { #[must_use] pub fn modify(mut self, mut f: impl FnMut(&mut K)) -> Self { match &mut self { - Self::Apply(obj) | Self::Delete(obj) | Self::InitApply(obj) => (f)(obj), - Self::Init | Self::InitDone => {} // markers, nothing to modify + Self::Apply(obj, ..) | Self::Delete(obj, ..) | Self::InitApply(obj, ..) => (f)(obj), + Self::Init(_) | Self::InitDone(_) => {} // markers, nothing to modify } self } @@ -279,6 +279,9 @@ pub struct Config { /// Requests watch bookmarks from the apiserver when enabled for improved watch precision and reduced list calls. /// This is default enabled and should generally not be turned off. pub bookmarks: bool, + + /// Custom identifier for this watcher config, which will be assigned to initial events for objects distinction. + pub identifier: Option, } impl Default for Config { @@ -293,6 +296,7 @@ impl Default for Config { // https://github.com/kubernetes/client-go/blob/aed71fa5cf054e1c196d67b2e21f66fd967b8ab1/tools/pager/pager.go#L31 page_size: Some(500), initial_list_strategy: InitialListStrategy::ListWatch, + identifier: None, } } } @@ -406,6 +410,13 @@ impl Config { self } + /// Sets a unique identifier for the watcher config + #[must_use] + fn identified(mut self, identifier: u64) -> Self { + self.identifier = Some(identifier); + self + } + /// Converts generic `watcher::Config` structure to the instance of `ListParams` used for list requests. fn to_list_params(&self) -> ListParams { let (resource_version, version_match) = match self.list_semantic { @@ -498,11 +509,14 @@ where { match state { State::Empty => match wc.initial_list_strategy { - InitialListStrategy::ListWatch => (Some(Ok(Event::Init)), State::InitPage { - continue_token: None, - objects: VecDeque::default(), - last_bookmark: None, - }), + InitialListStrategy::ListWatch => ( + Some(Ok(Event::Init(wc.identifier))), + State::InitPage { + continue_token: None, + objects: VecDeque::default(), + last_bookmark: None, + }, + ), InitialListStrategy::StreamingList => match api.watch(&wc.to_watch_params(), "0").await { Ok(stream) => (None, State::InitialWatch { stream }), Err(err) => { @@ -521,17 +535,20 @@ where last_bookmark, } => { if let Some(next) = objects.pop_front() { - return (Some(Ok(Event::InitApply(next))), State::InitPage { - continue_token, - objects, - last_bookmark, - }); + return ( + Some(Ok(Event::InitApply(next, wc.identifier))), + State::InitPage { + continue_token, + objects, + last_bookmark, + }, + ); } // check if we need to perform more pages if continue_token.is_none() { if let Some(resource_version) = last_bookmark { // we have drained the last page - move on to next stage - return (Some(Ok(Event::InitDone)), State::InitListed { resource_version }); + return (Some(Ok(Event::InitDone(wc.identifier))), State::InitListed { resource_version }); } } let mut lp = wc.to_list_params(); @@ -545,11 +562,14 @@ where } // Buffer page here, causing us to return to this enum branch (State::InitPage) // until the objects buffer has drained - (None, State::InitPage { - continue_token, - objects: list.items.into_iter().collect(), - last_bookmark, - }) + ( + None, + State::InitPage { + continue_token, + objects: list.items.into_iter().collect(), + last_bookmark, + }, + ) } Err(err) => { if std::matches!(err, ClientErr::Api(ErrorResponse { code: 403, .. })) { @@ -564,7 +584,7 @@ where State::InitialWatch { mut stream } => { match stream.next().await { Some(Ok(WatchEvent::Added(obj) | WatchEvent::Modified(obj))) => { - (Some(Ok(Event::InitApply(obj))), State::InitialWatch { stream }) + (Some(Ok(Event::InitApply(obj, wc.identifier))), State::InitialWatch { stream }) } Some(Ok(WatchEvent::Deleted(_obj))) => { // Kubernetes claims these events are impossible @@ -575,10 +595,13 @@ where Some(Ok(WatchEvent::Bookmark(bm))) => { let marks_initial_end = bm.metadata.annotations.contains_key("k8s.io/initial-events-end"); if marks_initial_end { - (Some(Ok(Event::InitDone)), State::Watching { - resource_version: bm.metadata.resource_version, - stream, - }) + ( + Some(Ok(Event::InitDone(wc.identifier))), + State::Watching { + resource_version: bm.metadata.resource_version, + stream, + }, + ) } else { (None, State::InitialWatch { stream }) } @@ -610,19 +633,23 @@ where } State::InitListed { resource_version } => { match api.watch(&wc.to_watch_params(), &resource_version).await { - Ok(stream) => (None, State::Watching { - resource_version, - stream, - }), + Ok(stream) => ( + None, + State::Watching { + resource_version, + stream, + }, + ), Err(err) => { if std::matches!(err, ClientErr::Api(ErrorResponse { code: 403, .. })) { warn!("watch initlist error with 403: {err:?}"); } else { debug!("watch initlist error: {err:?}"); } - (Some(Err(Error::WatchStartFailed(err))), State::InitListed { - resource_version, - }) + ( + Some(Err(Error::WatchStartFailed(err))), + State::InitListed { resource_version }, + ) } } } @@ -635,10 +662,13 @@ where if resource_version.is_empty() { (Some(Err(Error::NoResourceVersion)), State::default()) } else { - (Some(Ok(Event::Apply(obj))), State::Watching { - resource_version, - stream, - }) + ( + Some(Ok(Event::Apply(obj, wc.identifier))), + State::Watching { + resource_version, + stream, + }, + ) } } Some(Ok(WatchEvent::Deleted(obj))) => { @@ -646,16 +676,22 @@ where if resource_version.is_empty() { (Some(Err(Error::NoResourceVersion)), State::default()) } else { - (Some(Ok(Event::Delete(obj))), State::Watching { - resource_version, - stream, - }) + ( + Some(Ok(Event::Delete(obj, wc.identifier))), + State::Watching { + resource_version, + stream, + }, + ) } } - Some(Ok(WatchEvent::Bookmark(bm))) => (None, State::Watching { - resource_version: bm.metadata.resource_version, - stream, - }), + Some(Ok(WatchEvent::Bookmark(bm))) => ( + None, + State::Watching { + resource_version: bm.metadata.resource_version, + stream, + }, + ), Some(Ok(WatchEvent::Error(err))) => { // HTTP GONE, means we have desynced and need to start over and re-list :( let new_state = if err.code == 410 { @@ -679,10 +715,13 @@ where } else { debug!("watcher error: {err:?}"); } - (Some(Err(Error::WatchFailed(err))), State::Watching { - resource_version, - stream, - }) + ( + Some(Err(Error::WatchFailed(err))), + State::Watching { + resource_version, + stream, + }, + ) } None => (None, State::InitListed { resource_version }), }, @@ -860,9 +899,9 @@ pub fn watch_object Some(Ok(Some(obj))), + Ok(Event::Apply(obj, ..) | Event::InitApply(obj, ..)) => Some(Ok(Some(obj))), // Pass up `None` for Deleted - Ok(Event::Delete(_)) => Some(Ok(None)), + Ok(Event::Delete(..)) => Some(Ok(None)), // Pass up `None` if the object wasn't seen in the initial list - Ok(Event::InitDone) if !obj_seen => Some(Ok(None)), + Ok(Event::InitDone(_)) if !obj_seen => Some(Ok(None)), // Ignore marker events - Ok(Event::Init | Event::InitDone) => None, + Ok(Event::Init(_) | Event::InitDone(_)) => None, // Bubble up errors Err(err) => Some(Err(err)), }