stats: optional per-endpoint metrics (#29709)

Signed-off-by: Greg Greenway <[email protected]>

Author: ggreenway

Diff for: api/envoy/config/cluster/v3/cluster.proto

@@ -1257,4 +1257,19 @@ message TrackClusterStats {
   // <config_cluster_manager_cluster_stats_request_response_sizes>`  tracking header and body sizes
   // of requests and responses will be published.
   bool request_response_sizes = 2;
+
+  // If true, some stats will be emitted per-endpoint, similar to the stats in admin ``/clusters``
+  // output.
+  //
+  // This does not currently output correct stats during a hot-restart.
+  //
+  // This is not currently implemented by all stat sinks.
+  //
+  // These stats do not honor filtering or tag extraction rules in :ref:`StatsConfig
+  // <envoy_v3_api_msg_config.metrics.v3.StatsConfig>` (but fixed-value tags are supported). Admin
+  // endpoint filtering is supported.
+  //
+  // This may not be used at the same time as
+  // :ref:`load_stats_config <envoy_v3_api_field_config.bootstrap.v3.ClusterManager.load_stats_config>`.
+  bool per_endpoint_stats = 3;
 }

Diff for: changelogs/current.yaml

@@ -56,6 +56,10 @@ removed_config_or_runtime:
     runtime flag and legacy code path.
 
 new_features:
+- area: stats
+  change: |
+    added :ref:`per_endpoint_stats <envoy_v3_api_field_config.cluster.v3.TrackClusterStats.per_endpoint_stats>` to get some metrics
+    for each endpoint in a cluster.
 - area: jwt
   change: |
     The jwt filter can now serialize non-primitive custom claims when maping claims to headers.

Diff for: envoy/server/BUILD

@@ -43,6 +43,7 @@ envoy_cc_library(
     external_deps = ["abseil_optional"],
     deps = [
         "//envoy/http:context_interface",
+        "//envoy/stats:sink_interface",
         "//envoy/upstream:cluster_manager_interface",
         "@envoy_api//envoy/config/bootstrap/v3:pkg_cc_proto",
     ],

Diff for: envoy/stats/BUILD

@@ -14,36 +14,50 @@ envoy_cc_library(
     deps = ["//source/common/common:assert_lib"],
 )
 
+envoy_cc_library(
+    name = "tag_interface",
+    hdrs = ["tag.h"],
+)
+
 # TODO(jmarantz): atomize the build rules to match the include files.
 envoy_cc_library(
     name = "stats_interface",
     hdrs = [
         "allocator.h",
         "histogram.h",
         "scope.h",
-        "sink.h",
         "stats.h",
         "stats_matcher.h",
         "store.h",
-        "tag.h",
         "tag_extractor.h",
         "tag_producer.h",
     ],
     external_deps = ["abseil_inlined_vector"],
     deps = [
         ":refcount_ptr_interface",
+        ":tag_interface",
         "//envoy/common:interval_set_interface",
         "//envoy/common:optref_lib",
         "//envoy/common:time_interface",
     ],
 )
 
+envoy_cc_library(
+    name = "sink_interface",
+    hdrs = ["sink.h"],
+    deps = [
+        ":primitive_stats_interface",
+        ":stats_interface",
+    ],
+)
+
 envoy_cc_library(
     name = "primitive_stats_interface",
     hdrs = [
         "primitive_stats.h",
     ],
     deps = [
+        ":tag_interface",
         "//source/common/common:assert_lib",
         "//source/common/common:non_copyable",
     ],

Diff for: envoy/stats/primitive_stats.h

@@ -2,6 +2,8 @@
 
 #include <string>
 
+#include "envoy/stats/tag.h"
+
 #include "source/common/common/assert.h"
 #include "source/common/common/non_copyable.h"
 
@@ -34,7 +36,7 @@ class PrimitiveCounter : NonCopyable {
   std::atomic<uint64_t> pending_increment_{0};
 };
 
-using PrimitiveCounterReference = std::reference_wrapper<const PrimitiveCounter>;
+using PrimitiveCounterReference = std::reference_wrapper<PrimitiveCounter>;
 
 /**
  * Primitive, low-memory-overhead gauge with increment and decrement capabilities.
@@ -58,7 +60,52 @@ class PrimitiveGauge : NonCopyable {
   std::atomic<uint64_t> value_{0};
 };
 
-using PrimitiveGaugeReference = std::reference_wrapper<const PrimitiveGauge>;
+using PrimitiveGaugeReference = std::reference_wrapper<PrimitiveGauge>;
+
+class PrimitiveMetricMetadata {
+public:
+  // Mirror some of the API for Stats::Metric for use in templates that
+  // accept either Counter/Gauge or PrimitiveCounterSnapshot/PrimitiveGaugeSnapshot.
+  const std::string& tagExtractedName() const { return tag_extracted_name_; }
+  const std::string& name() const { return name_; }
+  const Stats::TagVector& tags() const { return tags_; }
+  bool used() const { return true; }
+  bool hidden() const { return false; }
+
+  void setName(std::string&& name) { name_ = std::move(name); }
+  void setTagExtractedName(std::string&& tag_extracted_name) {
+    tag_extracted_name_ = std::move(tag_extracted_name);
+  }
+  void setTags(const Stats::TagVector& tags) { tags_ = tags; }
+
+private:
+  std::string name_;
+  std::string tag_extracted_name_;
+  Stats::TagVector tags_;
+};
+
+class PrimitiveCounterSnapshot : public PrimitiveMetricMetadata {
+public:
+  PrimitiveCounterSnapshot(PrimitiveCounter& counter)
+      : value_(counter.value()), delta_(counter.latch()) {}
+
+  uint64_t value() const { return value_; }
+  uint64_t delta() const { return delta_; }
+
+private:
+  const uint64_t value_;
+  const uint64_t delta_;
+};
+
+class PrimitiveGaugeSnapshot : public PrimitiveMetricMetadata {
+public:
+  PrimitiveGaugeSnapshot(PrimitiveGauge& gauge) : value_(gauge.value()) {}
+
+  uint64_t value() const { return value_; }
+
+private:
+  const uint64_t value_;
+};
 
 } // namespace Stats
 } // namespace Envoy

Diff for: envoy/stats/sink.h

@@ -6,6 +6,7 @@
 #include "envoy/common/pure.h"
 #include "envoy/common/time.h"
 #include "envoy/stats/histogram.h"
+#include "envoy/stats/primitive_stats.h"
 #include "envoy/stats/stats.h"
 
 namespace Envoy {
@@ -42,6 +43,16 @@ class MetricSnapshot {
    */
   virtual const std::vector<std::reference_wrapper<const TextReadout>>& textReadouts() PURE;
 
+  /**
+   * @return a snapshot of all host/endpoint-specific primitive counters.
+   */
+  virtual const std::vector<Stats::PrimitiveCounterSnapshot>& hostCounters() PURE;
+
+  /**
+   * @return a snapshot of all host/endpoint-specific primitive gauges.
+   */
+  virtual const std::vector<Stats::PrimitiveGaugeSnapshot>& hostGauges() PURE;
+
   /**
    * @return the time in UTC since epoch when the snapshot was created.
    */

Diff for: envoy/stats/stats.h

@@ -26,6 +26,10 @@ class SymbolTable;
 
 /**
  * General interface for all stats objects.
+ *
+ * Note: some methods must match those in `PrimitiveMetricMetadata` because stats sinks
+ * use templates to handle either type. The interface is not used for size/performance
+ * reasons.
  */
 class Metric : public RefcountInterface {
 public:

Diff for: envoy/stats/store.h

@@ -193,6 +193,11 @@ class Store {
    */
   virtual void extractAndAppendTags(absl::string_view name, StatNamePool& pool,
                                     StatNameTagVector& stat_tags) PURE;
+
+  /**
+   * Returns the configured fixed tags (which don't depend on the name of the stat).
+   */
+  virtual const TagVector& fixedTags() PURE;
 };
 
 using StorePtr = std::unique_ptr<Store>;

Diff for: envoy/stats/tag_producer.h

@@ -30,6 +30,8 @@ class TagProducer {
    * @param tags TagVector a set of Stats::Tag.
    */
   virtual std::string produceTags(absl::string_view metric_name, TagVector& tags) const PURE;
+
+  virtual const TagVector& fixedTags() const PURE;
 };
 
 using TagProducerPtr = std::unique_ptr<const TagProducer>;

Diff for: envoy/upstream/host_description.h

@@ -46,12 +46,12 @@ struct HostStats {
   ALL_HOST_STATS(GENERATE_PRIMITIVE_COUNTER_STRUCT, GENERATE_PRIMITIVE_GAUGE_STRUCT);
 
   // Provide access to name,counter pairs.
-  std::vector<std::pair<absl::string_view, Stats::PrimitiveCounterReference>> counters() const {
+  std::vector<std::pair<absl::string_view, Stats::PrimitiveCounterReference>> counters() {
     return {ALL_HOST_STATS(PRIMITIVE_COUNTER_NAME_AND_REFERENCE, IGNORE_PRIMITIVE_GAUGE)};
   }
 
   // Provide access to name,gauge pairs.
-  std::vector<std::pair<absl::string_view, Stats::PrimitiveGaugeReference>> gauges() const {
+  std::vector<std::pair<absl::string_view, Stats::PrimitiveGaugeReference>> gauges() {
     return {ALL_HOST_STATS(IGNORE_PRIMITIVE_COUNTER, PRIMITIVE_GAUGE_NAME_AND_REFERENCE)};
   }
 };

Diff for: envoy/upstream/upstream.h

@@ -1172,6 +1172,11 @@ class ClusterInfo : public Http::FilterChainFactory {
    */
   virtual ClusterTimeoutBudgetStatsOptRef timeoutBudgetStats() const PURE;
 
+  /**
+   * @return true if this cluster should produce per-endpoint stats.
+   */
+  virtual bool perEndpointStatsEnabled() const PURE;
+
   /**
    * @return std::shared_ptr<UpstreamLocalAddressSelector> as upstream local address selector.
    */

Diff for: source/common/stats/BUILD

@@ -15,6 +15,7 @@ envoy_cc_library(
     deps = [
         ":metric_impl_lib",
         ":stat_merger_lib",
+        "//envoy/stats:sink_interface",
         "//source/common/common:assert_lib",
         "//source/common/common:hash_lib",
         "//source/common/common:thread_annotations",

Diff for: source/common/stats/isolated_store_impl.h

@@ -236,6 +236,7 @@ class IsolatedStoreImpl : public Store {
 
   void extractAndAppendTags(StatName, StatNamePool&, StatNameTagVector&) override {}
   void extractAndAppendTags(absl::string_view, StatNamePool&, StatNameTagVector&) override {}
+  const TagVector& fixedTags() override { CONSTRUCT_ON_FIRST_USE(TagVector); }
 
 protected:
   /**

Diff for: source/common/stats/tag_producer_impl.cc

@@ -21,6 +21,7 @@ TagProducerImpl::TagProducerImpl(const envoy::config::metrics::v3::StatsConfig&
 
   for (const auto& cli_tag : cli_tags) {
     addExtractor(std::make_unique<TagExtractorFixedImpl>(cli_tag.name_, cli_tag.value_));
+    fixed_tags_.push_back(cli_tag);
   }
 
   for (const auto& tag_specifier : config.stats_tags()) {
@@ -43,6 +44,7 @@ TagProducerImpl::TagProducerImpl(const envoy::config::metrics::v3::StatsConfig&
     } else if (tag_specifier.tag_value_case() ==
                envoy::config::metrics::v3::TagSpecifier::TagValueCase::kFixedValue) {
       addExtractor(std::make_unique<TagExtractorFixedImpl>(name, tag_specifier.fixed_value()));
+      fixed_tags_.push_back(Tag{name, tag_specifier.fixed_value()});
     }
   }
 }

Diff for: source/common/stats/tag_producer_impl.h

@@ -43,6 +43,8 @@ class TagProducerImpl : public TagProducer {
    */
   std::string produceTags(absl::string_view metric_name, TagVector& tags) const override;
 
+  const TagVector& fixedTags() const override { return fixed_tags_; }
+
 private:
   friend class DefaultTagRegexTester;
 
@@ -104,6 +106,8 @@ class TagProducerImpl : public TagProducer {
   // we need do elide duplicate extractors during extraction. It is not valid to
   // send duplicate tag names to Prometheus so this needs to be filtered out.
   absl::flat_hash_map<absl::string_view, std::reference_wrapper<TagExtractor>> extractor_map_;
+
+  TagVector fixed_tags_;
 };
 
 } // namespace Stats

Diff for: source/common/stats/thread_local_store.h

@@ -224,6 +224,7 @@ class ThreadLocalStoreImpl : Logger::Loggable<Logger::Id::stats>, public StoreRo
   void extractAndAppendTags(StatName name, StatNamePool& pool, StatNameTagVector& tags) override;
   void extractAndAppendTags(absl::string_view name, StatNamePool& pool,
                             StatNameTagVector& tags) override;
+  const TagVector& fixedTags() override { return tag_producer_->fixedTags(); };
 
 private:
   friend class ThreadLocalStoreTestingPeer;

Diff for: source/common/upstream/BUILD

@@ -250,8 +250,10 @@ envoy_cc_library(
     srcs = ["host_utility.cc"],
     hdrs = ["host_utility.h"],
     deps = [
+        "//envoy/stats:primitive_stats_interface",
         "//envoy/upstream:load_balancer_interface",
         "//envoy/upstream:upstream_interface",
+        "//source/common/config:well_known_names",
         "//source/common/runtime:runtime_lib",
     ],
 )

Diff for: source/common/upstream/host_utility.cc

@@ -2,6 +2,7 @@
 
 #include <string>
 
+#include "source/common/config/well_known_names.h"
 #include "source/common/runtime/runtime_features.h"
 
 namespace Envoy {
@@ -153,5 +154,71 @@ HostConstSharedPtr HostUtility::selectOverrideHost(const HostMap* host_map, Host
   return nullptr;
 }
 
+void HostUtility::forEachHostMetric(
+    const ClusterManager& cluster_manager,
+    const std::function<void(Stats::PrimitiveCounterSnapshot&& metric)>& counter_cb,
+    const std::function<void(Stats::PrimitiveGaugeSnapshot&& metric)>& gauge_cb) {
+  for (const auto& [unused_name, cluster_ref] : cluster_manager.clusters().active_clusters_) {
+    Upstream::ClusterInfoConstSharedPtr cluster_info = cluster_ref.get().info();
+    if (cluster_info->perEndpointStatsEnabled()) {
+      const std::string cluster_name =
+          Stats::Utility::sanitizeStatsName(cluster_info->observabilityName());
+
+      const Stats::TagVector& fixed_tags = cluster_info->statsScope().store().fixedTags();
+
+      for (auto& host_set : cluster_ref.get().prioritySet().hostSetsPerPriority()) {
+        for (auto& host : host_set->hosts()) {
+
+          Stats::TagVector tags;
+          tags.reserve(fixed_tags.size() + 3);
+          tags.insert(tags.end(), fixed_tags.begin(), fixed_tags.end());
+          tags.emplace_back(Stats::Tag{Envoy::Config::TagNames::get().CLUSTER_NAME, cluster_name});
+          tags.emplace_back(Stats::Tag{"envoy.endpoint_address", host->address()->asString()});
+
+          const auto& hostname = host->hostname();
+          if (!hostname.empty()) {
+            tags.push_back({"envoy.endpoint_hostname", hostname});
+          }
+
+          auto set_metric_metadata = [&](absl::string_view metric_name,
+                                         Stats::PrimitiveMetricMetadata& metric) {
+            metric.setName(
+                absl::StrCat("cluster.", cluster_name, ".endpoint.",
+                             Stats::Utility::sanitizeStatsName(host->address()->asStringView()),
+                             ".", metric_name));
+            metric.setTagExtractedName(absl::StrCat("cluster.endpoint.", metric_name));
+            metric.setTags(tags);
+
+            // Validate that all components were sanitized.
+            ASSERT(metric.name() == Stats::Utility::sanitizeStatsName(metric.name()));
+            ASSERT(metric.tagExtractedName() ==
+                   Stats::Utility::sanitizeStatsName(metric.tagExtractedName()));
+          };
+
+          for (auto& [metric_name, primitive] : host->counters()) {
+            Stats::PrimitiveCounterSnapshot metric(primitive.get());
+            set_metric_metadata(metric_name, metric);
+
+            counter_cb(std::move(metric));
+          }
+
+          auto gauges = host->gauges();
+
+          // Add synthetic "healthy" gauge.
+          Stats::PrimitiveGauge healthy_gauge;
+          healthy_gauge.set((host->coarseHealth() == Host::Health::Healthy) ? 1 : 0);
+          gauges.emplace_back(absl::string_view("healthy"), healthy_gauge);
+
+          for (auto& [metric_name, primitive] : gauges) {
+            Stats::PrimitiveGaugeSnapshot metric(primitive.get());
+            set_metric_metadata(metric_name, metric);
+            gauge_cb(std::move(metric));
+          }
+        }
+      }
+    }
+  }
+}
+
 } // namespace Upstream
 } // namespace Envoy