addid

Author: yangw-dev

aws/lambda/benchmark_regression_summary_report/common/config.py

@@ -48,13 +48,13 @@
         ),
         metrics={
             "passrate": RegressionPolicy(
-                name="passrate", condition="greater_than", threshold=0.9, baseline_aggregation="max"
+                name="passrate", condition="greater_equal", threshold=0.9, baseline_aggregation="max",
             ),
             "geomean": RegressionPolicy(
-                name="geomean", condition="greater_than", threshold=0.95,baseline_aggregation="max"
+                name="geomean", condition="greater_equal", threshold=0.95,baseline_aggregation="max",
             ),
             "compression_ratio": RegressionPolicy(
-                name="compression_ratio", condition="greater_than", threshold=0.9, baseline_aggregation="max"
+                name="compression_ratio", condition="greater_equal", threshold=0.9, baseline_aggregation="max",
             ),
         },
         notification_config={

aws/lambda/benchmark_regression_summary_report/common/config_model.py

@@ -98,12 +98,12 @@ def baseline_timedelta(self) -> timedelta:
 class RegressionPolicy:
     """
     Defines the policy for a given metric.
-        - "greater_than": higher is better; violation if new value < baseline * threshold
-        - "less_than":    lower  is better; violation if new value > baseline * threshold
+    - new value muset be {x} baseline value:
+        - "greater_than": higher is better; new value must be strictly greater to baseline
+        - "less_than":    lower  is better; new value must be strictly lower to baseline
         - "equal_to":     new value should be ~= baseline * threshold within rel_tol
-        - "greater_equal": higher is better; violation if new value <= baseline * threshold
-        - "less_equal":    lower  is better; violation if new value >= baseline * threshold
-
+        - "greater_equal": higher is better; new value must be greater or equal to baseline
+        - "less_equal":    lower  is better; new value must be less or equal to baseline
     """
     name: str
     condition: Literal["greater_than", "less_than", "equal_to","greater_equal","less_equal"]

aws/lambda/benchmark_regression_summary_report/lambda_function.py

@@ -88,7 +88,6 @@ def process(
     ):
         # ensure each thread has its own clickhouse client. clickhouse client
         # is not thread-safe.
-        logger.info("here")
         if cc is None:
             tlocal = threading.local()
             if not hasattr(tlocal, "cc") or tlocal.cc is None:
@@ -101,8 +100,6 @@ def process(
                 else:
                     tlocal.cc = get_clickhouse_client_environment()
             cc = tlocal.cc
-        logger.info("i'm here")
-
         try:
             config = get_benchmark_regression_config(config_id)
             logger.info("found config for config_id %s",config_id)
@@ -215,7 +212,7 @@ def _fetch_from_benchmark_ts_api(
         )
         url = source.api_query_url
 
-        logger.info("trying to call %s",url)
+        logger.info("[%s]trying to call %s, with query %s",config_id, url,query)
         try:
             resp: BenchmarkTimeSeriesApiResponse = (
                 BenchmarkTimeSeriesApiResponse.from_request(url, query)