From 5b385b5dab89d62fe986b9d38d1a022bc18ad8d0 Mon Sep 17 00:00:00 2001 From: Odrec Date: Fri, 29 May 2026 12:24:19 -0600 Subject: [PATCH] fix: split rating feedback tags by thumbsUp/thumbsDown direction The per-tag rating metric grouped on feedback.tag alone, dropping the rating direction. Since each LibreChat feedback tag belongs to exactly one rating (thumbsUp/thumbsDown), positive and negative tags could not be told apart in Grafana. - Group Query 3 by {tag, rating} and add a `rating` label to librechat_rating_counts_per_tag. - Update README examples to the current LibreChat tag keys and the new label (drops stale 'helpful'/'creative'/'unhelpful' tags). - Add Positive/Negative Feedback Tags panels to the Grafana template. Closes #64 --- README.md | 12 +++--- metrics.py | 27 ++++++++---- .../Grafana-Dashboard-template.json | 42 +++++++++++++++---- 3 files changed, 59 insertions(+), 22 deletions(-) diff --git a/README.md b/README.md index 7320e46..717eeff 100644 --- a/README.md +++ b/README.md @@ -312,11 +312,11 @@ librechat_thumbs_down_per_model{model="claude-3"} 2.0 librechat_rating_ratio_per_model{model="gpt-4"} 88.9 librechat_rating_ratio_per_model{model="claude-3"} 77.8 -# HELP librechat_rating_counts_per_tag Number of ratings per feedback tag +# HELP librechat_rating_counts_per_tag Number of ratings per feedback tag and rating direction # TYPE librechat_rating_counts_per_tag gauge -librechat_rating_counts_per_tag{tag="accurate_reliable"} 8.0 -librechat_rating_counts_per_tag{tag="helpful"} 6.0 -librechat_rating_counts_per_tag{tag="creative"} 4.0 +librechat_rating_counts_per_tag{tag="accurate_reliable",rating="thumbsUp"} 8.0 +librechat_rating_counts_per_tag{tag="clear_well_written",rating="thumbsUp"} 6.0 +librechat_rating_counts_per_tag{tag="not_matched",rating="thumbsDown"} 4.0 # HELP librechat_overall_rating_ratio Overall percentage of positive ratings (0-100) # TYPE librechat_overall_rating_ratio gauge @@ -337,11 +337,11 @@ librechat_rated_messages_total 18.0 # HELP librechat_model_tag_thumbs_up Number of thumbs up ratings per model and tag combination # TYPE librechat_model_tag_thumbs_up gauge librechat_model_tag_thumbs_up{model="gpt-4",tag="accurate_reliable"} 5.0 -librechat_model_tag_thumbs_up{model="claude-3",tag="creative"} 3.0 +librechat_model_tag_thumbs_up{model="claude-3",tag="creative_solution"} 3.0 # HELP librechat_model_tag_thumbs_down Number of thumbs down ratings per model and tag combination # TYPE librechat_model_tag_thumbs_down gauge -librechat_model_tag_thumbs_down{model="gpt-4",tag="unhelpful"} 1.0 +librechat_model_tag_thumbs_down{model="gpt-4",tag="not_helpful"} 1.0 # HELP librechat_tool_calls_total Total number of tool calls made # TYPE librechat_tool_calls_total gauge librechat_tool_calls_total 240.0 diff --git a/metrics.py b/metrics.py index 1eb98c5..70fc8dc 100644 --- a/metrics.py +++ b/metrics.py @@ -964,14 +964,23 @@ def _fetch_all_rating_metrics(self): cache['per_model'][model][rating] = item['count'] cache['per_model'][model]['total'] += item['count'] - # Query 3: Per-tag ratings + # Query 3: Per-tag ratings, split by rating direction. + # Each LibreChat feedback tag belongs to exactly one rating + # (thumbsUp/thumbsDown), so grouping by both keeps positive and + # negative tags from being collapsed into a single count. per_tag_pipeline = [ {"$match": {"feedback.tag": {"$exists": True, "$ne": None}}}, - {"$group": {"_id": "$feedback.tag", "count": {"$sum": 1}}} + { + "$group": { + "_id": {"tag": "$feedback.tag", "rating": "$feedback.rating"}, + "count": {"$sum": 1} + } + } ] for item in self.messages_collection.aggregate(per_tag_pipeline): - tag = item['_id'] or 'unknown' - cache['per_tag'][tag] = item['count'] + tag = item['_id']['tag'] or 'unknown' + rating = item['_id'].get('rating') or 'unknown' + cache['per_tag'][(tag, rating)] = item['count'] # Query 4: Model-tag combinations model_tag_pipeline = [ @@ -1134,13 +1143,13 @@ def collect_rating_counts_per_tag(self): metric = GaugeMetricFamily( "librechat_rating_counts_per_tag", - "Number of ratings per feedback tag", - labels=["tag"], + "Number of ratings per feedback tag and rating direction", + labels=["tag", "rating"], ) - for tag, count in data.items(): - metric.add_metric([tag], count) - logger.debug("Rating count for tag %s: %s", tag, count) + for (tag, rating), count in data.items(): + metric.add_metric([tag, rating], count) + logger.debug("Rating count for tag %s (%s): %s", tag, rating, count) yield metric except Exception as e: diff --git a/prometheus-dev/Grafana-Dashboard-template.json b/prometheus-dev/Grafana-Dashboard-template.json index c2e415c..b361e8b 100644 --- a/prometheus-dev/Grafana-Dashboard-template.json +++ b/prometheus-dev/Grafana-Dashboard-template.json @@ -434,10 +434,38 @@ { "expr": "librechat_thumbs_down_5m", "legendFormat": "down", "refId": "B", "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" } } ] }, + { + "type": "bargauge", + "title": "Positive Feedback Tags", + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 39 }, + "id": 27, + "options": { "orientation": "horizontal", "displayMode": "gradient", "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, "showUnfilled": true }, + "fieldConfig": { + "defaults": { "color": { "mode": "continuous-GrYlRd" }, "unit": "short" } + }, + "targets": [ + { "expr": "librechat_rating_counts_per_tag{rating=\"thumbsUp\"}", "legendFormat": "{{tag}}", "refId": "A", "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" } } + ] + }, + { + "type": "bargauge", + "title": "Negative Feedback Tags", + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 39 }, + "id": 28, + "options": { "orientation": "horizontal", "displayMode": "gradient", "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, "showUnfilled": true }, + "fieldConfig": { + "defaults": { "color": { "mode": "continuous-RdYlGr" }, "unit": "short" } + }, + "targets": [ + { "expr": "librechat_rating_counts_per_tag{rating=\"thumbsDown\"}", "legendFormat": "{{tag}}", "refId": "A", "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" } } + ] + }, { "type": "row", "title": "Tools & Files", - "gridPos": { "h": 1, "w": 24, "x": 0, "y": 39 }, + "gridPos": { "h": 1, "w": 24, "x": 0, "y": 47 }, "id": 103, "collapsed": false, "panels": [] @@ -446,7 +474,7 @@ "type": "stat", "title": "Tool Calls (total)", "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "gridPos": { "h": 4, "w": 6, "x": 0, "y": 40 }, + "gridPos": { "h": 4, "w": 6, "x": 0, "y": 48 }, "id": 21, "options": { "colorMode": "value", @@ -466,7 +494,7 @@ "type": "stat", "title": "Tool Errors (total)", "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "gridPos": { "h": 4, "w": 6, "x": 6, "y": 40 }, + "gridPos": { "h": 4, "w": 6, "x": 6, "y": 48 }, "id": 22, "options": { "colorMode": "value", @@ -486,7 +514,7 @@ "type": "stat", "title": "Active Tool Users", "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "gridPos": { "h": 4, "w": 6, "x": 12, "y": 40 }, + "gridPos": { "h": 4, "w": 6, "x": 12, "y": 48 }, "id": 23, "options": { "colorMode": "value", @@ -506,7 +534,7 @@ "type": "stat", "title": "Uploaded Files", "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "gridPos": { "h": 4, "w": 6, "x": 18, "y": 40 }, + "gridPos": { "h": 4, "w": 6, "x": 18, "y": 48 }, "id": 24, "options": { "colorMode": "value", @@ -526,7 +554,7 @@ "type": "bargauge", "title": "Tool Calls per Tool", "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "gridPos": { "h": 8, "w": 12, "x": 0, "y": 44 }, + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 52 }, "id": 25, "options": { "orientation": "horizontal", "displayMode": "gradient", "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, "showUnfilled": true }, "fieldConfig": { @@ -540,7 +568,7 @@ "type": "bargauge", "title": "Tool Success Rate per Tool", "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "gridPos": { "h": 8, "w": 12, "x": 12, "y": 44 }, + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 52 }, "id": 26, "options": { "orientation": "horizontal", "displayMode": "gradient", "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, "showUnfilled": true }, "fieldConfig": {