feat: add tenant to metrics labels (#151)

hindsight-api/hindsight_api/metrics.py

@@ -28,6 +28,15 @@ from opentelemetry.sdk.resources import Resource
 if TYPE_CHECKING:
     import asyncpg
 
+
+def _get_tenant() -> str:
+    """Get current tenant (schema) from context for metrics labeling."""
+    # Import here to avoid circular imports
+    from hindsight_api.engine.memory_engine import get_current_schema
+
+    return get_current_schema()
+
+
 # Custom bucket boundaries for operation duration (in seconds)
 # Fine granularity in 0-30s range where most operations complete
 DURATION_BUCKETS = (0.1, 0.25, 0.5, 0.75, 1.0, 2.0, 3.0, 5.0, 7.5, 10.0, 15.0, 20.0, 30.0, 60.0, 120.0)
@@ -323,6 +332,7 @@ class MetricsCollector(MetricsCollectorBase):
             "operation": operation,
             "bank_id": bank_id,
             "source": source,
+            "tenant": _get_tenant(),
         }
         if budget:
             attributes["budget"] = budget
@@ -373,6 +383,7 @@ class MetricsCollector(MetricsCollectorBase):
             "model": model,
             "scope": scope,
             "success": str(success).lower(),
+            "tenant": _get_tenant(),
         }
 
         # Record duration
@@ -425,10 +436,14 @@ class MetricsCollector(MetricsCollectorBase):
             status_code = status_code_getter()
             status_class = f"{status_code // 100}xx"
 
+            # Get tenant from context (may be set during request processing)
+            tenant = _get_tenant()
+
             attributes = {
                 **base_attributes,
                 "status_code": str(status_code),
                 "status_class": status_class,
+                "tenant": tenant,
             }
 
             # Record duration and count

monitoring/grafana/dashboards/hindsight-api-service.json

@@ -54,7 +54,7 @@
       "pluginVersion": "10.0.0",
       "targets": [
         {
-          "expr": "sum(hindsight_http_requests_total)",
+          "expr": "sum(hindsight_http_requests_total{tenant=~\"$tenant\"})",
           "refId": "A"
         }
       ],
@@ -99,7 +99,7 @@
       "pluginVersion": "10.0.0",
       "targets": [
         {
-          "expr": "sum(rate(hindsight_http_requests_total[1m]))",
+          "expr": "sum(rate(hindsight_http_requests_total{tenant=~\"$tenant\"}[1m]))",
           "refId": "A"
         }
       ],
@@ -191,7 +191,7 @@
       "pluginVersion": "10.0.0",
       "targets": [
         {
-          "expr": "sum(rate(hindsight_http_requests_total{status_class=\"5xx\"}[5m])) / sum(rate(hindsight_http_requests_total[5m]))",
+          "expr": "sum(rate(hindsight_http_requests_total{status_class=\"5xx\", tenant=~\"$tenant\"}[5m])) / sum(rate(hindsight_http_requests_total{tenant=~\"$tenant\"}[5m]))",
           "refId": "A"
         }
       ],
@@ -236,7 +236,7 @@
       "pluginVersion": "10.0.0",
       "targets": [
         {
-          "expr": "histogram_quantile(0.95, sum by (le) (rate(hindsight_http_duration_seconds_bucket[5m])))",
+          "expr": "histogram_quantile(0.95, sum by (le) (rate(hindsight_http_duration_seconds_bucket{tenant=~\"$tenant\"}[5m])))",
           "refId": "A"
         }
       ],
@@ -313,7 +313,7 @@
       "pluginVersion": "10.0.0",
       "targets": [
         {
-          "expr": "sum by (endpoint) (rate(hindsight_http_requests_total[1m]))",
+          "expr": "sum by (endpoint) (rate(hindsight_http_requests_total{tenant=~\"$tenant\"}[1m]))",
           "legendFormat": "{{endpoint}}",
           "refId": "A"
         }
@@ -404,17 +404,17 @@
       "pluginVersion": "10.0.0",
       "targets": [
         {
-          "expr": "histogram_quantile(0.50, sum by (le) (rate(hindsight_http_duration_seconds_bucket[5m])))",
+          "expr": "histogram_quantile(0.50, sum by (le) (rate(hindsight_http_duration_seconds_bucket{tenant=~\"$tenant\"}[5m])))",
           "legendFormat": "p50",
           "refId": "A"
         },
         {
-          "expr": "histogram_quantile(0.95, sum by (le) (rate(hindsight_http_duration_seconds_bucket[5m])))",
+          "expr": "histogram_quantile(0.95, sum by (le) (rate(hindsight_http_duration_seconds_bucket{tenant=~\"$tenant\"}[5m])))",
           "legendFormat": "p95",
           "refId": "B"
         },
         {
-          "expr": "histogram_quantile(0.99, sum by (le) (rate(hindsight_http_duration_seconds_bucket[5m])))",
+          "expr": "histogram_quantile(0.99, sum by (le) (rate(hindsight_http_duration_seconds_bucket{tenant=~\"$tenant\"}[5m])))",
           "legendFormat": "p99",
           "refId": "C"
         }
@@ -505,12 +505,12 @@
       "pluginVersion": "10.0.0",
       "targets": [
         {
-          "expr": "sum(rate(hindsight_http_requests_total{status_class=\"5xx\"}[1m])) / sum(rate(hindsight_http_requests_total[1m]))",
+          "expr": "sum(rate(hindsight_http_requests_total{status_class=\"5xx\", tenant=~\"$tenant\"}[1m])) / sum(rate(hindsight_http_requests_total{tenant=~\"$tenant\"}[1m]))",
           "legendFormat": "5xx Error Rate",
           "refId": "A"
         },
         {
-          "expr": "sum(rate(hindsight_http_requests_total{status_class=\"4xx\"}[1m])) / sum(rate(hindsight_http_requests_total[1m]))",
+          "expr": "sum(rate(hindsight_http_requests_total{status_class=\"4xx\", tenant=~\"$tenant\"}[1m])) / sum(rate(hindsight_http_requests_total{tenant=~\"$tenant\"}[1m]))",
           "legendFormat": "4xx Error Rate",
           "refId": "B"
         }
@@ -1276,7 +1276,36 @@
   "schemaVersion": 38,
   "tags": ["hindsight", "api", "service"],
   "templating": {
-    "list": []
+    "list": [
+      {
+        "allValue": ".*",
+        "current": {
+          "selected": true,
+          "text": "All",
+          "value": "$__all"
+        },
+        "datasource": {
+          "type": "prometheus",
+          "uid": "prometheus"
+        },
+        "definition": "label_values(hindsight_http_requests_total, tenant)",
+        "hide": 0,
+        "includeAll": true,
+        "label": "Tenant",
+        "multi": false,
+        "name": "tenant",
+        "options": [],
+        "query": {
+          "query": "label_values(hindsight_http_requests_total, tenant)",
+          "refId": "PrometheusVariableQueryEditor-VariableQuery"
+        },
+        "refresh": 2,
+        "regex": "",
+        "skipUrlSync": false,
+        "sort": 1,
+        "type": "query"
+      }
+    ]
   },
   "time": {
     "from": "now-30m",

monitoring/grafana/dashboards/hindsight-llm.json

@@ -46,7 +46,7 @@
       "pluginVersion": "10.0.0",
       "targets": [
         {
-          "expr": "sum(hindsight_llm_calls_total)",
+          "expr": "sum(hindsight_llm_calls_total{tenant=~\"$tenant\"})",
           "refId": "A"
         }
       ],
@@ -91,7 +91,7 @@
       "pluginVersion": "10.0.0",
       "targets": [
         {
-          "expr": "sum(hindsight_llm_tokens_input_tokens_total) + sum(hindsight_llm_tokens_output_tokens_total)",
+          "expr": "sum(hindsight_llm_tokens_input_tokens_total{tenant=~\"$tenant\"}) + sum(hindsight_llm_tokens_output_tokens_total{tenant=~\"$tenant\"})",
           "refId": "A"
         }
       ],
@@ -137,7 +137,7 @@
       "pluginVersion": "10.0.0",
       "targets": [
         {
-          "expr": "sum(hindsight_llm_tokens_input_tokens_total)",
+          "expr": "sum(hindsight_llm_tokens_input_tokens_total{tenant=~\"$tenant\"})",
           "refId": "A"
         }
       ],
@@ -183,7 +183,7 @@
       "pluginVersion": "10.0.0",
       "targets": [
         {
-          "expr": "sum(hindsight_llm_tokens_output_tokens_total)",
+          "expr": "sum(hindsight_llm_tokens_output_tokens_total{tenant=~\"$tenant\"})",
           "refId": "A"
         }
       ],
@@ -260,7 +260,7 @@
       "pluginVersion": "10.0.0",
       "targets": [
         {
-          "expr": "sum by (scope) (rate(hindsight_llm_calls_total[1m]))",
+          "expr": "sum by (scope) (rate(hindsight_llm_calls_total{tenant=~\"$tenant\"}[1m]))",
           "legendFormat": "{{scope}}",
           "refId": "A"
         }
@@ -347,12 +347,12 @@
       "pluginVersion": "10.0.0",
       "targets": [
         {
-          "expr": "sum(rate(hindsight_llm_tokens_input_tokens_total[1m]))",
+          "expr": "sum(rate(hindsight_llm_tokens_input_tokens_total{tenant=~\"$tenant\"}[1m]))",
           "legendFormat": "Input",
           "refId": "A"
         },
         {
-          "expr": "sum(rate(hindsight_llm_tokens_output_tokens_total[1m]))",
+          "expr": "sum(rate(hindsight_llm_tokens_output_tokens_total{tenant=~\"$tenant\"}[1m]))",
           "legendFormat": "Output",
           "refId": "B"
         }
@@ -430,7 +430,7 @@
       "pluginVersion": "10.0.0",
       "targets": [
         {
-          "expr": "histogram_quantile(0.95, sum by (scope, le) (rate(hindsight_llm_duration_seconds_bucket[5m])))",
+          "expr": "histogram_quantile(0.95, sum by (scope, le) (rate(hindsight_llm_duration_seconds_bucket{tenant=~\"$tenant\"}[5m])))",
           "legendFormat": "{{scope}}",
           "refId": "A"
         }
@@ -508,12 +508,12 @@
       "pluginVersion": "10.0.0",
       "targets": [
         {
-          "expr": "sum by (scope) (rate(hindsight_llm_tokens_input_tokens_total[1m]))",
+          "expr": "sum by (scope) (rate(hindsight_llm_tokens_input_tokens_total{tenant=~\"$tenant\"}[1m]))",
           "legendFormat": "{{scope}} (input)",
           "refId": "A"
         },
         {
-          "expr": "sum by (scope) (rate(hindsight_llm_tokens_output_tokens_total[1m]))",
+          "expr": "sum by (scope) (rate(hindsight_llm_tokens_output_tokens_total{tenant=~\"$tenant\"}[1m]))",
           "legendFormat": "{{scope}} (output)",
           "refId": "B"
         }
@@ -526,7 +526,36 @@
   "schemaVersion": 38,
   "tags": ["hindsight", "llm"],
   "templating": {
-    "list": []
+    "list": [
+      {
+        "allValue": ".*",
+        "current": {
+          "selected": true,
+          "text": "All",
+          "value": "$__all"
+        },
+        "datasource": {
+          "type": "prometheus",
+          "uid": "prometheus"
+        },
+        "definition": "label_values(hindsight_llm_calls_total, tenant)",
+        "hide": 0,
+        "includeAll": true,
+        "label": "Tenant",
+        "multi": false,
+        "name": "tenant",
+        "options": [],
+        "query": {
+          "query": "label_values(hindsight_llm_calls_total, tenant)",
+          "refId": "PrometheusVariableQueryEditor-VariableQuery"
+        },
+        "refresh": 2,
+        "regex": "",
+        "skipUrlSync": false,
+        "sort": 1,
+        "type": "query"
+      }
+    ]
   },
   "time": {
     "from": "now-30m",

monitoring/grafana/dashboards/hindsight-operations.json

@@ -46,7 +46,7 @@
       "pluginVersion": "10.0.0",
       "targets": [
         {
-          "expr": "sum(hindsight_operation_operations_total)",
+          "expr": "sum(hindsight_operation_operations_total{tenant=~\"$tenant\"})",
           "refId": "A"
         }
       ],
@@ -91,7 +91,7 @@
       "pluginVersion": "10.0.0",
       "targets": [
         {
-          "expr": "sum(rate(hindsight_operation_operations_total[1m]))",
+          "expr": "sum(rate(hindsight_operation_operations_total{tenant=~\"$tenant\"}[1m]))",
           "refId": "A"
         }
       ],
@@ -137,7 +137,7 @@
       "pluginVersion": "10.0.0",
       "targets": [
         {
-          "expr": "sum(rate(hindsight_operation_operations_total{operation=\"retain\"}[1m]))",
+          "expr": "sum(rate(hindsight_operation_operations_total{operation=\"retain\", tenant=~\"$tenant\"}[1m]))",
           "refId": "A"
         }
       ],
@@ -183,7 +183,7 @@
       "pluginVersion": "10.0.0",
       "targets": [
         {
-          "expr": "sum(rate(hindsight_operation_operations_total{operation=\"recall\"}[1m]))",
+          "expr": "sum(rate(hindsight_operation_operations_total{operation=\"recall\", tenant=~\"$tenant\"}[1m]))",
           "refId": "A"
         }
       ],
@@ -229,7 +229,7 @@
       "pluginVersion": "10.0.0",
       "targets": [
         {
-          "expr": "sum(rate(hindsight_operation_operations_total{operation=\"reflect\"}[1m]))",
+          "expr": "sum(rate(hindsight_operation_operations_total{operation=\"reflect\", tenant=~\"$tenant\"}[1m]))",
           "refId": "A"
         }
       ],
@@ -319,7 +319,7 @@
       "pluginVersion": "10.0.0",
       "targets": [
         {
-          "expr": "sum by (operation) (rate(hindsight_operation_operations_total[1m]))",
+          "expr": "sum by (operation) (rate(hindsight_operation_operations_total{tenant=~\"$tenant\"}[1m]))",
           "legendFormat": "{{operation}}",
           "refId": "A"
         }
@@ -410,17 +410,17 @@
       "pluginVersion": "10.0.0",
       "targets": [
         {
-          "expr": "histogram_quantile(0.50, sum by (le) (rate(hindsight_operation_duration_seconds_bucket{operation=\"recall\"}[5m])))",
+          "expr": "histogram_quantile(0.50, sum by (le) (rate(hindsight_operation_duration_seconds_bucket{operation=\"recall\", tenant=~\"$tenant\"}[5m])))",
           "legendFormat": "p50",
           "refId": "A"
         },
         {
-          "expr": "histogram_quantile(0.95, sum by (le) (rate(hindsight_operation_duration_seconds_bucket{operation=\"recall\"}[5m])))",
+          "expr": "histogram_quantile(0.95, sum by (le) (rate(hindsight_operation_duration_seconds_bucket{operation=\"recall\", tenant=~\"$tenant\"}[5m])))",
           "legendFormat": "p95",
           "refId": "B"
         },
         {
-          "expr": "histogram_quantile(0.99, sum by (le) (rate(hindsight_operation_duration_seconds_bucket{operation=\"recall\"}[5m])))",
+          "expr": "histogram_quantile(0.99, sum by (le) (rate(hindsight_operation_duration_seconds_bucket{operation=\"recall\", tenant=~\"$tenant\"}[5m])))",
           "legendFormat": "p99",
           "refId": "C"
         }
@@ -498,7 +498,7 @@
       "pluginVersion": "10.0.0",
       "targets": [
         {
-          "expr": "histogram_quantile(0.95, sum by (operation, le) (rate(hindsight_operation_duration_seconds_bucket[5m])))",
+          "expr": "histogram_quantile(0.95, sum by (operation, le) (rate(hindsight_operation_duration_seconds_bucket{tenant=~\"$tenant\"}[5m])))",
           "legendFormat": "{{operation}}",
           "refId": "A"
         }
@@ -576,7 +576,7 @@
       "pluginVersion": "10.0.0",
       "targets": [
         {
-          "expr": "sum by (bank_id) (rate(hindsight_operation_operations_total[1m]))",
+          "expr": "sum by (bank_id) (rate(hindsight_operation_operations_total{tenant=~\"$tenant\"}[1m]))",
           "legendFormat": "{{bank_id}}",
           "refId": "A"
         }
@@ -589,7 +589,36 @@
   "schemaVersion": 38,
   "tags": ["hindsight"],
   "templating": {
-    "list": []
+    "list": [
+      {
+        "allValue": ".*",
+        "current": {
+          "selected": true,
+          "text": "All",
+          "value": "$__all"
+        },
+        "datasource": {
+          "type": "prometheus",
+          "uid": "prometheus"
+        },
+        "definition": "label_values(hindsight_operation_operations_total, tenant)",
+        "hide": 0,
+        "includeAll": true,
+        "label": "Tenant",
+        "multi": false,
+        "name": "tenant",
+        "options": [],
+        "query": {
+          "query": "label_values(hindsight_operation_operations_total, tenant)",
+          "refId": "PrometheusVariableQueryEditor-VariableQuery"
+        },
+        "refresh": 2,
+        "regex": "",
+        "skipUrlSync": false,
+        "sort": 1,
+        "type": "query"
+      }
+    ]
   },
   "time": {
     "from": "now-30m",