diff --git a/hindsight-api/hindsight_api/metrics.py b/hindsight-api/hindsight_api/metrics.py index d5e829ba..f2056625 100644 --- a/hindsight-api/hindsight_api/metrics.py +++ b/hindsight-api/hindsight_api/metrics.py @@ -28,6 +28,15 @@ from opentelemetry.sdk.resources import Resource if TYPE_CHECKING: import asyncpg + +def _get_tenant() -> str: + """Get current tenant (schema) from context for metrics labeling.""" + # Import here to avoid circular imports + from hindsight_api.engine.memory_engine import get_current_schema + + return get_current_schema() + + # Custom bucket boundaries for operation duration (in seconds) # Fine granularity in 0-30s range where most operations complete DURATION_BUCKETS = (0.1, 0.25, 0.5, 0.75, 1.0, 2.0, 3.0, 5.0, 7.5, 10.0, 15.0, 20.0, 30.0, 60.0, 120.0) @@ -323,6 +332,7 @@ class MetricsCollector(MetricsCollectorBase): "operation": operation, "bank_id": bank_id, "source": source, + "tenant": _get_tenant(), } if budget: attributes["budget"] = budget @@ -373,6 +383,7 @@ class MetricsCollector(MetricsCollectorBase): "model": model, "scope": scope, "success": str(success).lower(), + "tenant": _get_tenant(), } # Record duration @@ -425,10 +436,14 @@ class MetricsCollector(MetricsCollectorBase): status_code = status_code_getter() status_class = f"{status_code // 100}xx" + # Get tenant from context (may be set during request processing) + tenant = _get_tenant() + attributes = { **base_attributes, "status_code": str(status_code), "status_class": status_class, + "tenant": tenant, } # Record duration and count diff --git a/monitoring/grafana/dashboards/hindsight-api-service.json b/monitoring/grafana/dashboards/hindsight-api-service.json index 766c0d77..fb3bc0d6 100644 --- a/monitoring/grafana/dashboards/hindsight-api-service.json +++ b/monitoring/grafana/dashboards/hindsight-api-service.json @@ -54,7 +54,7 @@ "pluginVersion": "10.0.0", "targets": [ { - "expr": "sum(hindsight_http_requests_total)", + "expr": "sum(hindsight_http_requests_total{tenant=~\"$tenant\"})", "refId": "A" } ], @@ -99,7 +99,7 @@ "pluginVersion": "10.0.0", "targets": [ { - "expr": "sum(rate(hindsight_http_requests_total[1m]))", + "expr": "sum(rate(hindsight_http_requests_total{tenant=~\"$tenant\"}[1m]))", "refId": "A" } ], @@ -191,7 +191,7 @@ "pluginVersion": "10.0.0", "targets": [ { - "expr": "sum(rate(hindsight_http_requests_total{status_class=\"5xx\"}[5m])) / sum(rate(hindsight_http_requests_total[5m]))", + "expr": "sum(rate(hindsight_http_requests_total{status_class=\"5xx\", tenant=~\"$tenant\"}[5m])) / sum(rate(hindsight_http_requests_total{tenant=~\"$tenant\"}[5m]))", "refId": "A" } ], @@ -236,7 +236,7 @@ "pluginVersion": "10.0.0", "targets": [ { - "expr": "histogram_quantile(0.95, sum by (le) (rate(hindsight_http_duration_seconds_bucket[5m])))", + "expr": "histogram_quantile(0.95, sum by (le) (rate(hindsight_http_duration_seconds_bucket{tenant=~\"$tenant\"}[5m])))", "refId": "A" } ], @@ -313,7 +313,7 @@ "pluginVersion": "10.0.0", "targets": [ { - "expr": "sum by (endpoint) (rate(hindsight_http_requests_total[1m]))", + "expr": "sum by (endpoint) (rate(hindsight_http_requests_total{tenant=~\"$tenant\"}[1m]))", "legendFormat": "{{endpoint}}", "refId": "A" } @@ -404,17 +404,17 @@ "pluginVersion": "10.0.0", "targets": [ { - "expr": "histogram_quantile(0.50, sum by (le) (rate(hindsight_http_duration_seconds_bucket[5m])))", + "expr": "histogram_quantile(0.50, sum by (le) (rate(hindsight_http_duration_seconds_bucket{tenant=~\"$tenant\"}[5m])))", "legendFormat": "p50", "refId": "A" }, { - "expr": "histogram_quantile(0.95, sum by (le) (rate(hindsight_http_duration_seconds_bucket[5m])))", + "expr": "histogram_quantile(0.95, sum by (le) (rate(hindsight_http_duration_seconds_bucket{tenant=~\"$tenant\"}[5m])))", "legendFormat": "p95", "refId": "B" }, { - "expr": "histogram_quantile(0.99, sum by (le) (rate(hindsight_http_duration_seconds_bucket[5m])))", + "expr": "histogram_quantile(0.99, sum by (le) (rate(hindsight_http_duration_seconds_bucket{tenant=~\"$tenant\"}[5m])))", "legendFormat": "p99", "refId": "C" } @@ -505,12 +505,12 @@ "pluginVersion": "10.0.0", "targets": [ { - "expr": "sum(rate(hindsight_http_requests_total{status_class=\"5xx\"}[1m])) / sum(rate(hindsight_http_requests_total[1m]))", + "expr": "sum(rate(hindsight_http_requests_total{status_class=\"5xx\", tenant=~\"$tenant\"}[1m])) / sum(rate(hindsight_http_requests_total{tenant=~\"$tenant\"}[1m]))", "legendFormat": "5xx Error Rate", "refId": "A" }, { - "expr": "sum(rate(hindsight_http_requests_total{status_class=\"4xx\"}[1m])) / sum(rate(hindsight_http_requests_total[1m]))", + "expr": "sum(rate(hindsight_http_requests_total{status_class=\"4xx\", tenant=~\"$tenant\"}[1m])) / sum(rate(hindsight_http_requests_total{tenant=~\"$tenant\"}[1m]))", "legendFormat": "4xx Error Rate", "refId": "B" } @@ -1276,7 +1276,36 @@ "schemaVersion": 38, "tags": ["hindsight", "api", "service"], "templating": { - "list": [] + "list": [ + { + "allValue": ".*", + "current": { + "selected": true, + "text": "All", + "value": "$__all" + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "definition": "label_values(hindsight_http_requests_total, tenant)", + "hide": 0, + "includeAll": true, + "label": "Tenant", + "multi": false, + "name": "tenant", + "options": [], + "query": { + "query": "label_values(hindsight_http_requests_total, tenant)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query" + } + ] }, "time": { "from": "now-30m", diff --git a/monitoring/grafana/dashboards/hindsight-llm.json b/monitoring/grafana/dashboards/hindsight-llm.json index 90a5d3a1..30f7663f 100644 --- a/monitoring/grafana/dashboards/hindsight-llm.json +++ b/monitoring/grafana/dashboards/hindsight-llm.json @@ -46,7 +46,7 @@ "pluginVersion": "10.0.0", "targets": [ { - "expr": "sum(hindsight_llm_calls_total)", + "expr": "sum(hindsight_llm_calls_total{tenant=~\"$tenant\"})", "refId": "A" } ], @@ -91,7 +91,7 @@ "pluginVersion": "10.0.0", "targets": [ { - "expr": "sum(hindsight_llm_tokens_input_tokens_total) + sum(hindsight_llm_tokens_output_tokens_total)", + "expr": "sum(hindsight_llm_tokens_input_tokens_total{tenant=~\"$tenant\"}) + sum(hindsight_llm_tokens_output_tokens_total{tenant=~\"$tenant\"})", "refId": "A" } ], @@ -137,7 +137,7 @@ "pluginVersion": "10.0.0", "targets": [ { - "expr": "sum(hindsight_llm_tokens_input_tokens_total)", + "expr": "sum(hindsight_llm_tokens_input_tokens_total{tenant=~\"$tenant\"})", "refId": "A" } ], @@ -183,7 +183,7 @@ "pluginVersion": "10.0.0", "targets": [ { - "expr": "sum(hindsight_llm_tokens_output_tokens_total)", + "expr": "sum(hindsight_llm_tokens_output_tokens_total{tenant=~\"$tenant\"})", "refId": "A" } ], @@ -260,7 +260,7 @@ "pluginVersion": "10.0.0", "targets": [ { - "expr": "sum by (scope) (rate(hindsight_llm_calls_total[1m]))", + "expr": "sum by (scope) (rate(hindsight_llm_calls_total{tenant=~\"$tenant\"}[1m]))", "legendFormat": "{{scope}}", "refId": "A" } @@ -347,12 +347,12 @@ "pluginVersion": "10.0.0", "targets": [ { - "expr": "sum(rate(hindsight_llm_tokens_input_tokens_total[1m]))", + "expr": "sum(rate(hindsight_llm_tokens_input_tokens_total{tenant=~\"$tenant\"}[1m]))", "legendFormat": "Input", "refId": "A" }, { - "expr": "sum(rate(hindsight_llm_tokens_output_tokens_total[1m]))", + "expr": "sum(rate(hindsight_llm_tokens_output_tokens_total{tenant=~\"$tenant\"}[1m]))", "legendFormat": "Output", "refId": "B" } @@ -430,7 +430,7 @@ "pluginVersion": "10.0.0", "targets": [ { - "expr": "histogram_quantile(0.95, sum by (scope, le) (rate(hindsight_llm_duration_seconds_bucket[5m])))", + "expr": "histogram_quantile(0.95, sum by (scope, le) (rate(hindsight_llm_duration_seconds_bucket{tenant=~\"$tenant\"}[5m])))", "legendFormat": "{{scope}}", "refId": "A" } @@ -508,12 +508,12 @@ "pluginVersion": "10.0.0", "targets": [ { - "expr": "sum by (scope) (rate(hindsight_llm_tokens_input_tokens_total[1m]))", + "expr": "sum by (scope) (rate(hindsight_llm_tokens_input_tokens_total{tenant=~\"$tenant\"}[1m]))", "legendFormat": "{{scope}} (input)", "refId": "A" }, { - "expr": "sum by (scope) (rate(hindsight_llm_tokens_output_tokens_total[1m]))", + "expr": "sum by (scope) (rate(hindsight_llm_tokens_output_tokens_total{tenant=~\"$tenant\"}[1m]))", "legendFormat": "{{scope}} (output)", "refId": "B" } @@ -526,7 +526,36 @@ "schemaVersion": 38, "tags": ["hindsight", "llm"], "templating": { - "list": [] + "list": [ + { + "allValue": ".*", + "current": { + "selected": true, + "text": "All", + "value": "$__all" + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "definition": "label_values(hindsight_llm_calls_total, tenant)", + "hide": 0, + "includeAll": true, + "label": "Tenant", + "multi": false, + "name": "tenant", + "options": [], + "query": { + "query": "label_values(hindsight_llm_calls_total, tenant)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query" + } + ] }, "time": { "from": "now-30m", diff --git a/monitoring/grafana/dashboards/hindsight-operations.json b/monitoring/grafana/dashboards/hindsight-operations.json index dfc717bc..cf415d66 100644 --- a/monitoring/grafana/dashboards/hindsight-operations.json +++ b/monitoring/grafana/dashboards/hindsight-operations.json @@ -46,7 +46,7 @@ "pluginVersion": "10.0.0", "targets": [ { - "expr": "sum(hindsight_operation_operations_total)", + "expr": "sum(hindsight_operation_operations_total{tenant=~\"$tenant\"})", "refId": "A" } ], @@ -91,7 +91,7 @@ "pluginVersion": "10.0.0", "targets": [ { - "expr": "sum(rate(hindsight_operation_operations_total[1m]))", + "expr": "sum(rate(hindsight_operation_operations_total{tenant=~\"$tenant\"}[1m]))", "refId": "A" } ], @@ -137,7 +137,7 @@ "pluginVersion": "10.0.0", "targets": [ { - "expr": "sum(rate(hindsight_operation_operations_total{operation=\"retain\"}[1m]))", + "expr": "sum(rate(hindsight_operation_operations_total{operation=\"retain\", tenant=~\"$tenant\"}[1m]))", "refId": "A" } ], @@ -183,7 +183,7 @@ "pluginVersion": "10.0.0", "targets": [ { - "expr": "sum(rate(hindsight_operation_operations_total{operation=\"recall\"}[1m]))", + "expr": "sum(rate(hindsight_operation_operations_total{operation=\"recall\", tenant=~\"$tenant\"}[1m]))", "refId": "A" } ], @@ -229,7 +229,7 @@ "pluginVersion": "10.0.0", "targets": [ { - "expr": "sum(rate(hindsight_operation_operations_total{operation=\"reflect\"}[1m]))", + "expr": "sum(rate(hindsight_operation_operations_total{operation=\"reflect\", tenant=~\"$tenant\"}[1m]))", "refId": "A" } ], @@ -319,7 +319,7 @@ "pluginVersion": "10.0.0", "targets": [ { - "expr": "sum by (operation) (rate(hindsight_operation_operations_total[1m]))", + "expr": "sum by (operation) (rate(hindsight_operation_operations_total{tenant=~\"$tenant\"}[1m]))", "legendFormat": "{{operation}}", "refId": "A" } @@ -410,17 +410,17 @@ "pluginVersion": "10.0.0", "targets": [ { - "expr": "histogram_quantile(0.50, sum by (le) (rate(hindsight_operation_duration_seconds_bucket{operation=\"recall\"}[5m])))", + "expr": "histogram_quantile(0.50, sum by (le) (rate(hindsight_operation_duration_seconds_bucket{operation=\"recall\", tenant=~\"$tenant\"}[5m])))", "legendFormat": "p50", "refId": "A" }, { - "expr": "histogram_quantile(0.95, sum by (le) (rate(hindsight_operation_duration_seconds_bucket{operation=\"recall\"}[5m])))", + "expr": "histogram_quantile(0.95, sum by (le) (rate(hindsight_operation_duration_seconds_bucket{operation=\"recall\", tenant=~\"$tenant\"}[5m])))", "legendFormat": "p95", "refId": "B" }, { - "expr": "histogram_quantile(0.99, sum by (le) (rate(hindsight_operation_duration_seconds_bucket{operation=\"recall\"}[5m])))", + "expr": "histogram_quantile(0.99, sum by (le) (rate(hindsight_operation_duration_seconds_bucket{operation=\"recall\", tenant=~\"$tenant\"}[5m])))", "legendFormat": "p99", "refId": "C" } @@ -498,7 +498,7 @@ "pluginVersion": "10.0.0", "targets": [ { - "expr": "histogram_quantile(0.95, sum by (operation, le) (rate(hindsight_operation_duration_seconds_bucket[5m])))", + "expr": "histogram_quantile(0.95, sum by (operation, le) (rate(hindsight_operation_duration_seconds_bucket{tenant=~\"$tenant\"}[5m])))", "legendFormat": "{{operation}}", "refId": "A" } @@ -576,7 +576,7 @@ "pluginVersion": "10.0.0", "targets": [ { - "expr": "sum by (bank_id) (rate(hindsight_operation_operations_total[1m]))", + "expr": "sum by (bank_id) (rate(hindsight_operation_operations_total{tenant=~\"$tenant\"}[1m]))", "legendFormat": "{{bank_id}}", "refId": "A" } @@ -589,7 +589,36 @@ "schemaVersion": 38, "tags": ["hindsight"], "templating": { - "list": [] + "list": [ + { + "allValue": ".*", + "current": { + "selected": true, + "text": "All", + "value": "$__all" + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "definition": "label_values(hindsight_operation_operations_total, tenant)", + "hide": 0, + "includeAll": true, + "label": "Tenant", + "multi": false, + "name": "tenant", + "options": [], + "query": { + "query": "label_values(hindsight_operation_operations_total, tenant)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query" + } + ] }, "time": { "from": "now-30m",