diff --git a/.gitignore b/.gitignore index 7dcfa594..adc3105a 100644 --- a/.gitignore +++ b/.gitignore @@ -27,6 +27,9 @@ docker-compose.override.yml # NLTK data (will be downloaded automatically) nltk_data/ +# Monitoring stack (Prometheus/Grafana binaries and data) +.monitoring/ + # Large benchmark datasets (will be downloaded automatically) **/longmemeval_s_cleaned.json diff --git a/hindsight-api/hindsight_api/config.py b/hindsight-api/hindsight_api/config.py index f1c8ed36..e50bdef9 100644 --- a/hindsight-api/hindsight_api/config.py +++ b/hindsight-api/hindsight_api/config.py @@ -49,6 +49,8 @@ ENV_RERANKER_COHERE_MODEL = "HINDSIGHT_API_RERANKER_COHERE_MODEL" ENV_RERANKER_PROVIDER = "HINDSIGHT_API_RERANKER_PROVIDER" ENV_RERANKER_LOCAL_MODEL = "HINDSIGHT_API_RERANKER_LOCAL_MODEL" ENV_RERANKER_TEI_URL = "HINDSIGHT_API_RERANKER_TEI_URL" +ENV_RERANKER_TEI_BATCH_SIZE = "HINDSIGHT_API_RERANKER_TEI_BATCH_SIZE" +ENV_RERANKER_TEI_MAX_CONCURRENT = "HINDSIGHT_API_RERANKER_TEI_MAX_CONCURRENT" ENV_HOST = "HINDSIGHT_API_HOST" ENV_PORT = "HINDSIGHT_API_PORT" @@ -99,6 +101,8 @@ DEFAULT_EMBEDDING_DIMENSION = 384 DEFAULT_RERANKER_PROVIDER = "local" DEFAULT_RERANKER_LOCAL_MODEL = "cross-encoder/ms-marco-MiniLM-L-6-v2" +DEFAULT_RERANKER_TEI_BATCH_SIZE = 128 +DEFAULT_RERANKER_TEI_MAX_CONCURRENT = 8 DEFAULT_EMBEDDINGS_COHERE_MODEL = "embed-english-v3.0" DEFAULT_RERANKER_COHERE_MODEL = "rerank-english-v3.0" @@ -205,6 +209,8 @@ class HindsightConfig: reranker_provider: str reranker_local_model: str reranker_tei_url: str | None + reranker_tei_batch_size: int + reranker_tei_max_concurrent: int # Server host: str @@ -272,6 +278,10 @@ class HindsightConfig: reranker_provider=os.getenv(ENV_RERANKER_PROVIDER, DEFAULT_RERANKER_PROVIDER), reranker_local_model=os.getenv(ENV_RERANKER_LOCAL_MODEL, DEFAULT_RERANKER_LOCAL_MODEL), reranker_tei_url=os.getenv(ENV_RERANKER_TEI_URL), + reranker_tei_batch_size=int(os.getenv(ENV_RERANKER_TEI_BATCH_SIZE, str(DEFAULT_RERANKER_TEI_BATCH_SIZE))), + reranker_tei_max_concurrent=int( + os.getenv(ENV_RERANKER_TEI_MAX_CONCURRENT, str(DEFAULT_RERANKER_TEI_MAX_CONCURRENT)) + ), # Server host=os.getenv(ENV_HOST, DEFAULT_HOST), port=int(os.getenv(ENV_PORT, DEFAULT_PORT)), diff --git a/hindsight-api/hindsight_api/engine/cross_encoder.py b/hindsight-api/hindsight_api/engine/cross_encoder.py index 6248d886..8fa370c8 100644 --- a/hindsight-api/hindsight_api/engine/cross_encoder.py +++ b/hindsight-api/hindsight_api/engine/cross_encoder.py @@ -6,6 +6,7 @@ Provides an interface for reranking with different backends. Configuration via environment variables - see hindsight_api.config for all env var names. """ +import asyncio import logging import os from abc import ABC, abstractmethod @@ -16,10 +17,14 @@ from ..config import ( DEFAULT_RERANKER_COHERE_MODEL, DEFAULT_RERANKER_LOCAL_MODEL, DEFAULT_RERANKER_PROVIDER, + DEFAULT_RERANKER_TEI_BATCH_SIZE, + DEFAULT_RERANKER_TEI_MAX_CONCURRENT, ENV_COHERE_API_KEY, ENV_RERANKER_COHERE_MODEL, ENV_RERANKER_LOCAL_MODEL, ENV_RERANKER_PROVIDER, + ENV_RERANKER_TEI_BATCH_SIZE, + ENV_RERANKER_TEI_MAX_CONCURRENT, ENV_RERANKER_TEI_URL, ) @@ -50,7 +55,7 @@ class CrossEncoderModel(ABC): pass @abstractmethod - def predict(self, pairs: list[tuple[str, str]]) -> list[float]: + async def predict(self, pairs: list[tuple[str, str]]) -> list[float]: """ Score query-document pairs for relevance. @@ -107,7 +112,7 @@ class LocalSTCrossEncoder(CrossEncoderModel): self._model = CrossEncoder(self.model_name) logger.info("Reranker: local provider initialized") - def predict(self, pairs: list[tuple[str, str]]) -> list[float]: + async def predict(self, pairs: list[tuple[str, str]]) -> list[float]: """ Score query-document pairs for relevance. @@ -119,7 +124,10 @@ class LocalSTCrossEncoder(CrossEncoderModel): """ if self._model is None: raise RuntimeError("Reranker not initialized. Call initialize() first.") - scores = self._model.predict(pairs, show_progress_bar=False) + + # Run CPU-bound inference in thread pool to avoid blocking event loop + loop = asyncio.get_event_loop() + scores = await loop.run_in_executor(None, lambda: self._model.predict(pairs, show_progress_bar=False)) return scores.tolist() if hasattr(scores, "tolist") else list(scores) @@ -131,13 +139,16 @@ class RemoteTEICrossEncoder(CrossEncoderModel): See: https://github.com/huggingface/text-embeddings-inference Note: The TEI server must be running a cross-encoder/reranker model. + + Requests are made in parallel with configurable batch size and max concurrency (backpressure). """ def __init__( self, base_url: str, timeout: float = 30.0, - batch_size: int = 32, + batch_size: int = DEFAULT_RERANKER_TEI_BATCH_SIZE, + max_concurrent: int = DEFAULT_RERANKER_TEI_MAX_CONCURRENT, max_retries: int = 3, retry_delay: float = 0.5, ): @@ -147,138 +158,178 @@ class RemoteTEICrossEncoder(CrossEncoderModel): Args: base_url: Base URL of the TEI server (e.g., "http://localhost:8080") timeout: Request timeout in seconds (default: 30.0) - batch_size: Maximum batch size for rerank requests (default: 32) + batch_size: Maximum batch size for rerank requests (default: 128) + max_concurrent: Maximum concurrent requests for backpressure (default: 8) max_retries: Maximum number of retries for failed requests (default: 3) retry_delay: Initial delay between retries in seconds, doubles each retry (default: 0.5) """ self.base_url = base_url.rstrip("/") self.timeout = timeout self.batch_size = batch_size + self.max_concurrent = max_concurrent self.max_retries = max_retries self.retry_delay = retry_delay - self._client: httpx.Client | None = None + self._async_client: httpx.AsyncClient | None = None self._model_id: str | None = None @property def provider_name(self) -> str: return "tei" - def _request_with_retry(self, method: str, url: str, **kwargs) -> httpx.Response: - """Make an HTTP request with automatic retries on transient errors.""" - import time - + async def _async_request_with_retry( + self, + client: httpx.AsyncClient, + semaphore: asyncio.Semaphore, + method: str, + url: str, + **kwargs, + ) -> httpx.Response: + """Make an async HTTP request with automatic retries on transient errors and semaphore for backpressure.""" last_error = None delay = self.retry_delay - for attempt in range(self.max_retries + 1): - try: - if method == "GET": - response = self._client.get(url, **kwargs) - else: - response = self._client.post(url, **kwargs) - response.raise_for_status() - return response - except (httpx.ConnectError, httpx.ReadTimeout, httpx.WriteTimeout) as e: - last_error = e - if attempt < self.max_retries: - logger.warning( - f"TEI request failed (attempt {attempt + 1}/{self.max_retries + 1}): {e}. Retrying in {delay}s..." - ) - time.sleep(delay) - delay *= 2 # Exponential backoff - except httpx.HTTPStatusError as e: - # Retry on 5xx server errors - if e.response.status_code >= 500 and attempt < self.max_retries: + async with semaphore: + for attempt in range(self.max_retries + 1): + try: + if method == "GET": + response = await client.get(url, **kwargs) + else: + response = await client.post(url, **kwargs) + response.raise_for_status() + return response + except (httpx.ConnectError, httpx.ReadTimeout, httpx.WriteTimeout) as e: last_error = e - logger.warning( - f"TEI server error (attempt {attempt + 1}/{self.max_retries + 1}): {e}. Retrying in {delay}s..." - ) - time.sleep(delay) - delay *= 2 - else: - raise + if attempt < self.max_retries: + logger.warning( + f"TEI request failed (attempt {attempt + 1}/{self.max_retries + 1}): {e}. " + f"Retrying in {delay}s..." + ) + await asyncio.sleep(delay) + delay *= 2 # Exponential backoff + except httpx.HTTPStatusError as e: + # Retry on 5xx server errors + if e.response.status_code >= 500 and attempt < self.max_retries: + last_error = e + logger.warning( + f"TEI server error (attempt {attempt + 1}/{self.max_retries + 1}): {e}. " + f"Retrying in {delay}s..." + ) + await asyncio.sleep(delay) + delay *= 2 + else: + raise raise last_error async def initialize(self) -> None: """Initialize the HTTP client and verify server connectivity.""" - if self._client is not None: + if self._async_client is not None: return - logger.info(f"Reranker: initializing TEI provider at {self.base_url}") - self._client = httpx.Client(timeout=self.timeout) + logger.info( + f"Reranker: initializing TEI provider at {self.base_url} " + f"(batch_size={self.batch_size}, max_concurrent={self.max_concurrent})" + ) + self._async_client = httpx.AsyncClient(timeout=self.timeout) # Verify server is reachable and get model info + # Use a temporary semaphore for initialization + init_semaphore = asyncio.Semaphore(1) try: - response = self._request_with_retry("GET", f"{self.base_url}/info") + response = await self._async_request_with_retry( + self._async_client, init_semaphore, "GET", f"{self.base_url}/info" + ) info = response.json() self._model_id = info.get("model_id", "unknown") logger.info(f"Reranker: TEI provider initialized (model: {self._model_id})") except httpx.HTTPError as e: + self._async_client = None raise RuntimeError(f"Failed to connect to TEI server at {self.base_url}: {e}") - def predict(self, pairs: list[tuple[str, str]]) -> list[float]: + async def _rerank_query_group( + self, + client: httpx.AsyncClient, + semaphore: asyncio.Semaphore, + query: str, + texts: list[str], + ) -> list[tuple[int, float]]: + """Rerank a single query group and return list of (original_index, score) tuples.""" + try: + response = await self._async_request_with_retry( + client, + semaphore, + "POST", + f"{self.base_url}/rerank", + json={ + "query": query, + "texts": texts, + "return_text": False, + }, + ) + results = response.json() + # TEI returns results sorted by score descending, with original index + return [(result["index"], result["score"]) for result in results] + except httpx.HTTPError as e: + raise RuntimeError(f"TEI rerank request failed: {e}") + + async def _predict_async(self, pairs: list[tuple[str, str]]) -> list[float]: + """Async implementation of predict that runs requests in parallel with backpressure.""" + if not pairs: + return [] + + # Group all pairs by query + query_groups: dict[str, list[tuple[int, str]]] = {} + for idx, (query, text) in enumerate(pairs): + if query not in query_groups: + query_groups[query] = [] + query_groups[query].append((idx, text)) + + # Split each query group into batches + tasks_info: list[tuple[str, list[int], list[str]]] = [] # (query, indices, texts) + for query, indexed_texts in query_groups.items(): + indices = [idx for idx, _ in indexed_texts] + texts = [text for _, text in indexed_texts] + + # Split into batches + for i in range(0, len(texts), self.batch_size): + batch_indices = indices[i : i + self.batch_size] + batch_texts = texts[i : i + self.batch_size] + tasks_info.append((query, batch_indices, batch_texts)) + + # Run all requests in parallel with semaphore for backpressure + all_scores = [0.0] * len(pairs) + semaphore = asyncio.Semaphore(self.max_concurrent) + + tasks = [ + self._rerank_query_group(self._async_client, semaphore, query, texts) + for query, _, texts in tasks_info + ] + results = await asyncio.gather(*tasks) + + # Map scores back to original positions + for (_, indices, _), result_scores in zip(tasks_info, results): + for original_idx_in_batch, score in result_scores: + global_idx = indices[original_idx_in_batch] + all_scores[global_idx] = score + + return all_scores + + async def predict(self, pairs: list[tuple[str, str]]) -> list[float]: """ Score query-document pairs using the remote TEI reranker. + Requests are made in parallel with configurable backpressure. + Args: pairs: List of (query, document) tuples to score Returns: List of relevance scores """ - if self._client is None: + if self._async_client is None: raise RuntimeError("Reranker not initialized. Call initialize() first.") - if not pairs: - return [] - - all_scores = [] - - # Process in batches - for i in range(0, len(pairs), self.batch_size): - batch = pairs[i : i + self.batch_size] - - # TEI rerank endpoint expects query and texts separately - # All pairs in a batch should have the same query for optimal performance - # but we handle mixed queries by making separate requests per unique query - query_groups: dict[str, list[tuple[int, str]]] = {} - for idx, (query, text) in enumerate(batch): - if query not in query_groups: - query_groups[query] = [] - query_groups[query].append((idx, text)) - - batch_scores = [0.0] * len(batch) - - for query, indexed_texts in query_groups.items(): - texts = [text for _, text in indexed_texts] - indices = [idx for idx, _ in indexed_texts] - - try: - response = self._request_with_retry( - "POST", - f"{self.base_url}/rerank", - json={ - "query": query, - "texts": texts, - "return_text": False, - }, - ) - results = response.json() - - # TEI returns results sorted by score descending, with original index - for result in results: - original_idx = result["index"] - score = result["score"] - # Map back to batch position - batch_scores[indices[original_idx]] = score - - except httpx.HTTPError as e: - raise RuntimeError(f"TEI rerank request failed: {e}") - - all_scores.extend(batch_scores) - - return all_scores + return await self._predict_async(pairs) class CohereCrossEncoder(CrossEncoderModel): @@ -325,7 +376,7 @@ class CohereCrossEncoder(CrossEncoderModel): self._client = cohere.Client(api_key=self.api_key, timeout=self.timeout) logger.info("Reranker: Cohere provider initialized") - def predict(self, pairs: list[tuple[str, str]]) -> list[float]: + async def predict(self, pairs: list[tuple[str, str]]) -> list[float]: """ Score query-document pairs using the Cohere Rerank API. @@ -341,6 +392,12 @@ class CohereCrossEncoder(CrossEncoderModel): if not pairs: return [] + # Run sync Cohere API calls in thread pool + loop = asyncio.get_event_loop() + return await loop.run_in_executor(None, self._predict_sync, pairs) + + def _predict_sync(self, pairs: list[tuple[str, str]]) -> list[float]: + """Synchronous predict implementation for Cohere API.""" # Group pairs by query for efficient batching # Cohere rerank expects one query with multiple documents query_groups: dict[str, list[tuple[int, str]]] = {} @@ -386,7 +443,9 @@ def create_cross_encoder_from_env() -> CrossEncoderModel: url = os.environ.get(ENV_RERANKER_TEI_URL) if not url: raise ValueError(f"{ENV_RERANKER_TEI_URL} is required when {ENV_RERANKER_PROVIDER} is 'tei'") - return RemoteTEICrossEncoder(base_url=url) + batch_size = int(os.environ.get(ENV_RERANKER_TEI_BATCH_SIZE, str(DEFAULT_RERANKER_TEI_BATCH_SIZE))) + max_concurrent = int(os.environ.get(ENV_RERANKER_TEI_MAX_CONCURRENT, str(DEFAULT_RERANKER_TEI_MAX_CONCURRENT))) + return RemoteTEICrossEncoder(base_url=url, batch_size=batch_size, max_concurrent=max_concurrent) elif provider == "local": model = os.environ.get(ENV_RERANKER_LOCAL_MODEL) model_name = model or DEFAULT_RERANKER_LOCAL_MODEL diff --git a/hindsight-api/hindsight_api/engine/memory_engine.py b/hindsight-api/hindsight_api/engine/memory_engine.py index 2d11fb6e..d730d24c 100644 --- a/hindsight-api/hindsight_api/engine/memory_engine.py +++ b/hindsight-api/hindsight_api/engine/memory_engine.py @@ -1778,7 +1778,7 @@ class MemoryEngine(MemoryEngineInterface): await reranker_instance.ensure_initialized() # Rerank using cross-encoder - scored_results = reranker_instance.rerank(query, merged_candidates) + scored_results = await reranker_instance.rerank(query, merged_candidates) step_duration = time.time() - step_start log_buffer.append(f" [4] Reranking: {len(scored_results)} candidates scored in {step_duration:.3f}s") diff --git a/hindsight-api/hindsight_api/engine/search/mpfp_retrieval.py b/hindsight-api/hindsight_api/engine/search/mpfp_retrieval.py index cb717f2e..f628e9dd 100644 --- a/hindsight-api/hindsight_api/engine/search/mpfp_retrieval.py +++ b/hindsight-api/hindsight_api/engine/search/mpfp_retrieval.py @@ -288,7 +288,6 @@ class MPFPGraphRetriever(GraphRetriever): config: Algorithm configuration (uses defaults if None) """ self.config = config or MPFPConfig() - self._adjacency_cache: dict[str, TypedAdjacency] = {} @property def name(self) -> str: diff --git a/hindsight-api/hindsight_api/engine/search/reranking.py b/hindsight-api/hindsight_api/engine/search/reranking.py index 073052f4..eab403b2 100644 --- a/hindsight-api/hindsight_api/engine/search/reranking.py +++ b/hindsight-api/hindsight_api/engine/search/reranking.py @@ -44,7 +44,7 @@ class CrossEncoderReranker: await cross_encoder.initialize() self._initialized = True - def rerank(self, query: str, candidates: list[MergedCandidate]) -> list[ScoredResult]: + async def rerank(self, query: str, candidates: list[MergedCandidate]) -> list[ScoredResult]: """ Rerank candidates using cross-encoder scores. @@ -85,7 +85,7 @@ class CrossEncoderReranker: pairs.append([query, doc_text]) # Get cross-encoder scores - scores = self.cross_encoder.predict(pairs) + scores = await self.cross_encoder.predict(pairs) # Normalize scores using sigmoid to [0, 1] range # Cross-encoder returns logits which can be negative diff --git a/hindsight-api/hindsight_api/main.py b/hindsight-api/hindsight_api/main.py index e0e57d1e..17e64c74 100644 --- a/hindsight-api/hindsight_api/main.py +++ b/hindsight-api/hindsight_api/main.py @@ -185,6 +185,8 @@ def main(): reranker_provider=config.reranker_provider, reranker_local_model=config.reranker_local_model, reranker_tei_url=config.reranker_tei_url, + reranker_tei_batch_size=config.reranker_tei_batch_size, + reranker_tei_max_concurrent=config.reranker_tei_max_concurrent, host=args.host, port=args.port, log_level=args.log_level, diff --git a/hindsight-api/tests/test_custom_embedding_dimension.py b/hindsight-api/tests/test_custom_embedding_dimension.py index ba1c034e..fd572ac9 100644 --- a/hindsight-api/tests/test_custom_embedding_dimension.py +++ b/hindsight-api/tests/test_custom_embedding_dimension.py @@ -514,14 +514,15 @@ class TestCohereCrossEncoder: """Test that Cohere cross-encoder initializes correctly.""" assert cohere_cross_encoder.provider_name == "cohere" - def test_cohere_cross_encoder_predict(self, cohere_cross_encoder): + @pytest.mark.asyncio + async def test_cohere_cross_encoder_predict(self, cohere_cross_encoder): """Test that Cohere cross-encoder can score pairs.""" pairs = [ ("What is the capital of France?", "Paris is the capital of France."), ("What is the capital of France?", "The Eiffel Tower is in Paris."), ("What is the capital of France?", "Python is a programming language."), ] - scores = cohere_cross_encoder.predict(pairs) + scores = await cohere_cross_encoder.predict(pairs) assert len(scores) == 3 assert all(isinstance(s, float) for s in scores) diff --git a/hindsight-api/tests/test_tei_cross_encoder.py b/hindsight-api/tests/test_tei_cross_encoder.py new file mode 100644 index 00000000..192daf4a --- /dev/null +++ b/hindsight-api/tests/test_tei_cross_encoder.py @@ -0,0 +1,546 @@ +""" +Tests for RemoteTEICrossEncoder (TEI reranker client). + +Tests cover: +- Initialization and server connectivity +- Basic predict functionality +- Batch splitting +- Parallel request handling +- Backpressure/semaphore behavior +- Retry logic on transient errors +- Multiple queries handling +""" + +import asyncio +import time +from unittest.mock import MagicMock, patch + +import httpx +import pytest + +from hindsight_api.engine.cross_encoder import RemoteTEICrossEncoder + + +class TestRemoteTEICrossEncoderInitialization: + """Tests for TEI cross-encoder initialization.""" + + @pytest.mark.asyncio + async def test_initialize_success(self): + """Test successful initialization with valid TEI server.""" + + async def mock_handler(request: httpx.Request) -> httpx.Response: + if request.url.path == "/info": + return httpx.Response( + 200, + json={"model_id": "BAAI/bge-reranker-base", "version": "1.0"}, + ) + return httpx.Response(404) + + transport = httpx.MockTransport(mock_handler) + + with patch.object(httpx, "AsyncClient", return_value=httpx.AsyncClient(transport=transport)): + encoder = RemoteTEICrossEncoder(base_url="http://localhost:8080") + await encoder.initialize() + + assert encoder._model_id == "BAAI/bge-reranker-base" + assert encoder._async_client is not None + + @pytest.mark.asyncio + async def test_initialize_server_unreachable(self): + """Test initialization fails when server is unreachable.""" + + async def mock_handler(request: httpx.Request) -> httpx.Response: + raise httpx.ConnectError("Connection refused") + + transport = httpx.MockTransport(mock_handler) + + with patch.object(httpx, "AsyncClient", return_value=httpx.AsyncClient(transport=transport)): + encoder = RemoteTEICrossEncoder( + base_url="http://localhost:8080", + max_retries=1, + retry_delay=0.01, + ) + + with pytest.raises(RuntimeError, match="Failed to connect to TEI server"): + await encoder.initialize() + + @pytest.mark.asyncio + async def test_initialize_idempotent(self): + """Test that initialize() is idempotent.""" + call_count = 0 + + async def mock_handler(request: httpx.Request) -> httpx.Response: + nonlocal call_count + if request.url.path == "/info": + call_count += 1 + return httpx.Response(200, json={"model_id": "test-model"}) + return httpx.Response(404) + + transport = httpx.MockTransport(mock_handler) + + with patch.object(httpx, "AsyncClient", return_value=httpx.AsyncClient(transport=transport)): + encoder = RemoteTEICrossEncoder(base_url="http://localhost:8080") + await encoder.initialize() + await encoder.initialize() + await encoder.initialize() + + assert call_count == 1 + + +def create_mock_async_client(handler): + """Create a mock AsyncClient that uses the given handler for requests.""" + + class MockAsyncClient: + def __init__(self, **kwargs): + self.timeout = kwargs.get("timeout", 30.0) + + async def __aenter__(self): + return self + + async def __aexit__(self, *args): + pass + + async def post(self, url, **kwargs): + return await handler("POST", url, **kwargs) + + async def get(self, url, **kwargs): + return await handler("GET", url, **kwargs) + + return MockAsyncClient() + + +class TestRemoteTEICrossEncoderPredict: + """Tests for TEI cross-encoder predict functionality.""" + + @pytest.mark.asyncio + async def test_predict_not_initialized(self): + """Test predict raises error when not initialized.""" + encoder = RemoteTEICrossEncoder(base_url="http://localhost:8080") + + with pytest.raises(RuntimeError, match="Reranker not initialized"): + await encoder.predict([("query", "doc")]) + + @pytest.mark.asyncio + async def test_predict_empty_pairs(self): + """Test predict returns empty list for empty input.""" + encoder = RemoteTEICrossEncoder(base_url="http://localhost:8080") + encoder._async_client = httpx.AsyncClient() + encoder._model_id = "test-model" + + result = await encoder.predict([]) + assert result == [] + + @pytest.mark.asyncio + async def test_predict_single_query(self): + """Test predict with single query and multiple documents.""" + rerank_calls = [] + + async def mock_handler(method, url, **kwargs): + if "/rerank" in url: + body = kwargs.get("json", {}) + rerank_calls.append(body) + texts = body["texts"] + # Return scores in descending order with original indices + results = [{"index": i, "score": 1.0 - (i * 0.1)} for i in range(len(texts))] + response = MagicMock() + response.status_code = 200 + response.raise_for_status = MagicMock() + response.json = MagicMock(return_value=results) + return response + raise httpx.HTTPStatusError("Not found", request=MagicMock(), response=MagicMock()) + + encoder = RemoteTEICrossEncoder(base_url="http://localhost:8080") + encoder._async_client = create_mock_async_client(mock_handler) + encoder._model_id = "test-model" + + pairs = [ + ("What is Python?", "Python is a programming language."), + ("What is Python?", "Python is a snake."), + ("What is Python?", "Java is also a language."), + ] + + scores = await encoder.predict(pairs) + + assert len(scores) == 3 + assert len(rerank_calls) == 1 + assert rerank_calls[0]["query"] == "What is Python?" + assert len(rerank_calls[0]["texts"]) == 3 + # Scores should be mapped back correctly + assert scores[0] == 1.0 + assert scores[1] == 0.9 + assert scores[2] == pytest.approx(0.8, rel=0.01) + + @pytest.mark.asyncio + async def test_predict_multiple_queries(self): + """Test predict with multiple different queries.""" + rerank_calls = [] + + async def mock_handler(method, url, **kwargs): + if "/rerank" in url: + body = kwargs.get("json", {}) + rerank_calls.append(body) + texts = body["texts"] + results = [{"index": i, "score": 0.5 + (i * 0.1)} for i in range(len(texts))] + response = MagicMock() + response.status_code = 200 + response.raise_for_status = MagicMock() + response.json = MagicMock(return_value=results) + return response + raise httpx.HTTPStatusError("Not found", request=MagicMock(), response=MagicMock()) + + encoder = RemoteTEICrossEncoder(base_url="http://localhost:8080") + encoder._async_client = create_mock_async_client(mock_handler) + encoder._model_id = "test-model" + + pairs = [ + ("Query A", "Doc A1"), + ("Query B", "Doc B1"), + ("Query A", "Doc A2"), + ("Query B", "Doc B2"), + ] + + scores = await encoder.predict(pairs) + + assert len(scores) == 4 + # Two queries = two rerank calls (run in parallel) + assert len(rerank_calls) == 2 + + +class TestRemoteTEICrossEncoderBatching: + """Tests for batch splitting behavior.""" + + @pytest.mark.asyncio + async def test_batch_splitting(self): + """Test that large inputs are split into batches.""" + rerank_calls = [] + + async def mock_handler(method, url, **kwargs): + if "/rerank" in url: + body = kwargs.get("json", {}) + rerank_calls.append(body) + texts = body["texts"] + results = [{"index": i, "score": 0.5} for i in range(len(texts))] + response = MagicMock() + response.status_code = 200 + response.raise_for_status = MagicMock() + response.json = MagicMock(return_value=results) + return response + raise httpx.HTTPStatusError("Not found", request=MagicMock(), response=MagicMock()) + + encoder = RemoteTEICrossEncoder( + base_url="http://localhost:8080", + batch_size=3, # Small batch for testing + ) + encoder._async_client = create_mock_async_client(mock_handler) + encoder._model_id = "test-model" + + # 7 documents with same query, batch_size=3 -> 3 batches (3+3+1) + pairs = [("Query", f"Doc {i}") for i in range(7)] + + scores = await encoder.predict(pairs) + + assert len(scores) == 7 + assert len(rerank_calls) == 3 + # Check batch sizes + batch_sizes = sorted([len(call["texts"]) for call in rerank_calls]) + assert batch_sizes == [1, 3, 3] + + @pytest.mark.asyncio + async def test_score_mapping_across_batches(self): + """Test that scores are correctly mapped back across batches.""" + call_counter = [0] + + async def mock_handler(method, url, **kwargs): + if "/rerank" in url: + body = kwargs.get("json", {}) + batch_num = call_counter[0] + call_counter[0] += 1 + texts = body["texts"] + # Each batch returns different scores to verify mapping + base_score = batch_num * 10 + results = [{"index": i, "score": float(base_score + i)} for i in range(len(texts))] + response = MagicMock() + response.status_code = 200 + response.raise_for_status = MagicMock() + response.json = MagicMock(return_value=results) + return response + raise httpx.HTTPStatusError("Not found", request=MagicMock(), response=MagicMock()) + + encoder = RemoteTEICrossEncoder( + base_url="http://localhost:8080", + batch_size=3, + ) + encoder._async_client = create_mock_async_client(mock_handler) + encoder._model_id = "test-model" + + pairs = [("Query", f"Doc {i}") for i in range(7)] + + scores = await encoder.predict(pairs) + + assert len(scores) == 7 + # All scores should be present (exact values depend on batch ordering) + assert all(isinstance(s, (int, float)) for s in scores) + + +class TestRemoteTEICrossEncoderParallelism: + """Tests for parallel request handling and backpressure.""" + + @pytest.mark.asyncio + async def test_parallel_requests(self): + """Test that requests are made in parallel.""" + concurrent_count = [0] + max_concurrent_observed = [0] + + async def mock_handler(method, url, **kwargs): + if "/rerank" in url: + concurrent_count[0] += 1 + max_concurrent_observed[0] = max(max_concurrent_observed[0], concurrent_count[0]) + + await asyncio.sleep(0.03) # Simulate latency + + concurrent_count[0] -= 1 + body = kwargs.get("json", {}) + texts = body["texts"] + results = [{"index": i, "score": 0.5} for i in range(len(texts))] + response = MagicMock() + response.status_code = 200 + response.raise_for_status = MagicMock() + response.json = MagicMock(return_value=results) + return response + raise httpx.HTTPStatusError("Not found", request=MagicMock(), response=MagicMock()) + + encoder = RemoteTEICrossEncoder( + base_url="http://localhost:8080", + batch_size=2, + max_concurrent=10, # High limit to allow parallelism + ) + encoder._async_client = create_mock_async_client(mock_handler) + encoder._model_id = "test-model" + + # 6 docs = 3 batches, should run in parallel + pairs = [("Query", f"Doc {i}") for i in range(6)] + + start = time.time() + scores = await encoder.predict(pairs) + elapsed = time.time() - start + + assert len(scores) == 6 + # If parallel, 3 batches with 30ms each should take ~30ms, not 90ms + assert elapsed < 0.08, f"Requests should run in parallel, took {elapsed}s" + assert max_concurrent_observed[0] > 1, "Multiple requests should run concurrently" + + @pytest.mark.asyncio + async def test_backpressure_semaphore(self): + """Test that semaphore limits concurrent requests.""" + concurrent_count = [0] + max_concurrent_observed = [0] + + async def mock_handler(method, url, **kwargs): + if "/rerank" in url: + concurrent_count[0] += 1 + max_concurrent_observed[0] = max(max_concurrent_observed[0], concurrent_count[0]) + + await asyncio.sleep(0.01) # Simulate latency + + concurrent_count[0] -= 1 + body = kwargs.get("json", {}) + texts = body["texts"] + results = [{"index": i, "score": 0.5} for i in range(len(texts))] + response = MagicMock() + response.status_code = 200 + response.raise_for_status = MagicMock() + response.json = MagicMock(return_value=results) + return response + raise httpx.HTTPStatusError("Not found", request=MagicMock(), response=MagicMock()) + + max_concurrent_limit = 2 + encoder = RemoteTEICrossEncoder( + base_url="http://localhost:8080", + batch_size=1, # 1 doc per batch to maximize requests + max_concurrent=max_concurrent_limit, + ) + encoder._async_client = create_mock_async_client(mock_handler) + encoder._model_id = "test-model" + + # 10 docs = 10 batches, but only 2 should run at a time + pairs = [("Query", f"Doc {i}") for i in range(10)] + + scores = await encoder.predict(pairs) + + assert len(scores) == 10 + assert max_concurrent_observed[0] <= max_concurrent_limit, ( + f"Semaphore should limit to {max_concurrent_limit}, observed {max_concurrent_observed[0]}" + ) + + +class TestRemoteTEICrossEncoderRetry: + """Tests for retry logic on transient errors.""" + + @pytest.mark.asyncio + async def test_retry_on_connect_error(self): + """Test that connect errors trigger retries.""" + attempt_count = [0] + + async def mock_handler(method, url, **kwargs): + if "/rerank" in url: + attempt_count[0] += 1 + if attempt_count[0] < 3: + raise httpx.ConnectError("Connection refused") + body = kwargs.get("json", {}) + texts = body["texts"] + results = [{"index": i, "score": 0.5} for i in range(len(texts))] + response = MagicMock() + response.status_code = 200 + response.raise_for_status = MagicMock() + response.json = MagicMock(return_value=results) + return response + raise httpx.HTTPStatusError("Not found", request=MagicMock(), response=MagicMock()) + + encoder = RemoteTEICrossEncoder( + base_url="http://localhost:8080", + max_retries=3, + retry_delay=0.01, + ) + encoder._async_client = create_mock_async_client(mock_handler) + encoder._model_id = "test-model" + + pairs = [("Query", "Doc 1")] + scores = await encoder.predict(pairs) + + assert len(scores) == 1 + assert attempt_count[0] == 3 # 2 failures + 1 success + + @pytest.mark.asyncio + async def test_retry_on_server_error(self): + """Test that 5xx errors trigger retries.""" + attempt_count = [0] + + async def mock_handler(method, url, **kwargs): + if "/rerank" in url: + attempt_count[0] += 1 + if attempt_count[0] < 2: + response = MagicMock() + response.status_code = 503 + + def raise_for_status(): + raise httpx.HTTPStatusError( + "Service unavailable", + request=MagicMock(), + response=response, + ) + + response.raise_for_status = raise_for_status + return response + + body = kwargs.get("json", {}) + texts = body["texts"] + results = [{"index": i, "score": 0.5} for i in range(len(texts))] + response = MagicMock() + response.status_code = 200 + response.raise_for_status = MagicMock() + response.json = MagicMock(return_value=results) + return response + raise httpx.HTTPStatusError("Not found", request=MagicMock(), response=MagicMock()) + + encoder = RemoteTEICrossEncoder( + base_url="http://localhost:8080", + max_retries=3, + retry_delay=0.01, + ) + encoder._async_client = create_mock_async_client(mock_handler) + encoder._model_id = "test-model" + + pairs = [("Query", "Doc 1")] + scores = await encoder.predict(pairs) + + assert len(scores) == 1 + assert attempt_count[0] == 2 + + @pytest.mark.asyncio + async def test_no_retry_on_client_error(self): + """Test that 4xx errors do not trigger retries.""" + attempt_count = [0] + + async def mock_handler(method, url, **kwargs): + if "/rerank" in url: + attempt_count[0] += 1 + response = MagicMock() + response.status_code = 400 + + def raise_for_status(): + raise httpx.HTTPStatusError( + "Bad request", + request=MagicMock(), + response=response, + ) + + response.raise_for_status = raise_for_status + return response + raise httpx.HTTPStatusError("Not found", request=MagicMock(), response=MagicMock()) + + encoder = RemoteTEICrossEncoder( + base_url="http://localhost:8080", + max_retries=3, + retry_delay=0.01, + ) + encoder._async_client = create_mock_async_client(mock_handler) + encoder._model_id = "test-model" + + pairs = [("Query", "Doc 1")] + + with pytest.raises(RuntimeError, match="TEI rerank request failed"): + await encoder.predict(pairs) + + assert attempt_count[0] == 1 # No retries for 4xx + + +class TestRemoteTEICrossEncoderConfig: + """Tests for configuration from environment variables.""" + + def test_default_values(self): + """Test default configuration values.""" + encoder = RemoteTEICrossEncoder(base_url="http://localhost:8080") + + assert encoder.batch_size == 128 + assert encoder.max_concurrent == 8 + assert encoder.timeout == 30.0 + assert encoder.max_retries == 3 + + def test_custom_values(self): + """Test custom configuration values.""" + encoder = RemoteTEICrossEncoder( + base_url="http://localhost:8080", + batch_size=64, + max_concurrent=4, + timeout=60.0, + max_retries=5, + retry_delay=1.0, + ) + + assert encoder.batch_size == 64 + assert encoder.max_concurrent == 4 + assert encoder.timeout == 60.0 + assert encoder.max_retries == 5 + assert encoder.retry_delay == 1.0 + + def test_create_from_env(self): + """Test creating encoder from environment variables.""" + import os + + from hindsight_api.engine.cross_encoder import create_cross_encoder_from_env + + with patch.dict( + os.environ, + { + "HINDSIGHT_API_RERANKER_PROVIDER": "tei", + "HINDSIGHT_API_RERANKER_TEI_URL": "http://test:9000", + "HINDSIGHT_API_RERANKER_TEI_BATCH_SIZE": "256", + "HINDSIGHT_API_RERANKER_TEI_MAX_CONCURRENT": "16", + }, + ): + encoder = create_cross_encoder_from_env() + + assert isinstance(encoder, RemoteTEICrossEncoder) + assert encoder.base_url == "http://test:9000" + assert encoder.batch_size == 256 + assert encoder.max_concurrent == 16 diff --git a/hindsight-control-plane/src/app/api/health/route.ts b/hindsight-control-plane/src/app/api/health/route.ts index 951bcd4e..f7c47a22 100644 --- a/hindsight-control-plane/src/app/api/health/route.ts +++ b/hindsight-control-plane/src/app/api/health/route.ts @@ -1,5 +1,7 @@ import { NextResponse } from "next/server"; -import { sdk, lowLevelClient } from "@/lib/hindsight-client"; +import { createClient, createConfig, sdk } from "@vectorize-io/hindsight-client"; + +const HEALTH_CHECK_TIMEOUT_MS = 3000; export async function GET() { const status: { @@ -15,19 +17,37 @@ export async function GET() { service: "hindsight-control-plane", }; - // Check dataplane connectivity + // Check dataplane connectivity with a short timeout const dataplaneUrl = process.env.HINDSIGHT_CP_DATAPLANE_API_URL || "http://localhost:8888"; try { - await sdk.listBanks({ client: lowLevelClient }); - status.dataplane = { - status: "connected", - url: dataplaneUrl, - }; + const controller = new AbortController(); + const timeoutId = setTimeout(() => controller.abort(), HEALTH_CHECK_TIMEOUT_MS); + + const healthClient = createClient( + createConfig({ + baseUrl: dataplaneUrl, + signal: controller.signal, + }) + ); + + try { + await sdk.listBanks({ client: healthClient }); + status.dataplane = { + status: "connected", + url: dataplaneUrl, + }; + } finally { + clearTimeout(timeoutId); + } } catch (error) { + let errorMessage = error instanceof Error ? error.message : String(error); + if (error instanceof Error && error.name === "AbortError") { + errorMessage = `Request timed out after ${HEALTH_CHECK_TIMEOUT_MS}ms`; + } status.dataplane = { status: "disconnected", url: dataplaneUrl, - error: error instanceof Error ? error.message : String(error), + error: errorMessage, }; } diff --git a/scripts/dev/monitoring/grafana/dashboards/hindsight-llm.json b/scripts/dev/monitoring/grafana/dashboards/hindsight-llm.json new file mode 100644 index 00000000..90a5d3a1 --- /dev/null +++ b/scripts/dev/monitoring/grafana/dashboards/hindsight-llm.json @@ -0,0 +1,541 @@ +{ + "annotations": { + "list": [] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "id": null, + "links": [], + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "green", "value": null } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { "h": 4, "w": 6, "x": 0, "y": 0 }, + "id": 1, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "10.0.0", + "targets": [ + { + "expr": "sum(hindsight_llm_calls_total)", + "refId": "A" + } + ], + "title": "Total LLM Calls", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "green", "value": null } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { "h": 4, "w": 6, "x": 6, "y": 0 }, + "id": 2, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "10.0.0", + "targets": [ + { + "expr": "sum(hindsight_llm_tokens_input_tokens_total) + sum(hindsight_llm_tokens_output_tokens_total)", + "refId": "A" + } + ], + "title": "Total Tokens Used", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "blue", + "mode": "fixed" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "green", "value": null } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { "h": 4, "w": 6, "x": 12, "y": 0 }, + "id": 3, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "10.0.0", + "targets": [ + { + "expr": "sum(hindsight_llm_tokens_input_tokens_total)", + "refId": "A" + } + ], + "title": "Input Tokens", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "green", + "mode": "fixed" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "green", "value": null } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { "h": 4, "w": 6, "x": 18, "y": 0 }, + "id": 4, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "10.0.0", + "targets": [ + { + "expr": "sum(hindsight_llm_tokens_output_tokens_total)", + "refId": "A" + } + ], + "title": "Output Tokens", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "green", "value": null } + ] + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { "h": 8, "w": 24, "x": 0, "y": 4 }, + "id": 5, + "options": { + "legend": { + "calcs": ["mean", "max"], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "10.0.0", + "targets": [ + { + "expr": "sum by (scope) (rate(hindsight_llm_calls_total[1m]))", + "legendFormat": "{{scope}}", + "refId": "A" + } + ], + "title": "LLM Calls per Second by Scope", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "green", "value": null } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { "id": "byName", "options": "Input" }, + "properties": [{ "id": "color", "value": { "fixedColor": "blue", "mode": "fixed" } }] + }, + { + "matcher": { "id": "byName", "options": "Output" }, + "properties": [{ "id": "color", "value": { "fixedColor": "green", "mode": "fixed" } }] + } + ] + }, + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 12 }, + "id": 6, + "options": { + "legend": { + "calcs": ["mean", "max"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "10.0.0", + "targets": [ + { + "expr": "sum(rate(hindsight_llm_tokens_input_tokens_total[1m]))", + "legendFormat": "Input", + "refId": "A" + }, + { + "expr": "sum(rate(hindsight_llm_tokens_output_tokens_total[1m]))", + "legendFormat": "Output", + "refId": "B" + } + ], + "title": "Token Usage Rate (tokens/sec)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "green", "value": null } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 12 }, + "id": 7, + "options": { + "legend": { + "calcs": ["mean", "max"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "10.0.0", + "targets": [ + { + "expr": "histogram_quantile(0.95, sum by (scope, le) (rate(hindsight_llm_duration_seconds_bucket[5m])))", + "legendFormat": "{{scope}}", + "refId": "A" + } + ], + "title": "LLM Call Latency p95 by Scope", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "green", "value": null } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { "h": 8, "w": 24, "x": 0, "y": 20 }, + "id": 8, + "options": { + "legend": { + "calcs": ["mean", "max"], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "10.0.0", + "targets": [ + { + "expr": "sum by (scope) (rate(hindsight_llm_tokens_input_tokens_total[1m]))", + "legendFormat": "{{scope}} (input)", + "refId": "A" + }, + { + "expr": "sum by (scope) (rate(hindsight_llm_tokens_output_tokens_total[1m]))", + "legendFormat": "{{scope}} (output)", + "refId": "B" + } + ], + "title": "Token Usage by Scope (tokens/sec)", + "type": "timeseries" + } + ], + "refresh": "5s", + "schemaVersion": 38, + "tags": ["hindsight", "llm"], + "templating": { + "list": [] + }, + "time": { + "from": "now-30m", + "to": "now" + }, + "timepicker": {}, + "timezone": "browser", + "title": "Hindsight LLM Metrics", + "uid": "hindsight-llm", + "version": 1, + "weekStart": "" +} diff --git a/scripts/dev/monitoring/grafana/dashboards/hindsight-operations.json b/scripts/dev/monitoring/grafana/dashboards/hindsight-operations.json new file mode 100644 index 00000000..dfc717bc --- /dev/null +++ b/scripts/dev/monitoring/grafana/dashboards/hindsight-operations.json @@ -0,0 +1,604 @@ +{ + "annotations": { + "list": [] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "id": null, + "links": [], + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "green", "value": null } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { "h": 4, "w": 6, "x": 0, "y": 0 }, + "id": 1, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "10.0.0", + "targets": [ + { + "expr": "sum(hindsight_operation_operations_total)", + "refId": "A" + } + ], + "title": "Total Operations", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "green", "value": null } + ] + }, + "unit": "ops" + }, + "overrides": [] + }, + "gridPos": { "h": 4, "w": 6, "x": 6, "y": 0 }, + "id": 2, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "10.0.0", + "targets": [ + { + "expr": "sum(rate(hindsight_operation_operations_total[1m]))", + "refId": "A" + } + ], + "title": "Operations/sec", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "green", + "mode": "fixed" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "green", "value": null } + ] + }, + "unit": "ops" + }, + "overrides": [] + }, + "gridPos": { "h": 4, "w": 4, "x": 12, "y": 0 }, + "id": 3, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "10.0.0", + "targets": [ + { + "expr": "sum(rate(hindsight_operation_operations_total{operation=\"retain\"}[1m]))", + "refId": "A" + } + ], + "title": "Retain/sec", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "blue", + "mode": "fixed" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "green", "value": null } + ] + }, + "unit": "ops" + }, + "overrides": [] + }, + "gridPos": { "h": 4, "w": 4, "x": 16, "y": 0 }, + "id": 4, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "10.0.0", + "targets": [ + { + "expr": "sum(rate(hindsight_operation_operations_total{operation=\"recall\"}[1m]))", + "refId": "A" + } + ], + "title": "Recall/sec", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "orange", + "mode": "fixed" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "green", "value": null } + ] + }, + "unit": "ops" + }, + "overrides": [] + }, + "gridPos": { "h": 4, "w": 4, "x": 20, "y": 0 }, + "id": 5, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "10.0.0", + "targets": [ + { + "expr": "sum(rate(hindsight_operation_operations_total{operation=\"reflect\"}[1m]))", + "refId": "A" + } + ], + "title": "Reflect/sec", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "green", "value": null } + ] + }, + "unit": "ops" + }, + "overrides": [ + { + "matcher": { "id": "byName", "options": "retain" }, + "properties": [{ "id": "color", "value": { "fixedColor": "green", "mode": "fixed" } }] + }, + { + "matcher": { "id": "byName", "options": "recall" }, + "properties": [{ "id": "color", "value": { "fixedColor": "blue", "mode": "fixed" } }] + }, + { + "matcher": { "id": "byName", "options": "reflect" }, + "properties": [{ "id": "color", "value": { "fixedColor": "orange", "mode": "fixed" } }] + } + ] + }, + "gridPos": { "h": 8, "w": 24, "x": 0, "y": 4 }, + "id": 6, + "options": { + "legend": { + "calcs": ["mean", "max"], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "10.0.0", + "targets": [ + { + "expr": "sum by (operation) (rate(hindsight_operation_operations_total[1m]))", + "legendFormat": "{{operation}}", + "refId": "A" + } + ], + "title": "Operations per Second by Type", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "green", "value": null } + ] + }, + "unit": "s" + }, + "overrides": [ + { + "matcher": { "id": "byName", "options": "p50" }, + "properties": [{ "id": "color", "value": { "fixedColor": "green", "mode": "fixed" } }] + }, + { + "matcher": { "id": "byName", "options": "p95" }, + "properties": [{ "id": "color", "value": { "fixedColor": "yellow", "mode": "fixed" } }] + }, + { + "matcher": { "id": "byName", "options": "p99" }, + "properties": [{ "id": "color", "value": { "fixedColor": "red", "mode": "fixed" } }] + } + ] + }, + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 12 }, + "id": 7, + "options": { + "legend": { + "calcs": ["mean", "max"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "10.0.0", + "targets": [ + { + "expr": "histogram_quantile(0.50, sum by (le) (rate(hindsight_operation_duration_seconds_bucket{operation=\"recall\"}[5m])))", + "legendFormat": "p50", + "refId": "A" + }, + { + "expr": "histogram_quantile(0.95, sum by (le) (rate(hindsight_operation_duration_seconds_bucket{operation=\"recall\"}[5m])))", + "legendFormat": "p95", + "refId": "B" + }, + { + "expr": "histogram_quantile(0.99, sum by (le) (rate(hindsight_operation_duration_seconds_bucket{operation=\"recall\"}[5m])))", + "legendFormat": "p99", + "refId": "C" + } + ], + "title": "Recall Latency Percentiles", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "green", "value": null } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 12 }, + "id": 8, + "options": { + "legend": { + "calcs": ["mean", "max"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "10.0.0", + "targets": [ + { + "expr": "histogram_quantile(0.95, sum by (operation, le) (rate(hindsight_operation_duration_seconds_bucket[5m])))", + "legendFormat": "{{operation}}", + "refId": "A" + } + ], + "title": "p95 Latency by Operation Type", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "green", "value": null } + ] + }, + "unit": "ops" + }, + "overrides": [] + }, + "gridPos": { "h": 8, "w": 24, "x": 0, "y": 20 }, + "id": 9, + "options": { + "legend": { + "calcs": ["mean", "max"], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "10.0.0", + "targets": [ + { + "expr": "sum by (bank_id) (rate(hindsight_operation_operations_total[1m]))", + "legendFormat": "{{bank_id}}", + "refId": "A" + } + ], + "title": "Operations per Second by Bank", + "type": "timeseries" + } + ], + "refresh": "5s", + "schemaVersion": 38, + "tags": ["hindsight"], + "templating": { + "list": [] + }, + "time": { + "from": "now-30m", + "to": "now" + }, + "timepicker": {}, + "timezone": "browser", + "title": "Hindsight Operations", + "uid": "hindsight-operations", + "version": 1, + "weekStart": "" +} diff --git a/scripts/dev/monitoring/start.sh b/scripts/dev/monitoring/start.sh new file mode 100755 index 00000000..7d1cf2e3 --- /dev/null +++ b/scripts/dev/monitoring/start.sh @@ -0,0 +1,221 @@ +#!/bin/bash +set -e + +# Script to start Prometheus and Grafana for Hindsight metrics +# This provides a single command for the full monitoring stack + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +PROJECT_ROOT="$(cd "$SCRIPT_DIR/../../.." && pwd)" +MONITORING_DATA_DIR="$PROJECT_ROOT/.monitoring" +API_PORT="${API_PORT:-8888}" +PROMETHEUS_PORT="${PROMETHEUS_PORT:-8889}" +GRAFANA_PORT="${GRAFANA_PORT:-8890}" + +# Versions +PROMETHEUS_VERSION="2.48.0" +GRAFANA_VERSION="10.2.2" + +# Detect OS and architecture +OS=$(uname -s | tr '[:upper:]' '[:lower:]') +ARCH=$(uname -m) + +case "$OS" in + darwin) OS_NAME="darwin" ;; + linux) OS_NAME="linux" ;; + *) echo "Unsupported OS: $OS"; exit 1 ;; +esac + +case "$ARCH" in + x86_64) ARCH_NAME="amd64" ;; + arm64|aarch64) ARCH_NAME="arm64" ;; + *) echo "Unsupported architecture: $ARCH"; exit 1 ;; +esac + +# Prometheus paths +PROMETHEUS_DIR="$MONITORING_DATA_DIR/prometheus" +PROMETHEUS_ARCHIVE="prometheus-${PROMETHEUS_VERSION}.${OS_NAME}-${ARCH_NAME}.tar.gz" +PROMETHEUS_URL="https://github.com/prometheus/prometheus/releases/download/v${PROMETHEUS_VERSION}/${PROMETHEUS_ARCHIVE}" +PROMETHEUS_BIN="$PROMETHEUS_DIR/prometheus-${PROMETHEUS_VERSION}.${OS_NAME}-${ARCH_NAME}/prometheus" + +# Grafana paths +GRAFANA_DIR="$MONITORING_DATA_DIR/grafana" +GRAFANA_ARCHIVE="grafana-${GRAFANA_VERSION}.${OS_NAME}-${ARCH_NAME}.tar.gz" +GRAFANA_URL="https://dl.grafana.com/oss/release/${GRAFANA_ARCHIVE}" +GRAFANA_HOME="$GRAFANA_DIR/grafana-v${GRAFANA_VERSION}" +GRAFANA_BIN="$GRAFANA_HOME/bin/grafana" + +# Cleanup function +cleanup() { + echo "" + echo "Shutting down monitoring stack..." + + if [ -n "$PROM_PID" ] && kill -0 "$PROM_PID" 2>/dev/null; then + kill "$PROM_PID" 2>/dev/null || true + fi + + if [ -n "$GRAFANA_PID" ] && kill -0 "$GRAFANA_PID" 2>/dev/null; then + kill "$GRAFANA_PID" 2>/dev/null || true + fi + + echo "Monitoring stack stopped" + exit 0 +} + +trap cleanup SIGINT SIGTERM + +# Download Prometheus if needed +if [ ! -f "$PROMETHEUS_BIN" ]; then + echo "Downloading Prometheus ${PROMETHEUS_VERSION}..." + mkdir -p "$PROMETHEUS_DIR" + cd "$PROMETHEUS_DIR" + curl -sL -o "$PROMETHEUS_ARCHIVE" "$PROMETHEUS_URL" + tar xzf "$PROMETHEUS_ARCHIVE" + rm "$PROMETHEUS_ARCHIVE" + echo "Prometheus ready" +fi + +# Download Grafana if needed +if [ ! -f "$GRAFANA_BIN" ]; then + echo "Downloading Grafana ${GRAFANA_VERSION}..." + mkdir -p "$GRAFANA_DIR" + cd "$GRAFANA_DIR" + curl -sL -o "$GRAFANA_ARCHIVE" "$GRAFANA_URL" + tar xzf "$GRAFANA_ARCHIVE" + rm "$GRAFANA_ARCHIVE" + echo "Grafana ready" +fi + +# Create Prometheus config +mkdir -p "$PROMETHEUS_DIR" +cat > "$PROMETHEUS_DIR/prometheus.yml" < "$GRAFANA_PROV_DIR/datasources/prometheus.yaml" < "$GRAFANA_PROV_DIR/dashboards/dashboards.yaml" < "$GRAFANA_DIR/grafana.ini" < /dev/null 2>&1; then + echo "WARNING: Hindsight API not detected at localhost:$API_PORT" + echo " Start the API first: ./scripts/dev/start-api.sh" + echo "" +fi + +# Start Prometheus in background +cd "$(dirname "$PROMETHEUS_BIN")" +"$PROMETHEUS_BIN" \ + --config.file="$PROMETHEUS_DIR/prometheus.yml" \ + --storage.tsdb.path="$PROMETHEUS_DIR/data" \ + --web.console.templates="$(dirname "$PROMETHEUS_BIN")/consoles" \ + --web.console.libraries="$(dirname "$PROMETHEUS_BIN")/console_libraries" \ + --web.listen-address="0.0.0.0:$PROMETHEUS_PORT" \ + --web.enable-lifecycle \ + --log.level=warn & +PROM_PID=$! + +# Start Grafana in background +cd "$GRAFANA_HOME" +"$GRAFANA_BIN" server \ + --homepath="$GRAFANA_HOME" \ + --config="$GRAFANA_DIR/grafana.ini" & +GRAFANA_PID=$! + +echo "Monitoring stack running. Press Ctrl+C to stop." +echo "" + +# Wait for processes +wait "$PROM_PID" "$GRAFANA_PID" 2>/dev/null || true + +# If we get here, clean up +cleanup diff --git a/scripts/dev/start-monitoring.sh b/scripts/dev/start-monitoring.sh new file mode 100755 index 00000000..61df287b --- /dev/null +++ b/scripts/dev/start-monitoring.sh @@ -0,0 +1,3 @@ +#!/bin/bash +# Convenience wrapper to start the monitoring stack +exec "$(dirname "${BASH_SOURCE[0]}")/monitoring/start.sh" "$@" diff --git a/scripts/prometheus-dashboard.html b/scripts/prometheus-dashboard.html deleted file mode 100644 index 4de23451..00000000 --- a/scripts/prometheus-dashboard.html +++ /dev/null @@ -1,384 +0,0 @@ - - - - - - Hindsight Metrics Dashboard - - - - -
-

🧠 Hindsight Metrics Dashboard

-
Real-time performance monitoring • Updates every 15s
-
- -
-
Loading metrics...
- - - - - - - - diff --git a/scripts/start-prometheus.sh b/scripts/start-prometheus.sh deleted file mode 100755 index 535735c9..00000000 --- a/scripts/start-prometheus.sh +++ /dev/null @@ -1,138 +0,0 @@ -#!/bin/bash -set -e - -# Script to download and start Prometheus for Hindsight metrics -# This creates a local Prometheus instance that scrapes metrics from the Hindsight API - -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -PROJECT_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" -PROMETHEUS_DIR="$PROJECT_ROOT/.prometheus" -PROMETHEUS_VERSION="2.48.0" - -# Detect OS and architecture -OS=$(uname -s | tr '[:upper:]' '[:lower:]') -ARCH=$(uname -m) - -case "$OS" in - darwin) - OS_NAME="darwin" - ;; - linux) - OS_NAME="linux" - ;; - *) - echo "Unsupported OS: $OS" - exit 1 - ;; -esac - -case "$ARCH" in - x86_64) - ARCH_NAME="amd64" - ;; - arm64|aarch64) - ARCH_NAME="arm64" - ;; - *) - echo "Unsupported architecture: $ARCH" - exit 1 - ;; -esac - -PROMETHEUS_ARCHIVE="prometheus-${PROMETHEUS_VERSION}.${OS_NAME}-${ARCH_NAME}.tar.gz" -PROMETHEUS_URL="https://github.com/prometheus/prometheus/releases/download/v${PROMETHEUS_VERSION}/${PROMETHEUS_ARCHIVE}" -PROMETHEUS_BIN="$PROMETHEUS_DIR/prometheus-${PROMETHEUS_VERSION}.${OS_NAME}-${ARCH_NAME}/prometheus" - -echo "🔧 Setting up Prometheus for Hindsight metrics..." -echo "" - -# Create prometheus directory -mkdir -p "$PROMETHEUS_DIR" -cd "$PROMETHEUS_DIR" - -# Download Prometheus if not exists -if [ ! -f "$PROMETHEUS_BIN" ]; then - echo "📥 Downloading Prometheus ${PROMETHEUS_VERSION} for ${OS_NAME}-${ARCH_NAME}..." - curl -L -o "$PROMETHEUS_ARCHIVE" "$PROMETHEUS_URL" - - echo "📦 Extracting..." - tar xzf "$PROMETHEUS_ARCHIVE" - - echo "✅ Prometheus downloaded successfully" - echo "" -else - echo "✅ Prometheus already downloaded" - echo "" -fi - -# Create prometheus.yml configuration -echo "📝 Creating Prometheus configuration..." -cat > "$PROMETHEUS_DIR/prometheus.yml" < /dev/null 2>&1; then - echo "✅ Hindsight API is running and serving metrics" - echo "" -else - echo "⚠️ WARNING: Hindsight API is not reachable at http://localhost:8000/metrics" - echo " Make sure to start the API before Prometheus can scrape metrics" - echo "" -fi - -# Start Prometheus -echo "🚀 Starting Prometheus..." -echo "" -echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" -echo " Prometheus UI: http://localhost:9090" -echo " Metrics source: http://localhost:8000/metrics" -echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" -echo "" -echo "📊 Example queries to try in the UI:" -echo "" -echo " p95 latency (all operations):" -echo " histogram_quantile(0.95, rate(hindsight_operation_duration_seconds_bucket[5m]))" -echo "" -echo " p95 latency by bank:" -echo " histogram_quantile(0.95, sum by (bank_id, le) (rate(hindsight_operation_duration_seconds_bucket{operation=\"recall\"}[5m])))" -echo "" -echo " Operations per second:" -echo " rate(hindsight_operation_total[5m])" -echo "" -echo " Token usage rate:" -echo " rate(hindsight_tokens_input_total[5m]) + rate(hindsight_tokens_output_total[5m])" -echo "" -echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" -echo "" -echo "Press Ctrl+C to stop Prometheus" -echo "" - -# Start Prometheus with config -cd "$(dirname "$PROMETHEUS_BIN")" -exec "$PROMETHEUS_BIN" \ - --config.file="$PROMETHEUS_DIR/prometheus.yml" \ - --storage.tsdb.path="$PROMETHEUS_DIR/data" \ - --web.console.templates="$PROMETHEUS_DIR/prometheus-${PROMETHEUS_VERSION}.${OS_NAME}-${ARCH_NAME}/consoles" \ - --web.console.libraries="$PROMETHEUS_DIR/prometheus-${PROMETHEUS_VERSION}.${OS_NAME}-${ARCH_NAME}/console_libraries"