From 66dedb8d414c12e541e83a5c69c045c9aaddfdfc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=B2=20Boschi?= Date: Wed, 11 Mar 2026 14:56:59 +0100 Subject: [PATCH] feat: make recall max query tokens configurable via env var (#544) * doc: add 0.4.17 release blog post * feat: make recall max query tokens configurable via env var Add HINDSIGHT_API_RECALL_MAX_QUERY_TOKENS env var (default: 500) to replace the hardcoded MAX_QUERY_TOKENS constant in http.py. --- hindsight-api/hindsight_api/api/http.py | 7 +++---- hindsight-api/hindsight_api/config.py | 4 ++++ hindsight-api/hindsight_api/main.py | 1 + hindsight-docs/docs/developer/configuration.md | 1 + 4 files changed, 9 insertions(+), 4 deletions(-) diff --git a/hindsight-api/hindsight_api/api/http.py b/hindsight-api/hindsight_api/api/http.py index aee66f82..0fc8d245 100644 --- a/hindsight-api/hindsight_api/api/http.py +++ b/hindsight-api/hindsight_api/api/http.py @@ -80,8 +80,6 @@ from hindsight_api.models import RequestContext logger = logging.getLogger(__name__) -MAX_QUERY_TOKENS = 500 # Maximum tokens allowed in recall query - class EntityIncludeOptions(BaseModel): """Options for including entity observations in recall results.""" @@ -2263,12 +2261,13 @@ def _register_routes(app: FastAPI): metrics = get_metrics_collector() # Validate query length to prevent expensive operations on oversized queries + max_query_tokens = get_config().recall_max_query_tokens encoding = _get_tiktoken_encoding() query_tokens = len(encoding.encode(request.query)) - if query_tokens > MAX_QUERY_TOKENS: + if query_tokens > max_query_tokens: raise HTTPException( status_code=400, - detail=f"Query too long: {query_tokens} tokens exceeds maximum of {MAX_QUERY_TOKENS}. Please shorten your query.", + detail=f"Query too long: {query_tokens} tokens exceeds maximum of {max_query_tokens}. Please shorten your query.", ) try: diff --git a/hindsight-api/hindsight_api/config.py b/hindsight-api/hindsight_api/config.py index fb6b243a..7341a734 100644 --- a/hindsight-api/hindsight_api/config.py +++ b/hindsight-api/hindsight_api/config.py @@ -238,6 +238,7 @@ ENV_GRAPH_RETRIEVER = "HINDSIGHT_API_GRAPH_RETRIEVER" ENV_MPFP_TOP_K_NEIGHBORS = "HINDSIGHT_API_MPFP_TOP_K_NEIGHBORS" ENV_RECALL_MAX_CONCURRENT = "HINDSIGHT_API_RECALL_MAX_CONCURRENT" ENV_RECALL_CONNECTION_BUDGET = "HINDSIGHT_API_RECALL_CONNECTION_BUDGET" +ENV_RECALL_MAX_QUERY_TOKENS = "HINDSIGHT_API_RECALL_MAX_QUERY_TOKENS" ENV_MENTAL_MODEL_REFRESH_CONCURRENCY = "HINDSIGHT_API_MENTAL_MODEL_REFRESH_CONCURRENCY" # OpenTelemetry tracing configuration @@ -425,6 +426,7 @@ DEFAULT_GRAPH_RETRIEVER = "link_expansion" # Options: "link_expansion", "mpfp", DEFAULT_MPFP_TOP_K_NEIGHBORS = 20 # Fan-out limit per node in MPFP graph traversal DEFAULT_RECALL_MAX_CONCURRENT = 32 # Max concurrent recall operations per worker DEFAULT_RECALL_CONNECTION_BUDGET = 4 # Max concurrent DB connections per recall operation +DEFAULT_RECALL_MAX_QUERY_TOKENS = 500 # Maximum tokens allowed in recall query DEFAULT_MENTAL_MODEL_REFRESH_CONCURRENCY = 8 # Max concurrent mental model refreshes # Retain settings @@ -694,6 +696,7 @@ class HindsightConfig: mpfp_top_k_neighbors: int recall_max_concurrent: int recall_connection_budget: int + recall_max_query_tokens: int mental_model_refresh_concurrency: int # Retain settings @@ -1126,6 +1129,7 @@ class HindsightConfig: recall_connection_budget=int( os.getenv(ENV_RECALL_CONNECTION_BUDGET, str(DEFAULT_RECALL_CONNECTION_BUDGET)) ), + recall_max_query_tokens=int(os.getenv(ENV_RECALL_MAX_QUERY_TOKENS, str(DEFAULT_RECALL_MAX_QUERY_TOKENS))), mental_model_refresh_concurrency=int( os.getenv(ENV_MENTAL_MODEL_REFRESH_CONCURRENCY, str(DEFAULT_MENTAL_MODEL_REFRESH_CONCURRENCY)) ), diff --git a/hindsight-api/hindsight_api/main.py b/hindsight-api/hindsight_api/main.py index 8151c4f7..43f0b9f1 100644 --- a/hindsight-api/hindsight_api/main.py +++ b/hindsight-api/hindsight_api/main.py @@ -246,6 +246,7 @@ def main(): mpfp_top_k_neighbors=config.mpfp_top_k_neighbors, recall_max_concurrent=config.recall_max_concurrent, recall_connection_budget=config.recall_connection_budget, + recall_max_query_tokens=config.recall_max_query_tokens, retain_max_completion_tokens=config.retain_max_completion_tokens, retain_chunk_size=config.retain_chunk_size, retain_extract_causal_links=config.retain_extract_causal_links, diff --git a/hindsight-docs/docs/developer/configuration.md b/hindsight-docs/docs/developer/configuration.md index 43f3368c..1c6a0e93 100644 --- a/hindsight-docs/docs/developer/configuration.md +++ b/hindsight-docs/docs/developer/configuration.md @@ -530,6 +530,7 @@ For advanced authentication (JWT, OAuth, multi-tenant schemas), implement a cust | `HINDSIGHT_API_GRAPH_RETRIEVER` | Graph retrieval algorithm: `link_expansion`, `mpfp`, or `bfs` | `link_expansion` | | `HINDSIGHT_API_RECALL_MAX_CONCURRENT` | Max concurrent recall operations per worker (backpressure) | `32` | | `HINDSIGHT_API_RECALL_CONNECTION_BUDGET` | Max concurrent DB connections per recall operation | `4` | +| `HINDSIGHT_API_RECALL_MAX_QUERY_TOKENS` | Maximum token length of a recall query; requests exceeding this limit are rejected with HTTP 400 | `500` | | `HINDSIGHT_API_RERANKER_MAX_CANDIDATES` | Max candidates to rerank per recall (RRF pre-filters the rest) | `300` | | `HINDSIGHT_API_MPFP_TOP_K_NEIGHBORS` | Fan-out limit per node in MPFP graph traversal | `20` | | `HINDSIGHT_API_MENTAL_MODEL_REFRESH_CONCURRENCY` | Max concurrent mental model refreshes | `8` |