feat(config): Add configurable observation thresholds (#83)

Allows tuning of entity observation generation via environment variables.

## New Environment Variables
- `HINDSIGHT_API_OBSERVATION_MIN_FACTS` - Minimum facts required to
  generate entity observations (default: 5)
- `HINDSIGHT_API_OBSERVATION_TOP_ENTITIES` - Maximum entities to process
  per retain batch (default: 5)

## Changes
- Added threshold configuration to HindsightConfig
- Updated memory_engine.py to use config values
- Updated observation_regeneration.py to use config values

## Use Case
Lower thresholds generate more observations (better recall, higher cost).
Higher thresholds are more selective (lower cost, may miss patterns).

Example:
```bash
# Generate more observations
docker run -e HINDSIGHT_API_OBSERVATION_MIN_FACTS=3 \
           -e HINDSIGHT_API_OBSERVATION_TOP_ENTITIES=10 ...
```

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-authored-by: Claude Opus 4.5 <noreply@anthropic.com>

hindsight-api/hindsight_api/config.py

@@ -33,6 +33,10 @@ ENV_GRAPH_RETRIEVER = "HINDSIGHT_API_GRAPH_RETRIEVER"
 ENV_MCP_LOCAL_BANK_ID = "HINDSIGHT_API_MCP_LOCAL_BANK_ID"
 ENV_MCP_INSTRUCTIONS = "HINDSIGHT_API_MCP_INSTRUCTIONS"
 
+# Observation thresholds
+ENV_OBSERVATION_MIN_FACTS = "HINDSIGHT_API_OBSERVATION_MIN_FACTS"
+ENV_OBSERVATION_TOP_ENTITIES = "HINDSIGHT_API_OBSERVATION_TOP_ENTITIES"
+
 # Optimization flags
 ENV_SKIP_LLM_VERIFICATION = "HINDSIGHT_API_SKIP_LLM_VERIFICATION"
 ENV_LAZY_RERANKER = "HINDSIGHT_API_LAZY_RERANKER"
@@ -55,6 +59,10 @@ DEFAULT_MCP_ENABLED = True
 DEFAULT_GRAPH_RETRIEVER = "bfs"  # Options: "bfs", "mpfp"
 DEFAULT_MCP_LOCAL_BANK_ID = "mcp"
 
+# Observation thresholds
+DEFAULT_OBSERVATION_MIN_FACTS = 5  # Min facts required to generate entity observations
+DEFAULT_OBSERVATION_TOP_ENTITIES = 5  # Max entities to process per retain batch
+
 # Default MCP tool descriptions (can be customized via env vars)
 DEFAULT_MCP_RETAIN_DESCRIPTION = """Store important information to long-term memory.
 
@@ -111,6 +119,10 @@ class HindsightConfig:
     # Recall
     graph_retriever: str
 
+    # Observation thresholds
+    observation_min_facts: int
+    observation_top_entities: int
+
     # Optimization flags
     skip_llm_verification: bool
     lazy_reranker: bool
@@ -144,6 +156,9 @@ class HindsightConfig:
             # Optimization flags
             skip_llm_verification=os.getenv(ENV_SKIP_LLM_VERIFICATION, "false").lower() == "true",
             lazy_reranker=os.getenv(ENV_LAZY_RERANKER, "false").lower() == "true",
+            # Observation thresholds
+            observation_min_facts=int(os.getenv(ENV_OBSERVATION_MIN_FACTS, str(DEFAULT_OBSERVATION_MIN_FACTS))),
+            observation_top_entities=int(os.getenv(ENV_OBSERVATION_TOP_ENTITIES, str(DEFAULT_OBSERVATION_TOP_ENTITIES))),
         )
 
     def get_llm_base_url(self) -> str:

hindsight-api/hindsight_api/engine/memory_engine.py

@@ -17,6 +17,8 @@ import uuid
 from datetime import UTC, datetime, timedelta
 from typing import TYPE_CHECKING, Any
 
+from ..config import get_config
+
 # Context variable for current schema (async-safe, per-task isolation)
 _current_schema: contextvars.ContextVar[str] = contextvars.ContextVar("current_schema", default="public")
 
@@ -3572,7 +3574,7 @@ Guidelines:
         self,
         bank_id: str,
         entity_ids: list[str],
-        min_facts: int = 5,
+        min_facts: int | None = None,
         conn=None,
         request_context: "RequestContext | None" = None,
     ) -> None:
@@ -3584,12 +3586,16 @@ Guidelines:
         Args:
             bank_id: Bank identifier
             entity_ids: List of entity IDs to process
-            min_facts: Minimum facts required to regenerate observations
+            min_facts: Minimum facts required to regenerate observations (uses config default if None)
             conn: Optional database connection (for transactional atomicity)
         """
         if not bank_id or not entity_ids:
             return
 
+        # Use config default if min_facts not specified
+        if min_facts is None:
+            min_facts = get_config().observation_min_facts
+
         # Convert to UUIDs
         entity_uuids = [uuid.UUID(eid) if isinstance(eid, str) else eid for eid in entity_ids]
 

hindsight-api/hindsight_api/engine/retain/observation_regeneration.py

@@ -9,6 +9,7 @@ import time
 import uuid
 from datetime import UTC, datetime
 
+from ...config import get_config
 from ..memory_engine import fq_table
 from ..search import observation_utils
 from . import embedding_utils
@@ -49,8 +50,9 @@ async def regenerate_observations_batch(
         entity_links: Entity links from this batch
         log_buffer: Optional log buffer for timing
     """
-    TOP_N_ENTITIES = 5
-    MIN_FACTS_THRESHOLD = 5
+    config = get_config()
+    TOP_N_ENTITIES = config.observation_top_entities
+    MIN_FACTS_THRESHOLD = config.observation_min_facts
 
     if not entity_links:
         return