diff --git a/hindsight-api/hindsight_api/config.py b/hindsight-api/hindsight_api/config.py index bdf5921c..87b59ceb 100644 --- a/hindsight-api/hindsight_api/config.py +++ b/hindsight-api/hindsight_api/config.py @@ -25,6 +25,17 @@ ENV_LLM_MAX_CONCURRENT = "HINDSIGHT_API_LLM_MAX_CONCURRENT" ENV_LLM_TIMEOUT = "HINDSIGHT_API_LLM_TIMEOUT" ENV_LLM_GROQ_SERVICE_TIER = "HINDSIGHT_API_LLM_GROQ_SERVICE_TIER" +# Per-operation LLM configuration (optional, falls back to global LLM config) +ENV_RETAIN_LLM_PROVIDER = "HINDSIGHT_API_RETAIN_LLM_PROVIDER" +ENV_RETAIN_LLM_API_KEY = "HINDSIGHT_API_RETAIN_LLM_API_KEY" +ENV_RETAIN_LLM_MODEL = "HINDSIGHT_API_RETAIN_LLM_MODEL" +ENV_RETAIN_LLM_BASE_URL = "HINDSIGHT_API_RETAIN_LLM_BASE_URL" + +ENV_REFLECT_LLM_PROVIDER = "HINDSIGHT_API_REFLECT_LLM_PROVIDER" +ENV_REFLECT_LLM_API_KEY = "HINDSIGHT_API_REFLECT_LLM_API_KEY" +ENV_REFLECT_LLM_MODEL = "HINDSIGHT_API_REFLECT_LLM_MODEL" +ENV_REFLECT_LLM_BASE_URL = "HINDSIGHT_API_REFLECT_LLM_BASE_URL" + ENV_EMBEDDINGS_PROVIDER = "HINDSIGHT_API_EMBEDDINGS_PROVIDER" ENV_EMBEDDINGS_LOCAL_MODEL = "HINDSIGHT_API_EMBEDDINGS_LOCAL_MODEL" ENV_EMBEDDINGS_TEI_URL = "HINDSIGHT_API_EMBEDDINGS_TEI_URL" @@ -127,7 +138,7 @@ class HindsightConfig: # Database database_url: str - # LLM + # LLM (default, used as fallback for per-operation config) llm_provider: str llm_api_key: str | None llm_model: str @@ -135,6 +146,17 @@ class HindsightConfig: llm_max_concurrent: int llm_timeout: float + # Per-operation LLM configuration (None = use default LLM config) + retain_llm_provider: str | None + retain_llm_api_key: str | None + retain_llm_model: str | None + retain_llm_base_url: str | None + + reflect_llm_provider: str | None + reflect_llm_api_key: str | None + reflect_llm_model: str | None + reflect_llm_base_url: str | None + # Embeddings embeddings_provider: str embeddings_local_model: str @@ -181,6 +203,15 @@ class HindsightConfig: llm_base_url=os.getenv(ENV_LLM_BASE_URL) or None, llm_max_concurrent=int(os.getenv(ENV_LLM_MAX_CONCURRENT, str(DEFAULT_LLM_MAX_CONCURRENT))), llm_timeout=float(os.getenv(ENV_LLM_TIMEOUT, str(DEFAULT_LLM_TIMEOUT))), + # Per-operation LLM config (None = use default) + retain_llm_provider=os.getenv(ENV_RETAIN_LLM_PROVIDER) or None, + retain_llm_api_key=os.getenv(ENV_RETAIN_LLM_API_KEY) or None, + retain_llm_model=os.getenv(ENV_RETAIN_LLM_MODEL) or None, + retain_llm_base_url=os.getenv(ENV_RETAIN_LLM_BASE_URL) or None, + reflect_llm_provider=os.getenv(ENV_REFLECT_LLM_PROVIDER) or None, + reflect_llm_api_key=os.getenv(ENV_REFLECT_LLM_API_KEY) or None, + reflect_llm_model=os.getenv(ENV_REFLECT_LLM_MODEL) or None, + reflect_llm_base_url=os.getenv(ENV_REFLECT_LLM_BASE_URL) or None, # Embeddings embeddings_provider=os.getenv(ENV_EMBEDDINGS_PROVIDER, DEFAULT_EMBEDDINGS_PROVIDER), embeddings_local_model=os.getenv(ENV_EMBEDDINGS_LOCAL_MODEL, DEFAULT_EMBEDDINGS_LOCAL_MODEL), @@ -251,6 +282,14 @@ class HindsightConfig: """Log the current configuration (without sensitive values).""" logger.info(f"Database: {self.database_url}") logger.info(f"LLM: provider={self.llm_provider}, model={self.llm_model}") + if self.retain_llm_provider or self.retain_llm_model: + retain_provider = self.retain_llm_provider or self.llm_provider + retain_model = self.retain_llm_model or self.llm_model + logger.info(f"LLM (retain): provider={retain_provider}, model={retain_model}") + if self.reflect_llm_provider or self.reflect_llm_model: + reflect_provider = self.reflect_llm_provider or self.llm_provider + reflect_model = self.reflect_llm_model or self.llm_model + logger.info(f"LLM (reflect): provider={reflect_provider}, model={reflect_model}") logger.info(f"Embeddings: provider={self.embeddings_provider}") logger.info(f"Reranker: provider={self.reranker_provider}") logger.info(f"Graph retriever: {self.graph_retriever}") diff --git a/hindsight-api/hindsight_api/engine/llm_wrapper.py b/hindsight-api/hindsight_api/engine/llm_wrapper.py index 996ae7ab..701a1719 100644 --- a/hindsight-api/hindsight_api/engine/llm_wrapper.py +++ b/hindsight-api/hindsight_api/engine/llm_wrapper.py @@ -88,10 +88,14 @@ class LLMProvider: self.groq_service_tier = groq_service_tier or os.getenv(ENV_LLM_GROQ_SERVICE_TIER, "auto") # Validate provider - valid_providers = ["openai", "groq", "ollama", "gemini", "anthropic", "lmstudio"] + valid_providers = ["openai", "groq", "ollama", "gemini", "anthropic", "lmstudio", "mock"] if self.provider not in valid_providers: raise ValueError(f"Invalid LLM provider: {self.provider}. Must be one of: {', '.join(valid_providers)}") + # Mock provider tracking (for testing) + self._mock_calls: list[dict] = [] + self._mock_response: Any = None + # Set default base URLs if not self.base_url: if self.provider == "groq": @@ -101,8 +105,8 @@ class LLMProvider: elif self.provider == "lmstudio": self.base_url = "http://localhost:1234/v1" - # Validate API key (not needed for ollama or lmstudio) - if self.provider not in ("ollama", "lmstudio") and not self.api_key: + # Validate API key (not needed for ollama, lmstudio, or mock) + if self.provider not in ("ollama", "lmstudio", "mock") and not self.api_key: raise ValueError(f"API key not found for {self.provider}") # Get timeout config (set HINDSIGHT_API_LLM_TIMEOUT for local LLMs that need longer timeouts) @@ -113,7 +117,10 @@ class LLMProvider: self._gemini_client = None self._anthropic_client = None - if self.provider == "gemini": + if self.provider == "mock": + # Mock provider - no client needed + pass + elif self.provider == "gemini": self._gemini_client = genai.Client(api_key=self.api_key) elif self.provider == "anthropic": from anthropic import AsyncAnthropic @@ -205,6 +212,15 @@ class LLMProvider: async with _global_llm_semaphore: start_time = time.time() + # Handle Mock provider (for testing) + if self.provider == "mock": + return await self._call_mock( + messages, + response_format, + scope, + return_usage, + ) + # Handle Gemini provider separately if self.provider == "gemini": return await self._call_gemini( @@ -954,6 +970,61 @@ class LLMProvider: raise last_exception raise RuntimeError("Gemini call failed after all retries") + async def _call_mock( + self, + messages: list[dict[str, str]], + response_format: Any | None, + scope: str, + return_usage: bool, + ) -> Any: + """ + Handle mock provider calls for testing. + + Records the call and returns a configurable mock response. + """ + # Record the call for test verification + call_record = { + "provider": self.provider, + "model": self.model, + "messages": messages, + "response_format": response_format.__name__ + if response_format and hasattr(response_format, "__name__") + else str(response_format), + "scope": scope, + } + self._mock_calls.append(call_record) + logger.debug(f"Mock LLM call recorded: scope={scope}, model={self.model}") + + # Return mock response + if self._mock_response is not None: + result = self._mock_response + elif response_format is not None: + # Try to create a minimal valid instance of the response format + try: + # For Pydantic models, try to create with minimal valid data + result = {"mock": True} + except Exception: + result = {"mock": True} + else: + result = "mock response" + + if return_usage: + token_usage = TokenUsage(input_tokens=10, output_tokens=5, total_tokens=15) + return result, token_usage + return result + + def set_mock_response(self, response: Any) -> None: + """Set the response to return from mock calls.""" + self._mock_response = response + + def get_mock_calls(self) -> list[dict]: + """Get the list of recorded mock calls.""" + return self._mock_calls + + def clear_mock_calls(self) -> None: + """Clear the recorded mock calls.""" + self._mock_calls = [] + @classmethod def for_memory(cls) -> "LLMProvider": """Create provider for memory operations from environment variables.""" diff --git a/hindsight-api/hindsight_api/engine/memory_engine.py b/hindsight-api/hindsight_api/engine/memory_engine.py index bce48cdd..684c5877 100644 --- a/hindsight-api/hindsight_api/engine/memory_engine.py +++ b/hindsight-api/hindsight_api/engine/memory_engine.py @@ -203,6 +203,15 @@ class MemoryEngine(MemoryEngineInterface): memory_llm_api_key: str | None = None, memory_llm_model: str | None = None, memory_llm_base_url: str | None = None, + # Per-operation LLM config (optional, falls back to memory_llm_* params) + retain_llm_provider: str | None = None, + retain_llm_api_key: str | None = None, + retain_llm_model: str | None = None, + retain_llm_base_url: str | None = None, + reflect_llm_provider: str | None = None, + reflect_llm_api_key: str | None = None, + reflect_llm_model: str | None = None, + reflect_llm_base_url: str | None = None, embeddings: Embeddings | None = None, cross_encoder: CrossEncoderModel | None = None, query_analyzer: QueryAnalyzer | None = None, @@ -228,6 +237,14 @@ class MemoryEngine(MemoryEngineInterface): memory_llm_api_key: API key for the LLM provider. Defaults to HINDSIGHT_API_LLM_API_KEY env var. memory_llm_model: Model name. Defaults to HINDSIGHT_API_LLM_MODEL env var. memory_llm_base_url: Base URL for the LLM API. Defaults based on provider. + retain_llm_provider: LLM provider for retain operations. Falls back to memory_llm_provider. + retain_llm_api_key: API key for retain LLM. Falls back to memory_llm_api_key. + retain_llm_model: Model for retain operations. Falls back to memory_llm_model. + retain_llm_base_url: Base URL for retain LLM. Falls back to memory_llm_base_url. + reflect_llm_provider: LLM provider for reflect operations. Falls back to memory_llm_provider. + reflect_llm_api_key: API key for reflect LLM. Falls back to memory_llm_api_key. + reflect_llm_model: Model for reflect operations. Falls back to memory_llm_model. + reflect_llm_base_url: Base URL for reflect LLM. Falls back to memory_llm_base_url. embeddings: Embeddings implementation. If not provided, created from env vars. cross_encoder: Cross-encoder model. If not provided, created from env vars. query_analyzer: Query analyzer implementation. If not provided, uses DateparserQueryAnalyzer. @@ -260,8 +277,8 @@ class MemoryEngine(MemoryEngineInterface): db_url = db_url or config.database_url memory_llm_provider = memory_llm_provider or config.llm_provider memory_llm_api_key = memory_llm_api_key or config.llm_api_key - # Ollama doesn't require an API key - if not memory_llm_api_key and memory_llm_provider != "ollama": + # Ollama and mock don't require an API key + if not memory_llm_api_key and memory_llm_provider not in ("ollama", "mock"): raise ValueError("LLM API key is required. Set HINDSIGHT_API_LLM_API_KEY environment variable.") memory_llm_model = memory_llm_model or config.llm_model memory_llm_base_url = memory_llm_base_url or config.get_llm_base_url() or None @@ -310,7 +327,7 @@ class MemoryEngine(MemoryEngineInterface): self.query_analyzer = DateparserQueryAnalyzer() - # Initialize LLM configuration + # Initialize LLM configuration (default, used as fallback) self._llm_config = LLMConfig( provider=memory_llm_provider, api_key=memory_llm_api_key, @@ -322,6 +339,49 @@ class MemoryEngine(MemoryEngineInterface): self._llm_client = self._llm_config._client self._llm_model = self._llm_config.model + # Initialize per-operation LLM configs (fall back to default if not specified) + # Retain LLM config - for fact extraction (benefits from strong structured output) + retain_provider = retain_llm_provider or config.retain_llm_provider or memory_llm_provider + retain_api_key = retain_llm_api_key or config.retain_llm_api_key or memory_llm_api_key + retain_model = retain_llm_model or config.retain_llm_model or memory_llm_model + retain_base_url = retain_llm_base_url or config.retain_llm_base_url or memory_llm_base_url + # Apply provider-specific base URL defaults for retain + if retain_base_url is None: + if retain_provider.lower() == "groq": + retain_base_url = "https://api.groq.com/openai/v1" + elif retain_provider.lower() == "ollama": + retain_base_url = "http://localhost:11434/v1" + else: + retain_base_url = "" + + self._retain_llm_config = LLMConfig( + provider=retain_provider, + api_key=retain_api_key, + base_url=retain_base_url, + model=retain_model, + ) + + # Reflect LLM config - for think/observe operations (can use lighter models) + reflect_provider = reflect_llm_provider or config.reflect_llm_provider or memory_llm_provider + reflect_api_key = reflect_llm_api_key or config.reflect_llm_api_key or memory_llm_api_key + reflect_model = reflect_llm_model or config.reflect_llm_model or memory_llm_model + reflect_base_url = reflect_llm_base_url or config.reflect_llm_base_url or memory_llm_base_url + # Apply provider-specific base URL defaults for reflect + if reflect_base_url is None: + if reflect_provider.lower() == "groq": + reflect_base_url = "https://api.groq.com/openai/v1" + elif reflect_provider.lower() == "ollama": + reflect_base_url = "http://localhost:11434/v1" + else: + reflect_base_url = "" + + self._reflect_llm_config = LLMConfig( + provider=reflect_provider, + api_key=reflect_api_key, + base_url=reflect_base_url, + model=reflect_model, + ) + # Initialize cross-encoder reranker (cached for performance) self._cross_encoder_reranker = CrossEncoderReranker(cross_encoder=cross_encoder) @@ -609,9 +669,27 @@ class MemoryEngine(MemoryEngineInterface): await loop.run_in_executor(None, self.query_analyzer.load) async def verify_llm(): - """Verify LLM connection is working.""" + """Verify LLM connections are working for all unique configs.""" if not self._skip_llm_verification: + # Verify default config await self._llm_config.verify_connection() + # Verify retain config if different from default + retain_is_different = ( + self._retain_llm_config.provider != self._llm_config.provider + or self._retain_llm_config.model != self._llm_config.model + ) + if retain_is_different: + await self._retain_llm_config.verify_connection() + # Verify reflect config if different from default and retain + reflect_is_different = ( + self._reflect_llm_config.provider != self._llm_config.provider + or self._reflect_llm_config.model != self._llm_config.model + ) and ( + self._reflect_llm_config.provider != self._retain_llm_config.provider + or self._reflect_llm_config.model != self._retain_llm_config.model + ) + if reflect_is_different: + await self._reflect_llm_config.verify_connection() # Build list of initialization tasks init_tasks = [ @@ -1175,7 +1253,7 @@ class MemoryEngine(MemoryEngineInterface): return await orchestrator.retain_batch( pool=pool, embeddings_model=self.embeddings, - llm_config=self._llm_config, + llm_config=self._retain_llm_config, entity_resolver=self.entity_resolver, task_backend=self._task_backend, format_date_fn=self._format_readable_date, @@ -2822,7 +2900,7 @@ Guidelines: - Small changes in confidence are normal; large jumps should be rare""" try: - result = await self._llm_config.call( + result = await self._reflect_llm_config.call( messages=[ {"role": "system", "content": "You evaluate and update opinions based on new information."}, {"role": "user", "content": evaluation_prompt}, @@ -2932,7 +3010,7 @@ Guidelines: return # Use cached LLM config - if self._llm_config is None: + if self._reflect_llm_config is None: logger.error("[REINFORCE] LLM config not available, skipping opinion reinforcement") return @@ -3077,7 +3155,9 @@ Guidelines: """ await self._authenticate_tenant(request_context) pool = await self._get_pool() - return await bank_utils.merge_bank_background(pool, self._llm_config, bank_id, new_info, update_disposition) + return await bank_utils.merge_bank_background( + pool, self._reflect_llm_config, bank_id, new_info, update_disposition + ) async def list_banks( self, @@ -3137,7 +3217,7 @@ Guidelines: - structured_output: Optional dict if response_schema was provided """ # Use cached LLM config - if self._llm_config is None: + if self._reflect_llm_config is None: raise ValueError("Memory LLM API key not set. Set HINDSIGHT_API_LLM_API_KEY environment variable.") # Authenticate tenant and set schema in context (for fq_table()) @@ -3232,7 +3312,7 @@ Guidelines: response_format = JsonSchemaWrapper(response_schema) llm_start = time.time() - llm_result, usage = await self._llm_config.call( + llm_result, usage = await self._reflect_llm_config.call( messages=messages, scope="memory_reflect", max_completion_tokens=max_tokens, @@ -3318,7 +3398,9 @@ Guidelines: """ try: # Extract opinions from the answer - new_opinions = await think_utils.extract_opinions_from_text(self._llm_config, text=answer_text, query=query) + new_opinions = await think_utils.extract_opinions_from_text( + self._reflect_llm_config, text=answer_text, query=query + ) # Store new opinions if new_opinions: @@ -3569,7 +3651,9 @@ Guidelines: ) # Step 3: Extract observations using LLM (no personality) - observations = await observation_utils.extract_observations_from_facts(self._llm_config, entity_name, facts) + observations = await observation_utils.extract_observations_from_facts( + self._reflect_llm_config, entity_name, facts + ) if not observations: return [] diff --git a/hindsight-api/hindsight_api/main.py b/hindsight-api/hindsight_api/main.py index 56d15767..b7026c1a 100644 --- a/hindsight-api/hindsight_api/main.py +++ b/hindsight-api/hindsight_api/main.py @@ -171,6 +171,14 @@ def main(): llm_base_url=config.llm_base_url, llm_max_concurrent=config.llm_max_concurrent, llm_timeout=config.llm_timeout, + retain_llm_provider=config.retain_llm_provider, + retain_llm_api_key=config.retain_llm_api_key, + retain_llm_model=config.retain_llm_model, + retain_llm_base_url=config.retain_llm_base_url, + reflect_llm_provider=config.reflect_llm_provider, + reflect_llm_api_key=config.reflect_llm_api_key, + reflect_llm_model=config.reflect_llm_model, + reflect_llm_base_url=config.reflect_llm_base_url, embeddings_provider=config.embeddings_provider, embeddings_local_model=config.embeddings_local_model, embeddings_tei_url=config.embeddings_tei_url, diff --git a/hindsight-api/tests/test_per_operation_llm_config.py b/hindsight-api/tests/test_per_operation_llm_config.py new file mode 100644 index 00000000..c1a37517 --- /dev/null +++ b/hindsight-api/tests/test_per_operation_llm_config.py @@ -0,0 +1,277 @@ +""" +Tests for per-operation LLM configuration. + +Verifies that retain and reflect operations use their respective LLM configs. +""" + +import os + +import pytest + + +@pytest.fixture(autouse=True) +def setup_test_env(): + """Set up environment for each test, restoring original values after.""" + from hindsight_api.config import clear_config_cache + + # Save original environment values + env_vars_to_set = { + "HINDSIGHT_API_SKIP_LLM_VERIFICATION": "true", + "HINDSIGHT_API_LAZY_RERANKER": "true", + "HINDSIGHT_API_LLM_PROVIDER": "mock", + "HINDSIGHT_API_LLM_MODEL": "default-model", + "HINDSIGHT_API_RETAIN_LLM_PROVIDER": "mock", + "HINDSIGHT_API_RETAIN_LLM_MODEL": "retain-model", + "HINDSIGHT_API_REFLECT_LLM_PROVIDER": "mock", + "HINDSIGHT_API_REFLECT_LLM_MODEL": "reflect-model", + } + + # Save original values + original_values = {} + for key in env_vars_to_set: + original_values[key] = os.environ.get(key) + + # Set test values + for key, value in env_vars_to_set.items(): + os.environ[key] = value + + clear_config_cache() + + yield + + # Restore original environment + for key, original_value in original_values.items(): + if original_value is None: + os.environ.pop(key, None) + else: + os.environ[key] = original_value + + clear_config_cache() + + +class TestPerOperationLLMConfig: + """Test that per-operation LLM configs are correctly applied.""" + + def test_config_loads_per_operation_settings(self): + """Test that config correctly loads per-operation LLM settings.""" + from hindsight_api.config import get_config + + config = get_config() + + # Default config + assert config.llm_provider == "mock" + assert config.llm_model == "default-model" + + # Retain config + assert config.retain_llm_provider == "mock" + assert config.retain_llm_model == "retain-model" + + # Reflect config + assert config.reflect_llm_provider == "mock" + assert config.reflect_llm_model == "reflect-model" + + def test_memory_engine_creates_separate_llm_configs(self): + """Test that MemoryEngine creates separate LLM configs for each operation.""" + from hindsight_api import MemoryEngine + + engine = MemoryEngine( + skip_llm_verification=True, + lazy_reranker=True, + ) + + # Verify default config + assert engine._llm_config.provider == "mock" + assert engine._llm_config.model == "default-model" + + # Verify retain config + assert engine._retain_llm_config.provider == "mock" + assert engine._retain_llm_config.model == "retain-model" + + # Verify reflect config + assert engine._reflect_llm_config.provider == "mock" + assert engine._reflect_llm_config.model == "reflect-model" + + def test_memory_engine_with_explicit_params(self): + """Test that explicit params override env config.""" + from hindsight_api import MemoryEngine + + engine = MemoryEngine( + memory_llm_provider="mock", + memory_llm_model="explicit-default", + retain_llm_provider="mock", + retain_llm_model="explicit-retain", + reflect_llm_provider="mock", + reflect_llm_model="explicit-reflect", + skip_llm_verification=True, + lazy_reranker=True, + ) + + assert engine._llm_config.model == "explicit-default" + assert engine._retain_llm_config.model == "explicit-retain" + assert engine._reflect_llm_config.model == "explicit-reflect" + + def test_memory_engine_fallback_when_no_per_operation_config(self): + """Test that per-operation configs fall back to default when not set.""" + from hindsight_api.config import clear_config_cache as clear_cache + + # Temporarily clear per-operation env vars + retain_provider = os.environ.pop("HINDSIGHT_API_RETAIN_LLM_PROVIDER", None) + retain_model = os.environ.pop("HINDSIGHT_API_RETAIN_LLM_MODEL", None) + reflect_provider = os.environ.pop("HINDSIGHT_API_REFLECT_LLM_PROVIDER", None) + reflect_model = os.environ.pop("HINDSIGHT_API_REFLECT_LLM_MODEL", None) + + try: + clear_cache() + from hindsight_api import MemoryEngine + + engine = MemoryEngine( + skip_llm_verification=True, + lazy_reranker=True, + ) + + # All should fall back to default + assert engine._llm_config.model == "default-model" + assert engine._retain_llm_config.model == "default-model" + assert engine._reflect_llm_config.model == "default-model" + finally: + # Restore env vars + if retain_provider: + os.environ["HINDSIGHT_API_RETAIN_LLM_PROVIDER"] = retain_provider + if retain_model: + os.environ["HINDSIGHT_API_RETAIN_LLM_MODEL"] = retain_model + if reflect_provider: + os.environ["HINDSIGHT_API_REFLECT_LLM_PROVIDER"] = reflect_provider + if reflect_model: + os.environ["HINDSIGHT_API_REFLECT_LLM_MODEL"] = reflect_model + clear_cache() + + +class TestMockLLMProvider: + """Test the mock LLM provider functionality.""" + + def test_mock_provider_records_calls(self): + """Test that mock provider records calls.""" + from hindsight_api.engine.llm_wrapper import LLMProvider + + provider = LLMProvider( + provider="mock", + api_key="", + base_url="", + model="test-model", + ) + + import asyncio + + async def make_call(): + return await provider.call( + messages=[{"role": "user", "content": "test"}], + scope="test_scope", + ) + + result = asyncio.get_event_loop().run_until_complete(make_call()) + + # Verify call was recorded + calls = provider.get_mock_calls() + assert len(calls) == 1 + assert calls[0]["model"] == "test-model" + assert calls[0]["scope"] == "test_scope" + assert calls[0]["messages"] == [{"role": "user", "content": "test"}] + + def test_mock_provider_returns_custom_response(self): + """Test that mock provider can return custom responses.""" + from hindsight_api.engine.llm_wrapper import LLMProvider + + provider = LLMProvider( + provider="mock", + api_key="", + base_url="", + model="test-model", + ) + + provider.set_mock_response({"custom": "response"}) + + import asyncio + + async def make_call(): + return await provider.call( + messages=[{"role": "user", "content": "test"}], + ) + + result = asyncio.get_event_loop().run_until_complete(make_call()) + assert result == {"custom": "response"} + + def test_mock_provider_returns_usage_when_requested(self): + """Test that mock provider returns token usage.""" + from hindsight_api.engine.llm_wrapper import LLMProvider + + provider = LLMProvider( + provider="mock", + api_key="", + base_url="", + model="test-model", + ) + + import asyncio + + async def make_call(): + return await provider.call( + messages=[{"role": "user", "content": "test"}], + return_usage=True, + ) + + result, usage = asyncio.get_event_loop().run_until_complete(make_call()) + assert usage.input_tokens == 10 + assert usage.output_tokens == 5 + assert usage.total_tokens == 15 + + +class TestRetainUsesRetainLLMConfig: + """Test that retain operations use the retain LLM config.""" + + def test_retain_llm_config_is_passed_to_orchestrator(self): + """Verify retain operation is configured to use _retain_llm_config.""" + from hindsight_api import MemoryEngine + + engine = MemoryEngine( + memory_llm_provider="mock", + memory_llm_model="default-model", + retain_llm_provider="mock", + retain_llm_model="retain-specific-model", + reflect_llm_provider="mock", + reflect_llm_model="reflect-specific-model", + skip_llm_verification=True, + lazy_reranker=True, + ) + + # Verify the retain LLM config is set correctly + assert engine._retain_llm_config.model == "retain-specific-model" + assert engine._retain_llm_config.provider == "mock" + + # Verify it's different from the reflect config + assert engine._retain_llm_config.model != engine._reflect_llm_config.model + + +class TestReflectUsesReflectLLMConfig: + """Test that reflect operations use the reflect LLM config.""" + + def test_reflect_llm_config_is_set_correctly(self): + """Verify reflect/think operation is configured to use _reflect_llm_config.""" + from hindsight_api import MemoryEngine + + engine = MemoryEngine( + memory_llm_provider="mock", + memory_llm_model="default-model", + retain_llm_provider="mock", + retain_llm_model="retain-specific-model", + reflect_llm_provider="mock", + reflect_llm_model="reflect-specific-model", + skip_llm_verification=True, + lazy_reranker=True, + ) + + # Verify the reflect LLM config is set correctly + assert engine._reflect_llm_config.model == "reflect-specific-model" + assert engine._reflect_llm_config.provider == "mock" + + # Verify it's different from the retain config + assert engine._reflect_llm_config.model != engine._retain_llm_config.model diff --git a/hindsight-docs/docs/developer/configuration.md b/hindsight-docs/docs/developer/configuration.md index a1e5d017..33441626 100644 --- a/hindsight-docs/docs/developer/configuration.md +++ b/hindsight-docs/docs/developer/configuration.md @@ -86,6 +86,44 @@ export HINDSIGHT_API_LLM_API_KEY=your-api-key export HINDSIGHT_API_LLM_MODEL=your-model-name ``` +### Per-Operation LLM Configuration + +Different memory operations have different requirements. **Retain** (fact extraction) benefits from models with strong structured output capabilities, while **Reflect** (reasoning/response generation) can use lighter, faster models. Configure separate LLM models for each operation to optimize for cost and performance. + +| Variable | Description | Default | +|----------|-------------|---------| +| `HINDSIGHT_API_RETAIN_LLM_PROVIDER` | LLM provider for retain operations | Falls back to `HINDSIGHT_API_LLM_PROVIDER` | +| `HINDSIGHT_API_RETAIN_LLM_API_KEY` | API key for retain LLM | Falls back to `HINDSIGHT_API_LLM_API_KEY` | +| `HINDSIGHT_API_RETAIN_LLM_MODEL` | Model for retain operations | Falls back to `HINDSIGHT_API_LLM_MODEL` | +| `HINDSIGHT_API_RETAIN_LLM_BASE_URL` | Base URL for retain LLM | Falls back to `HINDSIGHT_API_LLM_BASE_URL` | +| `HINDSIGHT_API_REFLECT_LLM_PROVIDER` | LLM provider for reflect operations | Falls back to `HINDSIGHT_API_LLM_PROVIDER` | +| `HINDSIGHT_API_REFLECT_LLM_API_KEY` | API key for reflect LLM | Falls back to `HINDSIGHT_API_LLM_API_KEY` | +| `HINDSIGHT_API_REFLECT_LLM_MODEL` | Model for reflect operations | Falls back to `HINDSIGHT_API_LLM_MODEL` | +| `HINDSIGHT_API_REFLECT_LLM_BASE_URL` | Base URL for reflect LLM | Falls back to `HINDSIGHT_API_LLM_BASE_URL` | + +:::tip When to Use Per-Operation Config +- **Retain**: Use models with strong structured output (e.g., GPT-4o, Claude) for accurate fact extraction +- **Reflect**: Use faster/cheaper models (e.g., GPT-4o-mini, Groq) for reasoning and response generation +- **Recall**: Does not use LLM (pure retrieval), so no configuration needed +::: + +**Example: Separate Models for Retain and Reflect** + +```bash +# Default LLM (used as fallback) +export HINDSIGHT_API_LLM_PROVIDER=openai +export HINDSIGHT_API_LLM_API_KEY=sk-xxxxxxxxxxxx +export HINDSIGHT_API_LLM_MODEL=gpt-4o + +# Use GPT-4o for retain (strong structured output) +export HINDSIGHT_API_RETAIN_LLM_MODEL=gpt-4o + +# Use faster/cheaper model for reflect +export HINDSIGHT_API_REFLECT_LLM_PROVIDER=groq +export HINDSIGHT_API_REFLECT_LLM_API_KEY=gsk_xxxxxxxxxxxx +export HINDSIGHT_API_REFLECT_LLM_MODEL=llama-3.3-70b-versatile +``` + ### Embeddings | Variable | Description | Default |