diff --git a/hindsight-api/hindsight_api/engine/providers/claude_code_llm.py b/hindsight-api/hindsight_api/engine/providers/claude_code_llm.py index bf8f24fc..d2124ce4 100644 --- a/hindsight-api/hindsight_api/engine/providers/claude_code_llm.py +++ b/hindsight-api/hindsight_api/engine/providers/claude_code_llm.py @@ -291,61 +291,202 @@ class ClaudeCodeLLM(LLMInterface): tool_choice: str | dict[str, Any] = "auto", ) -> LLMToolCallResult: """ - Make an LLM API call with tool/function calling support. + Make an LLM API call with tool/function calling support using Claude Agent SDK. - Note: This is a simplified implementation. Full tool support would require - integrating with Claude Agent SDK's tool system. + This implementation uses ClaudeSDKClient (not query()) because custom tools via + SDK MCP servers are only supported with the client. Tools are converted from OpenAI + format to SDK MCP tools, and tool names are formatted as mcp__hindsight_tools__{name}. Args: messages: List of message dicts. Can include tool results with role='tool'. tools: List of tool definitions in OpenAI format. - max_completion_tokens: Maximum tokens in response. - temperature: Sampling temperature. + max_completion_tokens: Maximum tokens in response (not used by Claude Agent SDK). + temperature: Sampling temperature (not used by Claude Agent SDK). scope: Scope identifier for tracking. max_retries: Maximum retry attempts. initial_backoff: Initial backoff time in seconds. max_backoff: Maximum backoff time in seconds. - tool_choice: How to choose tools - "auto", "none", "required", or specific function. + tool_choice: How to choose tools (not used by Claude Agent SDK). Returns: LLMToolCallResult with content and/or tool_calls. """ - # For now, use regular call without tools - # Full implementation would require mapping OpenAI tool format to Claude Agent SDK tools - logger.warning( - "Claude Code provider does not fully support tool calling yet. Falling back to regular text completion." + from claude_agent_sdk import ( + AssistantMessage, + ClaudeAgentOptions, + ClaudeSDKClient, + SdkMcpTool, + TextBlock, + ToolUseBlock, + create_sdk_mcp_server, ) - result = await self.call( - messages=messages, - response_format=None, - max_completion_tokens=max_completion_tokens, - temperature=temperature, - scope=scope, - max_retries=max_retries, - initial_backoff=initial_backoff, - max_backoff=max_backoff, - return_usage=True, + start_time = time.time() + + # Convert OpenAI tool format to Claude Agent SDK SdkMcpTool format + sdk_tools: list[SdkMcpTool] = [] + tool_names: list[str] = [] + + for tool in tools: + func = tool.get("function", {}) + tool_name = func.get("name", "") + tool_description = func.get("description", "") + parameters = func.get("parameters", {}) + + # Create a handler with proper closure to avoid transport issues + def make_handler(name: str): + async def handler(args: dict[str, Any]) -> dict[str, Any]: + # Return immediately with success - tool execution happens externally + return { + "content": [ + { + "type": "text", + "text": f"[Tool {name} called successfully]", + } + ] + } + + return handler + + sdk_tools.append( + SdkMcpTool( + name=tool_name, + description=tool_description, + input_schema=parameters, + handler=make_handler(tool_name), + ) + ) + tool_names.append(tool_name) + + # Create an MCP server with the tools + mcp_server = create_sdk_mcp_server( + name="hindsight_tools", + version="1.0.0", + tools=sdk_tools if sdk_tools else None, ) - if isinstance(result, tuple): - text, usage = result - return LLMToolCallResult( - content=text, - tool_calls=[], - finish_reason="stop", - input_tokens=usage.input_tokens, - output_tokens=usage.output_tokens, - ) - else: - # Fallback if return_usage didn't work as expected - return LLMToolCallResult( - content=str(result), - tool_calls=[], - finish_reason="stop", - input_tokens=0, - output_tokens=0, - ) + # Build system prompt and user content from messages + system_prompt = "" + user_content = "" + + for msg in messages: + role = msg.get("role", "user") + content = msg.get("content", "") + + if role == "system": + system_prompt += ("\n\n" + content) if system_prompt else content + elif role == "user": + user_content += ("\n\n" + content) if user_content else content + elif role == "assistant": + # Include previous assistant messages as context + user_content += f"\n\n[Previous assistant response: {content}]" + elif role == "tool": + # Tool results are already in tool_results_map, append to user context + tool_call_id = msg.get("tool_call_id", "") + user_content += f"\n\n[Tool result for {tool_call_id}: {content}]" + + # Format tool names for SDK MCP servers: mcp__{server_name}__{tool_name} + # This is required by the Claude Agent SDK for MCP server tools + allowed_tool_names = [f"mcp__hindsight_tools__{name}" for name in tool_names] + + # Configure SDK options with MCP server + options = ClaudeAgentOptions( + system_prompt=system_prompt if system_prompt else None, + max_turns=1, # Single-turn for API-style interactions + mcp_servers={"hindsight_tools": mcp_server} if sdk_tools else {}, + allowed_tools=allowed_tool_names if allowed_tool_names else [], + ) + + # Call Claude Agent SDK with retry logic + last_exception = None + for attempt in range(max_retries + 1): + try: + full_text = "" + tool_calls: list[LLMToolCall] = [] + + # Use ClaudeSDKClient for tool calling support + # Note: query() does NOT support custom tools, only ClaudeSDKClient does + async with ClaudeSDKClient(options=options) as client: + # Send the query + await client.query(user_content) + + # Receive response + async for message in client.receive_response(): + if isinstance(message, AssistantMessage): + for block in message.content: + if isinstance(block, TextBlock): + full_text += block.text + elif isinstance(block, ToolUseBlock): + # SDK returns tool names with MCP prefix (mcp__hindsight_tools__{name}) + # Strip the prefix to return original tool name expected by caller + tool_name = block.name + if tool_name.startswith("mcp__hindsight_tools__"): + tool_name = tool_name.replace("mcp__hindsight_tools__", "", 1) + + tool_calls.append( + LLMToolCall( + id=block.id, + name=tool_name, + arguments=block.input, + ) + ) + + # Record metrics + duration = time.time() - start_time + metrics = get_metrics_collector() + + # Estimate token usage (Claude Agent SDK doesn't report exact counts) + estimated_input = sum(len(m.get("content", "")) for m in messages) // 4 + estimated_output = len(full_text) // 4 + + metrics.record_llm_call( + provider=self.provider, + model=self.model, + scope=scope, + duration=duration, + input_tokens=estimated_input, + output_tokens=estimated_output, + success=True, + ) + + # Log slow calls + if duration > 10.0: + logger.info( + f"slow llm call: scope={scope}, model={self.provider}/{self.model}, time={duration:.3f}s" + ) + + return LLMToolCallResult( + content=full_text if full_text else None, + tool_calls=tool_calls, + finish_reason="tool_calls" if tool_calls else "stop", + input_tokens=estimated_input, + output_tokens=estimated_output, + ) + + except Exception as e: + last_exception = e + + # Check for authentication errors + error_str = str(e).lower() + if "auth" in error_str or "login" in error_str or "credential" in error_str: + logger.error(f"Claude Code authentication error: {e}") + raise RuntimeError( + f"Claude Code authentication failed: {e}\n\n" + "Run 'claude auth login' to authenticate with Claude Pro/Max." + ) from e + + if attempt < max_retries: + backoff = min(initial_backoff * (2**attempt), max_backoff) + logger.warning(f"Claude Code tool call error (attempt {attempt + 1}/{max_retries + 1}): {e}") + await asyncio.sleep(backoff) + continue + else: + logger.error(f"Claude Code tool call error after {max_retries + 1} attempts: {e}") + raise + + if last_exception: + raise last_exception + raise RuntimeError("Claude Code tool call failed after all retries") async def cleanup(self) -> None: """Clean up resources (no HTTP client to close for Claude Agent SDK).""" diff --git a/hindsight-api/hindsight_api/engine/providers/codex_llm.py b/hindsight-api/hindsight_api/engine/providers/codex_llm.py index 7775e51d..d3bf2925 100644 --- a/hindsight-api/hindsight_api/engine/providers/codex_llm.py +++ b/hindsight-api/hindsight_api/engine/providers/codex_llm.py @@ -177,6 +177,9 @@ class CodexLLM(LLMInterface): schema_msg = f"\n\nYou must respond with valid JSON matching this schema:\n{json.dumps(schema, indent=2)}" system_instruction += schema_msg + # gpt-5.2-codex only supports "detailed" reasoning summary + reasoning_summary = "detailed" if "5.2" in self.model else self.reasoning_summary + # Build Codex request payload payload = { "model": self.model, @@ -192,7 +195,7 @@ class CodexLLM(LLMInterface): "tools": [], "tool_choice": "auto", "parallel_tool_calls": True, - "reasoning": {"summary": self.reasoning_summary}, + "reasoning": {"summary": reasoning_summary}, "store": False, # Codex uses stateless mode "stream": True, # SSE streaming "include": ["reasoning.encrypted_content"], @@ -283,13 +286,20 @@ class CodexLLM(LLMInterface): "Run 'codex auth login' to re-authenticate." ) from e + # Log the actual error message from the API + error_detail = e.response.text[:500] if hasattr(e.response, "text") else str(e) + if attempt < max_retries: backoff = min(initial_backoff * (2**attempt), max_backoff) - logger.warning(f"Codex HTTP error {status_code} (attempt {attempt + 1}/{max_retries + 1})") + logger.warning( + f"Codex HTTP error {status_code} (attempt {attempt + 1}/{max_retries + 1}): {error_detail}" + ) await asyncio.sleep(backoff) continue else: - logger.error(f"Codex HTTP error after {max_retries + 1} attempts: {e}") + logger.error( + f"Codex HTTP error after {max_retries + 1} attempts: Status {status_code}, Detail: {error_detail}" + ) raise except httpx.RequestError as e: @@ -379,8 +389,22 @@ class CodexLLM(LLMInterface): """ Make API call with tool calling support. - Note: This is a basic implementation. Full tool calling support for Codex - may require additional SSE event parsing. + Parses Codex SSE stream to extract tool calls from response.output_item.done events. + Tools are converted from OpenAI format to Codex format (flat structure at top level). + + Args: + messages: List of message dicts. Can include tool results with role='tool'. + tools: List of tool definitions in OpenAI format. + max_completion_tokens: Maximum tokens in response. + temperature: Sampling temperature. + scope: Scope identifier for tracking. + max_retries: Maximum retry attempts. + initial_backoff: Initial backoff time in seconds. + max_backoff: Maximum backoff time in seconds. + tool_choice: How to choose tools - "auto", "none", "required", or specific function. + + Returns: + LLMToolCallResult with content and/or tool_calls. """ start_time = time.time() @@ -413,20 +437,22 @@ class CodexLLM(LLMInterface): ) # Convert tools to Codex format + # Codex expects tools with type and name/description/parameters at top level codex_tools = [] for tool in tools: func = tool.get("function", {}) codex_tools.append( { "type": "function", - "function": { - "name": func.get("name", ""), - "description": func.get("description", ""), - "parameters": func.get("parameters", {}), - }, + "name": func.get("name", ""), + "description": func.get("description", ""), + "parameters": func.get("parameters", {}), } ) + # gpt-5.2-codex only supports "detailed" reasoning summary + reasoning_summary = "detailed" if "5.2" in self.model else self.reasoning_summary + payload = { "model": self.model, "instructions": system_instruction, @@ -434,7 +460,7 @@ class CodexLLM(LLMInterface): "tools": codex_tools, "tool_choice": tool_choice, "parallel_tool_calls": True, - "reasoning": {"summary": self.reasoning_summary}, + "reasoning": {"summary": reasoning_summary}, "store": False, "stream": True, "include": ["reasoning.encrypted_content"], @@ -451,8 +477,16 @@ class CodexLLM(LLMInterface): url = f"{self.base_url}/codex/responses" + # Debug logging for troubleshooting + logger.debug(f"Codex tool call request: url={url}, model={payload['model']}, tools={len(codex_tools)}") + try: response = await self._client.post(url, json=payload, headers=headers, timeout=120.0) + + # Log response details on error + if response.status_code != 200: + logger.error(f"Codex API error {response.status_code}: {response.text[:500]}") + response.raise_for_status() # Parse SSE for tool calls and content @@ -512,13 +546,30 @@ class CodexLLM(LLMInterface): if event_type == "response.text.delta" and "delta" in data: content += data["delta"] - # Extract tool calls - elif event_type == "response.function_call_arguments.delta": - # Handle tool call events (implementation depends on actual Codex SSE format) - pass + # Extract completed tool calls from response.output_item.done + elif event_type == "response.output_item.done": + item = data.get("item", {}) + if item.get("type") == "function_call" and item.get("status") == "completed": + tool_name = item.get("name", "") + arguments_str = item.get("arguments", "{}") + call_id = item.get("call_id", "") - except json.JSONDecodeError: - pass + try: + arguments = json.loads(arguments_str) + except json.JSONDecodeError: + logger.warning(f"Failed to parse tool arguments: {arguments_str}") + arguments = {} + + tool_calls.append( + LLMToolCall( + id=call_id, + name=tool_name, + arguments=arguments, + ) + ) + + except json.JSONDecodeError as e: + logger.warning(f"Failed to parse SSE data: {e}, data_str: {data_str[:200]}") return content if content else None, tool_calls diff --git a/hindsight-api/hindsight_api/engine/reflect/prompts.py b/hindsight-api/hindsight_api/engine/reflect/prompts.py index 0d064ccd..7a0296b3 100644 --- a/hindsight-api/hindsight_api/engine/reflect/prompts.py +++ b/hindsight-api/hindsight_api/engine/reflect/prompts.py @@ -463,9 +463,12 @@ def build_final_prompt( parts.append( "\n## Instructions\n" "Provide a thoughtful answer by synthesizing and reasoning from the retrieved data above. " - "You can make reasonable inferences from the memories, but don't completely fabricate information." + "You can make reasonable inferences from the memories, but don't completely fabricate information. " "If the exact answer isn't stated, use what IS stated to give the best possible answer. " - "Only say 'I don't have information' if the retrieved data is truly unrelated to the question." + "Only say 'I don't have information' if the retrieved data is truly unrelated to the question.\n\n" + "IMPORTANT: Output ONLY the final answer. Do NOT include meta-commentary like " + '"I\'ll search..." or "Let me analyze...". Do NOT explain your reasoning process. ' + "Just provide the direct synthesized answer." ) return "\n".join(parts) @@ -480,4 +483,10 @@ Your approach: - Be helpful - if you have related information, use it to give the best possible answer Only say "I don't have information" if the retrieved data is truly unrelated to the question. -Do NOT fabricate information that has no basis in the retrieved data.""" +Do NOT fabricate information that has no basis in the retrieved data. + +CRITICAL: Output ONLY the final synthesized answer. Do NOT include: +- Meta-commentary about what you're doing ("I'll search...", "Let me analyze...") +- Explanations of your reasoning process +- Descriptions of your approach +Just provide the direct answer.""" diff --git a/hindsight-api/tests/conftest.py b/hindsight-api/tests/conftest.py index b3ba72fe..2a5822fe 100644 --- a/hindsight-api/tests/conftest.py +++ b/hindsight-api/tests/conftest.py @@ -220,3 +220,34 @@ async def memory(pg0_db_url, embeddings, cross_encoder, query_analyzer): await mem.close() except Exception: pass + + +@pytest_asyncio.fixture(scope="function") +async def memory_no_llm_verify(pg0_db_url, embeddings, cross_encoder, query_analyzer): + """ + Provide a MemoryEngine instance that skips LLM connection verification. + + This fixture is useful for tests that override the LLM configuration + after initialization (e.g., to test specific providers). + """ + mem = MemoryEngine( + db_url=pg0_db_url, + memory_llm_provider="mock", # Use mock provider as placeholder + memory_llm_api_key="", + memory_llm_model="mock", + embeddings=embeddings, + cross_encoder=cross_encoder, + query_analyzer=query_analyzer, + pool_min_size=1, + pool_max_size=5, + run_migrations=False, + task_backend=SyncTaskBackend(), + skip_llm_verification=True, # Skip verification - will be overridden by test + ) + await mem.initialize() + yield mem + try: + if mem._pool and not mem._pool._closing: + await mem.close() + except Exception: + pass diff --git a/hindsight-api/tests/test_llm_provider.py b/hindsight-api/tests/test_llm_provider.py index df4f0975..f7c10aba 100644 --- a/hindsight-api/tests/test_llm_provider.py +++ b/hindsight-api/tests/test_llm_provider.py @@ -1,5 +1,10 @@ """ -Test LLM provider with different models using actual memory operations. +Test LLM provider with different models using actual Hindsight memory operations. + +Tests validate that providers work correctly with: +1. Retain (memory ingestion with fact extraction) +2. Reflect (memory retrieval with tool calling) +3. Mental models (consolidated knowledge generation) """ import os from datetime import datetime @@ -33,6 +38,12 @@ MODEL_MATRIX = [ # Ollama models (local) ("ollama", "gemma3:12b"), ("ollama", "gemma3:1b"), + # Claude Code (uses Claude Agent SDK with Claude models) + ("claude-code", "claude-sonnet-4-20250514"), + # OpenAI Codex (uses MCP with Codex-specific models) + ("openai-codex", "gpt-5.2-codex"), + # Mock provider (for testing) + ("mock", "mock"), ] @@ -48,6 +59,165 @@ def get_api_key_for_provider(provider: str) -> str | None: return os.getenv(env_var) if env_var else None +def should_skip_provider(provider: str, model: str = "") -> tuple[bool, str]: + """Check if provider should be skipped and return reason.""" + # Never skip mock provider + if provider == "mock": + return False, "" + + # Skip claude-code and openai-codex in CI (require local auth) + if os.getenv("CI") and provider in ("claude-code", "openai-codex"): + return True, f"{provider} not available in CI (requires local authentication)" + + # Skip Ollama in CI (no models available) + if provider == "ollama" and os.getenv("CI"): + return True, "Ollama not available in CI" + + # Skip Ollama gemma models (don't support tool calling) + if provider == "ollama" and "gemma" in model.lower(): + return True, f"Ollama {model} does not support tool calling" + + # Other providers need an API key + if provider not in ("ollama", "claude-code", "openai-codex", "mock"): + api_key = get_api_key_for_provider(provider) + if not api_key: + return True, f"No API key available (set {provider.upper()}_API_KEY)" + + return False, "" + + +@pytest.mark.parametrize("provider,model", MODEL_MATRIX) +@pytest.mark.asyncio +async def test_llm_provider_api_methods(provider: str, model: str): + """ + Test all LLM API methods used by Hindsight at runtime. + This validates that the provider correctly implements the LLMInterface. + + Tests: + 1. verify_connection() - Connection verification + 2. call() with plain text - Basic LLM call + 3. call() with response_format - Structured output (used in fact extraction) + 4. call_with_tools() - Tool calling (used in reflect agent) + """ + # Skip mock provider - it's a test stub, not a real LLM implementation + if provider == "mock": + pytest.skip("Mock provider is a test stub, not a real LLM") + + should_skip, reason = should_skip_provider(provider, model) + if should_skip: + pytest.skip(f"Skipping {provider}/{model}: {reason}") + + api_key = get_api_key_for_provider(provider) + + llm = LLMProvider( + provider=provider, + api_key=api_key or "", + base_url="", + model=model, + ) + + print(f"\n{provider}/{model} - API methods test:") + + # Test 1: verify_connection() + try: + await llm.verify_connection() + print(" ✓ verify_connection()") + except Exception as e: + pytest.fail(f"{provider}/{model} verify_connection() failed: {e}") + + # Test 2: call() with plain text + try: + response = await llm.call( + messages=[ + {"role": "system", "content": "You are a helpful assistant."}, + {"role": "user", "content": "What is 2+2? Answer in one word."}, + ], + max_completion_tokens=50, + ) + assert response is not None, "call() returned None" + assert len(response) > 0, "call() returned empty string" + print(f" ✓ call() plain text: {response[:50]}") + except Exception as e: + pytest.fail(f"{provider}/{model} call() plain text failed: {e}") + + # Test 3: call() with response_format (structured output) + try: + from pydantic import BaseModel + + class TestResponse(BaseModel): + answer: str + confidence: str + + response = await llm.call( + messages=[ + {"role": "system", "content": "You are a math assistant."}, + {"role": "user", "content": "What is the capital of France?"}, + ], + response_format=TestResponse, + max_completion_tokens=100, + ) + assert isinstance(response, TestResponse), f"Expected TestResponse, got {type(response)}" + assert hasattr(response, "answer"), "Structured output missing 'answer' field" + assert hasattr(response, "confidence"), "Structured output missing 'confidence' field" + print(f" ✓ call() structured output: answer={response.answer}, confidence={response.confidence}") + except Exception as e: + pytest.fail(f"{provider}/{model} call() structured output failed: {e}") + + # Test 4: call_with_tools() (tool calling) + try: + tools = [ + { + "type": "function", + "function": { + "name": "get_weather", + "description": "Get the weather for a location", + "parameters": { + "type": "object", + "properties": { + "location": {"type": "string", "description": "City name"}, + "unit": {"type": "string", "enum": ["celsius", "fahrenheit"]}, + }, + "required": ["location"], + }, + }, + } + ] + + result = await llm.call_with_tools( + messages=[ + {"role": "system", "content": "You are a helpful assistant with access to tools."}, + {"role": "user", "content": "What's the weather like in Paris?"}, + ], + tools=tools, + max_completion_tokens=200, + ) + + assert result is not None, "call_with_tools() returned None" + assert hasattr(result, "tool_calls"), "Result missing 'tool_calls' attribute" + + # Nano models may hit token limits before making tool calls - that's acceptable + is_nano_model = "nano" in model.lower() + if is_nano_model and len(result.tool_calls) == 0: + # Check if it hit length limit (expected for nano models) + if hasattr(result, "finish_reason") and result.finish_reason == "length": + print(f" ✓ call_with_tools(): nano model hit token limit (expected)") + else: + pytest.fail(f"Nano model made 0 tool calls but didn't hit length limit (finish_reason={getattr(result, 'finish_reason', 'unknown')})") + else: + assert len(result.tool_calls) > 0, f"Expected at least 1 tool call, got {len(result.tool_calls)}" + + # Verify tool call structure + tool_call = result.tool_calls[0] + assert hasattr(tool_call, "name"), "Tool call missing 'name'" + assert hasattr(tool_call, "arguments"), "Tool call missing 'arguments'" + assert tool_call.name == "get_weather", f"Expected 'get_weather', got '{tool_call.name}'" + assert "location" in tool_call.arguments, "Tool call arguments missing 'location'" + + print(f" ✓ call_with_tools(): {tool_call.name}({tool_call.arguments})") + except Exception as e: + pytest.fail(f"{provider}/{model} call_with_tools() failed: {e}") + + @pytest.mark.parametrize("provider,model", MODEL_MATRIX) @pytest.mark.asyncio async def test_llm_provider_memory_operations(provider: str, model: str): @@ -55,16 +225,16 @@ async def test_llm_provider_memory_operations(provider: str, model: str): Test LLM provider with actual memory operations: fact extraction and reflect. All models must pass this test. """ + # Skip mock provider - it's a test stub, not designed for real operations + if provider == "mock": + pytest.skip("Mock provider is a test stub, not designed for real operations") + + should_skip, reason = should_skip_provider(provider, model) + if should_skip: + pytest.skip(f"Skipping {provider}/{model}: {reason}") + api_key = get_api_key_for_provider(provider) - # Skip Ollama tests in CI (no models available) - if provider == "ollama" and os.getenv("CI"): - pytest.skip(f"Skipping {provider}/{model}: Ollama not available in CI") - - # Other providers need an API key - if provider != "ollama" and not api_key: - pytest.skip(f"Skipping {provider}/{model}: no API key available") - llm = LLMProvider( provider=provider, api_key=api_key or "", @@ -122,3 +292,115 @@ async def test_llm_provider_memory_operations(provider: str, model: str): assert response is not None, f"{provider}/{model} reflect returned None" assert len(response) > 10, f"{provider}/{model} reflect response too short" + + +@pytest.mark.parametrize("provider,model", [ + ("claude-code", "claude-sonnet-4-20250514"), + ("openai-codex", "gpt-5.2-codex"), +]) +@pytest.mark.asyncio +async def test_llm_provider_consolidation(memory_no_llm_verify, request_context, provider: str, model: str): + """ + Test LLM provider with consolidation (automatic mental model generation from observations). + This validates that the provider can generate synthesized knowledge from raw memories. + + This test is limited to claude-code and codex since they're the critical providers + that needed tool calling fixes for reflect and consolidation operations. + """ + should_skip, reason = should_skip_provider(provider, model) + if should_skip: + pytest.skip(f"Skipping {provider}/{model}: {reason}") + + # Use provider-specific LLM for this test + api_key = get_api_key_for_provider(provider) + memory_no_llm_verify._consolidation_llm = LLMProvider( + provider=provider, + api_key=api_key or "", + base_url="", + model=model, + ) + # Also need retain LLM for ingesting data + memory_no_llm_verify._retain_llm = memory_no_llm_verify._consolidation_llm + + test_bank_id = f"llm_test_consolidation_{provider}_{model}_{datetime.now().timestamp()}" + + # Enable observations for this bank + from hindsight_api.config import get_config + config = get_config() + original_value = config.enable_observations + config.enable_observations = True + + try: + # Retain memories to consolidate + test_content = """ + Bob prefers functional programming with Rust and Haskell. + He emphasizes immutability and pure functions in code reviews. + Bob advocates for type safety and compile-time guarantees. + He avoids mutable state and prefers declarative code patterns. + """ + + await memory_no_llm_verify.retain_async( + bank_id=test_bank_id, + content=test_content, + context="Team coding preferences", + event_date=datetime(2024, 12, 1), + request_context=request_context, + ) + + print(f"\n{provider}/{model} - Consolidation test:") + + # Run consolidation to generate observations (mental models) + from hindsight_api.engine.consolidation.consolidator import run_consolidation_job + + result = await run_consolidation_job( + memory_engine=memory_no_llm_verify, + bank_id=test_bank_id, + request_context=request_context, + ) + + print(f" Processed: {result.get('memories_processed', 0)} memories") + print(f" Created: {result.get('observations_created', 0)} observations") + print(f" Updated: {result.get('observations_updated', 0)} observations") + + # Verify consolidation ran successfully + assert result["status"] in ["success", "no_new_memories"], f"{provider}/{model} consolidation failed" + + # If observations were created, verify they contain relevant content + if result.get("observations_created", 0) > 0: + observations = await memory_no_llm_verify.list_mental_models_consolidated( + bank_id=test_bank_id, + request_context=request_context, + ) + + assert len(observations) > 0, f"{provider}/{model} consolidation created 0 observations" + + # Check first observation contains relevant information + obs_content = observations[0].get("content", "").lower() + relevant_terms = ["bob", "functional", "rust", "immutab", "type"] + matches = [term for term in relevant_terms if term in obs_content] + + print(f" Observation preview: {observations[0].get('content', '')[:200]}...") + print(f" Found {len(matches)} relevant terms: {matches}") + + assert len(matches) >= 2, ( + f"{provider}/{model} consolidated observation doesn't contain relevant info. " + f"Expected at least 2 of {relevant_terms}, found {len(matches)}: {matches}" + ) + + finally: + # Restore original config + config.enable_observations = original_value + + +# NOTE: The tests above validate the critical Hindsight operations: +# +# test_llm_provider_memory_operations (ALL providers): +# - Fact extraction (retain): tests structured output generation +# - Reflect: tests memory retrieval and reasoning (uses tool calling for claude-code/codex) +# +# test_llm_provider_consolidation (claude-code and codex only): +# - Consolidation: tests automatic mental model generation from observations +# - Requires MemoryEngine fixture with working LLM (from .env or env vars) +# - Run your local LLM server OR set HINDSIGHT_API_LLM_PROVIDER/API_KEY/MODEL env vars +# +# For full end-to-end integration tests using the HTTP API, see tests/test_http_api_integration.py