From 539190b69e07ca11311d823c5fe2b32f250397a1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=B2=20Boschi?= Date: Mon, 2 Feb 2026 12:54:44 +0100 Subject: [PATCH] feat: support for codex and claude-code as llm (#276) * feat: support for codex and claude-code as llm * Remove refactoring plan file * Consolidate Anthropic tests into main LLM provider test suite - Add Anthropic models (Sonnet, Opus, Haiku) to MODEL_MATRIX - Remove separate test_anthropic_provider.py file - All Anthropic models now tested with standard memory operations * Add provider-specific default models Each LLM provider now has a sensible default model that's used when HINDSIGHT_API_LLM_MODEL is not explicitly set. This simplifies configuration - users can specify just the provider and API key. Changes: - Add PROVIDER_DEFAULT_MODELS mapping in config.py - Update config logic to use provider defaults for both global and per-operation LLM configs - Add comprehensive tests for provider default model selection - Document provider defaults in models.md Example usage: export HINDSIGHT_API_LLM_PROVIDER=anthropic export HINDSIGHT_API_LLM_API_KEY=sk-ant-xxx # Automatically uses claude-sonnet-4-20250514 Provider defaults: - openai: gpt-5-mini - anthropic: claude-sonnet-4-20250514 - gemini: gemini-2.5-flash - groq: openai/gpt-oss-120b - ollama: gemma3:12b - lmstudio: local-model - vertexai: gemini-2.0-flash-001 - openai-codex: o3-mini - claude-code: claude-sonnet-4-20250514 - mock: mock-model * Update provider default models - openai: gpt-5-mini -> o3-mini - anthropic: claude-sonnet-4-20250514 -> claude-haiku-4-5-20251001 - openai-codex: o3-mini -> gpt-5.2-codex - claude-code: claude-sonnet-4-20250514 -> claude-sonnet-4-5-20250929 Updated tests and documentation to reflect new defaults. * Move OpenAI Codex and Claude Code setup to models.md Moved detailed setup instructions for OpenAI Codex and Claude Code from configuration.md to models.md where they better fit with model-specific documentation. Changes: - Move "OpenAI Codex Setup" section from configuration.md to models.md - Move "Claude Code Setup" section from configuration.md to models.md - Add cross-reference tip in configuration.md pointing to models.md - Update default model in Claude Code example to claude-sonnet-4-5-20250929 - Keep basic provider examples in configuration.md for quick reference This makes the configuration.md page more focused on environment variables while models.md contains provider-specific setup details. --- hindsight-api/hindsight_api/config.py | 50 +- .../hindsight_api/engine/llm_interface.py | 146 ++ .../hindsight_api/engine/llm_wrapper.py | 1653 ++++------------- .../hindsight_api/engine/memory_engine.py | 6 +- .../engine/providers/__init__.py | 14 + .../engine/providers/anthropic_llm.py | 434 +++++ .../engine/providers/claude_code_llm.py | 352 ++++ .../engine/providers/codex_llm.py | 527 ++++++ .../engine/providers/gemini_llm.py | 502 +++++ .../engine/providers/mock_llm.py | 234 +++ .../engine/providers/openai_compatible_llm.py | 745 ++++++++ hindsight-api/pyproject.toml | 19 +- hindsight-api/tests/test_llm_provider.py | 5 + hindsight-api/tests/test_llm_token_metrics.py | 6 +- .../tests/test_provider_default_models.py | 123 ++ .../docs/developer/configuration.md | 16 +- hindsight-docs/docs/developer/models.md | 182 ++ uv.lock | 18 + 18 files changed, 3674 insertions(+), 1358 deletions(-) create mode 100644 hindsight-api/hindsight_api/engine/llm_interface.py create mode 100644 hindsight-api/hindsight_api/engine/providers/__init__.py create mode 100644 hindsight-api/hindsight_api/engine/providers/anthropic_llm.py create mode 100644 hindsight-api/hindsight_api/engine/providers/claude_code_llm.py create mode 100644 hindsight-api/hindsight_api/engine/providers/codex_llm.py create mode 100644 hindsight-api/hindsight_api/engine/providers/gemini_llm.py create mode 100644 hindsight-api/hindsight_api/engine/providers/mock_llm.py create mode 100644 hindsight-api/hindsight_api/engine/providers/openai_compatible_llm.py create mode 100644 hindsight-api/tests/test_provider_default_models.py diff --git a/hindsight-api/hindsight_api/config.py b/hindsight-api/hindsight_api/config.py index 764e5c82..c999f9dd 100644 --- a/hindsight-api/hindsight_api/config.py +++ b/hindsight-api/hindsight_api/config.py @@ -154,7 +154,21 @@ ENV_REFLECT_MAX_ITERATIONS = "HINDSIGHT_API_REFLECT_MAX_ITERATIONS" DEFAULT_DATABASE_URL = "pg0" DEFAULT_DATABASE_SCHEMA = "public" DEFAULT_LLM_PROVIDER = "openai" -DEFAULT_LLM_MODEL = "gpt-5-mini" + +# Provider-specific default models +PROVIDER_DEFAULT_MODELS = { + "openai": "o3-mini", + "anthropic": "claude-haiku-4-5-20251001", + "gemini": "gemini-2.5-flash", + "groq": "openai/gpt-oss-120b", + "ollama": "gemma3:12b", + "lmstudio": "local-model", + "vertexai": "gemini-2.0-flash-001", + "openai-codex": "gpt-5.2-codex", + "claude-code": "claude-sonnet-4-5-20250929", + "mock": "mock-model", +} +DEFAULT_LLM_MODEL = "o3-mini" # Fallback if provider not in table DEFAULT_LLM_MAX_CONCURRENT = 32 DEFAULT_LLM_MAX_RETRIES = 10 # Max retry attempts for LLM API calls DEFAULT_LLM_INITIAL_BACKOFF = 1.0 # Initial backoff in seconds for retry exponential backoff @@ -303,6 +317,11 @@ def _validate_extraction_mode(mode: str) -> str: return mode_lower +def _get_default_model_for_provider(provider: str) -> str: + """Get the default model for a given provider.""" + return PROVIDER_DEFAULT_MODELS.get(provider.lower(), DEFAULT_LLM_MODEL) + + @dataclass class HindsightConfig: """Configuration container for Hindsight API.""" @@ -431,14 +450,18 @@ class HindsightConfig: @classmethod def from_env(cls) -> "HindsightConfig": """Create configuration from environment variables.""" + # Get provider first to determine default model + llm_provider = os.getenv(ENV_LLM_PROVIDER, DEFAULT_LLM_PROVIDER) + llm_model = os.getenv(ENV_LLM_MODEL) or _get_default_model_for_provider(llm_provider) + return cls( # Database database_url=os.getenv(ENV_DATABASE_URL, DEFAULT_DATABASE_URL), database_schema=os.getenv(ENV_DATABASE_SCHEMA, DEFAULT_DATABASE_SCHEMA), # LLM - llm_provider=os.getenv(ENV_LLM_PROVIDER, DEFAULT_LLM_PROVIDER), + llm_provider=llm_provider, llm_api_key=os.getenv(ENV_LLM_API_KEY), - llm_model=os.getenv(ENV_LLM_MODEL, DEFAULT_LLM_MODEL), + llm_model=llm_model, llm_base_url=os.getenv(ENV_LLM_BASE_URL) or None, llm_max_concurrent=int(os.getenv(ENV_LLM_MAX_CONCURRENT, str(DEFAULT_LLM_MAX_CONCURRENT))), llm_max_retries=int(os.getenv(ENV_LLM_MAX_RETRIES, str(DEFAULT_LLM_MAX_RETRIES))), @@ -453,7 +476,12 @@ class HindsightConfig: # Per-operation LLM config (None = use default) retain_llm_provider=os.getenv(ENV_RETAIN_LLM_PROVIDER) or None, retain_llm_api_key=os.getenv(ENV_RETAIN_LLM_API_KEY) or None, - retain_llm_model=os.getenv(ENV_RETAIN_LLM_MODEL) or None, + retain_llm_model=os.getenv(ENV_RETAIN_LLM_MODEL) + or ( + _get_default_model_for_provider(os.getenv(ENV_RETAIN_LLM_PROVIDER)) + if os.getenv(ENV_RETAIN_LLM_PROVIDER) + else None + ), retain_llm_base_url=os.getenv(ENV_RETAIN_LLM_BASE_URL) or None, retain_llm_max_concurrent=int(os.getenv(ENV_RETAIN_LLM_MAX_CONCURRENT)) if os.getenv(ENV_RETAIN_LLM_MAX_CONCURRENT) @@ -470,7 +498,12 @@ class HindsightConfig: retain_llm_timeout=float(os.getenv(ENV_RETAIN_LLM_TIMEOUT)) if os.getenv(ENV_RETAIN_LLM_TIMEOUT) else None, reflect_llm_provider=os.getenv(ENV_REFLECT_LLM_PROVIDER) or None, reflect_llm_api_key=os.getenv(ENV_REFLECT_LLM_API_KEY) or None, - reflect_llm_model=os.getenv(ENV_REFLECT_LLM_MODEL) or None, + reflect_llm_model=os.getenv(ENV_REFLECT_LLM_MODEL) + or ( + _get_default_model_for_provider(os.getenv(ENV_REFLECT_LLM_PROVIDER)) + if os.getenv(ENV_REFLECT_LLM_PROVIDER) + else None + ), reflect_llm_base_url=os.getenv(ENV_REFLECT_LLM_BASE_URL) or None, reflect_llm_max_concurrent=int(os.getenv(ENV_REFLECT_LLM_MAX_CONCURRENT)) if os.getenv(ENV_REFLECT_LLM_MAX_CONCURRENT) @@ -489,7 +522,12 @@ class HindsightConfig: else None, consolidation_llm_provider=os.getenv(ENV_CONSOLIDATION_LLM_PROVIDER) or None, consolidation_llm_api_key=os.getenv(ENV_CONSOLIDATION_LLM_API_KEY) or None, - consolidation_llm_model=os.getenv(ENV_CONSOLIDATION_LLM_MODEL) or None, + consolidation_llm_model=os.getenv(ENV_CONSOLIDATION_LLM_MODEL) + or ( + _get_default_model_for_provider(os.getenv(ENV_CONSOLIDATION_LLM_PROVIDER)) + if os.getenv(ENV_CONSOLIDATION_LLM_PROVIDER) + else None + ), consolidation_llm_base_url=os.getenv(ENV_CONSOLIDATION_LLM_BASE_URL) or None, consolidation_llm_max_concurrent=int(os.getenv(ENV_CONSOLIDATION_LLM_MAX_CONCURRENT)) if os.getenv(ENV_CONSOLIDATION_LLM_MAX_CONCURRENT) diff --git a/hindsight-api/hindsight_api/engine/llm_interface.py b/hindsight-api/hindsight_api/engine/llm_interface.py new file mode 100644 index 00000000..ee1af600 --- /dev/null +++ b/hindsight-api/hindsight_api/engine/llm_interface.py @@ -0,0 +1,146 @@ +""" +Abstract interface for LLM providers. + +This module defines the interface that all LLM providers must implement, +enabling support for multiple LLM backends (OpenAI, Anthropic, Gemini, Codex, etc.) +""" + +from abc import ABC, abstractmethod +from typing import Any + +from .response_models import LLMToolCallResult, TokenUsage + + +class LLMInterface(ABC): + """ + Abstract interface for LLM providers. + + All LLM provider implementations must inherit from this class and implement + the required methods. + """ + + def __init__( + self, + provider: str, + api_key: str, + base_url: str, + model: str, + reasoning_effort: str = "low", + **kwargs: Any, + ): + """ + Initialize LLM provider. + + Args: + provider: Provider name (e.g., "openai", "codex", "anthropic", "gemini"). + api_key: API key or authentication token. + base_url: Base URL for the API. + model: Model name. + reasoning_effort: Reasoning effort level for supported providers. + **kwargs: Additional provider-specific parameters. + """ + self.provider = provider.lower() + self.api_key = api_key + self.base_url = base_url + self.model = model + self.reasoning_effort = reasoning_effort + + @abstractmethod + async def verify_connection(self) -> None: + """ + Verify that the LLM provider is configured correctly by making a simple test call. + + Raises: + RuntimeError: If the connection test fails. + """ + pass + + @abstractmethod + async def call( + self, + messages: list[dict[str, str]], + response_format: Any | None = None, + max_completion_tokens: int | None = None, + temperature: float | None = None, + scope: str = "memory", + max_retries: int = 10, + initial_backoff: float = 1.0, + max_backoff: float = 60.0, + skip_validation: bool = False, + strict_schema: bool = False, + return_usage: bool = False, + ) -> Any: + """ + Make an LLM API call with retry logic. + + Args: + messages: List of message dicts with 'role' and 'content'. + response_format: Optional Pydantic model for structured output. + max_completion_tokens: Maximum tokens in response. + temperature: Sampling temperature (0.0-2.0). + scope: Scope identifier for tracking. + max_retries: Maximum retry attempts. + initial_backoff: Initial backoff time in seconds. + max_backoff: Maximum backoff time in seconds. + skip_validation: Return raw JSON without Pydantic validation. + strict_schema: Use strict JSON schema enforcement (OpenAI only). + return_usage: If True, return tuple (result, TokenUsage) instead of just result. + + Returns: + If return_usage=False: Parsed response if response_format is provided, otherwise text content. + If return_usage=True: Tuple of (result, TokenUsage) with token counts. + + Raises: + OutputTooLongError: If output exceeds token limits. + Exception: Re-raises API errors after retries exhausted. + """ + pass + + @abstractmethod + async def call_with_tools( + self, + messages: list[dict[str, Any]], + tools: list[dict[str, Any]], + max_completion_tokens: int | None = None, + temperature: float | None = None, + scope: str = "tools", + max_retries: int = 5, + initial_backoff: float = 1.0, + max_backoff: float = 30.0, + tool_choice: str | dict[str, Any] = "auto", + ) -> LLMToolCallResult: + """ + Make an LLM API call with tool/function calling support. + + Args: + messages: List of message dicts. Can include tool results with role='tool'. + tools: List of tool definitions in OpenAI format. + max_completion_tokens: Maximum tokens in response. + temperature: Sampling temperature (0.0-2.0). + scope: Scope identifier for tracking. + max_retries: Maximum retry attempts. + initial_backoff: Initial backoff time in seconds. + max_backoff: Maximum backoff time in seconds. + tool_choice: How to choose tools - "auto", "none", "required", or specific function. + + Returns: + LLMToolCallResult with content and/or tool_calls. + """ + pass + + @abstractmethod + async def cleanup(self) -> None: + """Clean up resources (close connections, etc.).""" + pass + + +class OutputTooLongError(Exception): + """ + Bridge exception raised when LLM output exceeds token limits. + + This wraps provider-specific errors (e.g., OpenAI's LengthFinishReasonError) + to allow callers to handle output length issues without depending on + provider-specific implementations. + """ + + pass diff --git a/hindsight-api/hindsight_api/engine/llm_wrapper.py b/hindsight-api/hindsight_api/engine/llm_wrapper.py index a7d5878c..85d22940 100644 --- a/hindsight-api/hindsight_api/engine/llm_wrapper.py +++ b/hindsight-api/hindsight_api/engine/llm_wrapper.py @@ -8,15 +8,14 @@ import logging import os import re import time +import uuid +from pathlib import Path from typing import Any import httpx -from google import genai -from google.genai import errors as genai_errors -from google.genai import types as genai_types from openai import APIConnectionError, APIStatusError, AsyncOpenAI, LengthFinishReasonError -# Vertex AI imports (conditional) +# Vertex AI imports (conditional - for LLMProvider to pass credentials to GeminiLLM) try: import google.auth from google.oauth2 import service_account @@ -61,6 +60,108 @@ class OutputTooLongError(Exception): pass +def create_llm_provider( + provider: str, + api_key: str, + base_url: str, + model: str, + reasoning_effort: str, + groq_service_tier: str | None = None, + vertexai_project_id: str | None = None, + vertexai_region: str | None = None, + vertexai_credentials: Any = None, +) -> Any: # Returns LLMInterface + """ + Factory function to create the appropriate LLM provider implementation. + + Args: + provider: Provider name ("openai", "groq", "ollama", "gemini", "anthropic", etc.). + api_key: API key (may be None for local providers or OAuth providers). + base_url: Base URL for the API. + model: Model name. + reasoning_effort: Reasoning effort level for supported providers. + groq_service_tier: Groq service tier (for Groq provider). + vertexai_project_id: Vertex AI project ID (for VertexAI provider). + vertexai_region: Vertex AI region (for VertexAI provider). + vertexai_credentials: Vertex AI credentials object (for VertexAI provider). + + Returns: + LLMInterface implementation for the specified provider. + """ + from .llm_interface import LLMInterface + from .providers import ( + AnthropicLLM, + ClaudeCodeLLM, + CodexLLM, + GeminiLLM, + MockLLM, + OpenAICompatibleLLM, + ) + + provider_lower = provider.lower() + + if provider_lower == "openai-codex": + return CodexLLM( + provider=provider, + api_key=api_key, + base_url=base_url, + model=model, + reasoning_effort=reasoning_effort, + ) + + elif provider_lower == "claude-code": + return ClaudeCodeLLM( + provider=provider, + api_key=api_key, + base_url=base_url, + model=model, + reasoning_effort=reasoning_effort, + ) + + elif provider_lower == "mock": + return MockLLM( + provider=provider, + api_key=api_key, + base_url=base_url, + model=model, + reasoning_effort=reasoning_effort, + ) + + elif provider_lower in ("gemini", "vertexai"): + return GeminiLLM( + provider=provider, + api_key=api_key, + base_url=base_url, + model=model, + reasoning_effort=reasoning_effort, + vertexai_project_id=vertexai_project_id, + vertexai_region=vertexai_region, + vertexai_credentials=vertexai_credentials, + ) + + elif provider_lower == "anthropic": + return AnthropicLLM( + provider=provider, + api_key=api_key, + base_url=base_url, + model=model, + reasoning_effort=reasoning_effort, + ) + + elif provider_lower in ("openai", "groq", "ollama", "lmstudio"): + return OpenAICompatibleLLM( + provider=provider, + api_key=api_key, + base_url=base_url, + model=model, + reasoning_effort=reasoning_effort, + groq_service_tier=groq_service_tier, + ) + + else: + raise ValueError(f"Unknown provider: {provider}") + + class LLMProvider: """ Unified LLM provider. @@ -97,14 +198,21 @@ class LLMProvider: self.groq_service_tier = groq_service_tier or os.getenv(ENV_LLM_GROQ_SERVICE_TIER, "auto") # Validate provider - valid_providers = ["openai", "groq", "ollama", "gemini", "anthropic", "lmstudio", "vertexai", "mock"] + valid_providers = [ + "openai", + "groq", + "ollama", + "gemini", + "anthropic", + "lmstudio", + "vertexai", + "openai-codex", + "claude-code", + "mock", + ] if self.provider not in valid_providers: raise ValueError(f"Invalid LLM provider: {self.provider}. Must be one of: {', '.join(valid_providers)}") - # Mock provider tracking (for testing) - self._mock_calls: list[dict] = [] - self._mock_response: Any = None - # Set default base URLs if not self.base_url: if self.provider == "groq": @@ -114,24 +222,24 @@ class LLMProvider: elif self.provider == "lmstudio": self.base_url = "http://localhost:1234/v1" - # Vertex AI config — stored for client creation below - self._vertexai_project_id: str | None = None - self._vertexai_region: str | None = None - self._vertexai_credentials: Any = None + # Prepare Vertex AI config (if applicable) + vertexai_project_id = None + vertexai_region = None + vertexai_credentials = None if self.provider == "vertexai": from ..config import get_config config = get_config() - self._vertexai_project_id = config.llm_vertexai_project_id - if not self._vertexai_project_id: + vertexai_project_id = config.llm_vertexai_project_id + if not vertexai_project_id: raise ValueError( "HINDSIGHT_API_LLM_VERTEXAI_PROJECT_ID is required for Vertex AI provider. " "Set it to your GCP project ID." ) - self._vertexai_region = config.llm_vertexai_region or "us-central1" + vertexai_region = config.llm_vertexai_region or "us-central1" service_account_key = config.llm_vertexai_service_account_key # Load explicit service account credentials if provided @@ -141,75 +249,71 @@ class LLMProvider: "Vertex AI service account auth requires 'google-auth' package. " "Install with: pip install google-auth" ) - self._vertexai_credentials = service_account.Credentials.from_service_account_file( + vertexai_credentials = service_account.Credentials.from_service_account_file( service_account_key, scopes=["https://www.googleapis.com/auth/cloud-platform"], ) logger.info(f"Vertex AI: Using service account key: {service_account_key}") # Strip google/ prefix from model name — native SDK uses bare names - # e.g. "google/gemini-2.0-flash-lite-001" -> "gemini-2.0-flash-lite-001" if self.model.startswith("google/"): self.model = self.model[len("google/") :] logger.info( - f"Vertex AI: project={self._vertexai_project_id}, region={self._vertexai_region}, " + f"Vertex AI: project={vertexai_project_id}, region={vertexai_region}, " f"model={self.model}, auth={'service_account' if service_account_key else 'ADC'}" ) - # Validate API key (not needed for ollama, lmstudio, vertexai, or mock) - if self.provider not in ("ollama", "lmstudio", "vertexai", "mock") and not self.api_key: - raise ValueError(f"API key not found for {self.provider}") + # Create provider implementation using factory + self._provider_impl = create_llm_provider( + provider=self.provider, + api_key=self.api_key, + base_url=self.base_url, + model=self.model, + reasoning_effort=self.reasoning_effort, + groq_service_tier=self.groq_service_tier, + vertexai_project_id=vertexai_project_id, + vertexai_region=vertexai_region, + vertexai_credentials=vertexai_credentials, + ) - # Get timeout config (set HINDSIGHT_API_LLM_TIMEOUT for local LLMs that need longer timeouts) - self.timeout = float(os.getenv(ENV_LLM_TIMEOUT, str(DEFAULT_LLM_TIMEOUT))) + # Backward compatibility: Keep mock provider properties + self._mock_calls: list[dict] = [] + self._mock_response: Any = None - # Create client based on provider - self._client = None - self._gemini_client = None - self._anthropic_client = None + @property + def _client(self) -> Any: + """ + Get the OpenAI client for OpenAI-compatible providers. - if self.provider == "mock": - # Mock provider - no client needed - pass - elif self.provider == "gemini": - self._gemini_client = genai.Client(api_key=self.api_key) - elif self.provider == "anthropic": - from anthropic import AsyncAnthropic + This property provides backward compatibility for code that directly accesses + the _client attribute (e.g., benchmarks, memory_engine). - # Only pass base_url if it's set (Anthropic uses default URL otherwise) - anthropic_kwargs = {"api_key": self.api_key} - if self.base_url: - anthropic_kwargs["base_url"] = self.base_url - if self.timeout: - anthropic_kwargs["timeout"] = self.timeout - self._anthropic_client = AsyncAnthropic(**anthropic_kwargs) - elif self.provider == "vertexai": - # Native genai SDK with Vertex AI — handles ADC automatically, - # or uses explicit service account credentials if provided - client_kwargs = { - "vertexai": True, - "project": self._vertexai_project_id, - "location": self._vertexai_region, - } - if self._vertexai_credentials is not None: - client_kwargs["credentials"] = self._vertexai_credentials - self._gemini_client = genai.Client(**client_kwargs) - elif self.provider in ("ollama", "lmstudio"): - # Use dummy key if not provided for local - api_key = self.api_key or "local" - client_kwargs = {"api_key": api_key, "base_url": self.base_url, "max_retries": 0} - if self.timeout: - client_kwargs["timeout"] = self.timeout - self._client = AsyncOpenAI(**client_kwargs) - else: - # Only pass base_url if it's set (OpenAI uses default URL otherwise) - client_kwargs = {"api_key": self.api_key, "max_retries": 0} - if self.base_url: - client_kwargs["base_url"] = self.base_url - if self.timeout: - client_kwargs["timeout"] = self.timeout - self._client = AsyncOpenAI(**client_kwargs) + Returns: + AsyncOpenAI client instance for OpenAI-compatible providers, or None for other providers. + """ + from .providers.openai_compatible_llm import OpenAICompatibleLLM + + if isinstance(self._provider_impl, OpenAICompatibleLLM): + return self._provider_impl._client + return None + + @property + def _gemini_client(self) -> Any: + """ + Get the Gemini client for Gemini/VertexAI providers. + + This property provides backward compatibility for code that directly accesses + the _gemini_client attribute. + + Returns: + genai.Client instance for Gemini/VertexAI providers, or None for other providers. + """ + from .providers.gemini_llm import GeminiLLM + + if isinstance(self._provider_impl, GeminiLLM): + return self._provider_impl._client + return None async def verify_connection(self) -> None: """ @@ -218,21 +322,7 @@ class LLMProvider: Raises: RuntimeError: If the connection test fails. """ - try: - logger.info( - f"Verifying LLM: provider={self.provider}, model={self.model}, base_url={self.base_url or 'default'}..." - ) - await self.call( - messages=[{"role": "user", "content": "Say 'ok'"}], - max_completion_tokens=100, - max_retries=2, - initial_backoff=0.5, - max_backoff=2.0, - ) - # If we get here without exception, the connection is working - logger.info(f"LLM verified: {self.provider}/{self.model}") - except Exception as e: - raise RuntimeError(f"LLM connection verification failed for {self.provider}/{self.model}: {e}") from e + await self._provider_impl.verify_connection() async def call( self, @@ -272,340 +362,32 @@ class LLMProvider: OutputTooLongError: If output exceeds token limits. Exception: Re-raises API errors after retries exhausted. """ - semaphore_start = time.time() async with _global_llm_semaphore: - semaphore_wait_time = time.time() - semaphore_start - start_time = time.time() + # Delegate to provider implementation + result = await self._provider_impl.call( + messages=messages, + response_format=response_format, + max_completion_tokens=max_completion_tokens, + temperature=temperature, + scope=scope, + max_retries=max_retries, + initial_backoff=initial_backoff, + max_backoff=max_backoff, + skip_validation=skip_validation, + strict_schema=strict_schema, + return_usage=return_usage, + ) - # Handle Mock provider (for testing) + # Backward compatibility: Update mock call tracking for mock provider + # This allows existing tests using LLMProvider._mock_calls to continue working if self.provider == "mock": - return await self._call_mock( - messages, - response_format, - scope, - return_usage, - ) + from .providers.mock_llm import MockLLM - # Handle Gemini and Vertex AI providers (both use native genai SDK) - if self.provider in ("gemini", "vertexai"): - return await self._call_gemini( - messages, - response_format, - max_retries, - initial_backoff, - max_backoff, - skip_validation, - start_time, - scope, - return_usage, - semaphore_wait_time, - ) + if isinstance(self._provider_impl, MockLLM): + # Sync the mock calls from provider implementation to wrapper + self._mock_calls = self._provider_impl.get_mock_calls() - # Handle Anthropic provider separately - if self.provider == "anthropic": - return await self._call_anthropic( - messages, - response_format, - max_completion_tokens, - max_retries, - initial_backoff, - max_backoff, - skip_validation, - start_time, - scope, - return_usage, - semaphore_wait_time, - ) - - # Handle Ollama with native API for structured output (better schema enforcement) - if self.provider == "ollama" and response_format is not None: - return await self._call_ollama_native( - messages, - response_format, - max_completion_tokens, - temperature, - max_retries, - initial_backoff, - max_backoff, - skip_validation, - start_time, - scope, - return_usage, - semaphore_wait_time, - ) - - call_params = { - "model": self.model, - "messages": messages, - } - - # Check if model supports reasoning parameter (o1, o3, gpt-5 families) - model_lower = self.model.lower() - is_reasoning_model = any(x in model_lower for x in ["gpt-5", "o1", "o3", "deepseek"]) - - # For GPT-4 and GPT-4.1 models, cap max_completion_tokens to 32000 - # For GPT-4o models, cap to 16384 - is_gpt4_model = any(x in model_lower for x in ["gpt-4.1", "gpt-4-"]) - is_gpt4o_model = "gpt-4o" in model_lower - if max_completion_tokens is not None: - if is_gpt4o_model and max_completion_tokens > 16384: - max_completion_tokens = 16384 - elif is_gpt4_model and max_completion_tokens > 32000: - max_completion_tokens = 32000 - # For reasoning models, max_completion_tokens includes reasoning + output tokens - # Enforce minimum of 16000 to ensure enough space for both - if is_reasoning_model and max_completion_tokens < 16000: - max_completion_tokens = 16000 - call_params["max_completion_tokens"] = max_completion_tokens - - # GPT-5/o1/o3 family doesn't support custom temperature (only default 1) - if temperature is not None and not is_reasoning_model: - call_params["temperature"] = temperature - - # Set reasoning_effort for reasoning models (OpenAI gpt-5, o1, o3) - if is_reasoning_model: - call_params["reasoning_effort"] = self.reasoning_effort - - # Provider-specific parameters - if self.provider == "groq": - call_params["seed"] = DEFAULT_LLM_SEED - extra_body: dict[str, Any] = {} - # Add service_tier if configured (requires paid plan for flex/auto) - if self.groq_service_tier: - extra_body["service_tier"] = self.groq_service_tier - # Add reasoning parameters for reasoning models - if is_reasoning_model: - extra_body["include_reasoning"] = False - if extra_body: - call_params["extra_body"] = extra_body - - last_exception = None - - # Prepare response format ONCE before the retry loop - # (to avoid appending schema to messages on every retry) - if response_format is not None: - schema = None - if hasattr(response_format, "model_json_schema"): - schema = response_format.model_json_schema() - - if strict_schema and schema is not None: - # Use OpenAI's strict JSON schema enforcement - # This guarantees all required fields are returned - call_params["response_format"] = { - "type": "json_schema", - "json_schema": { - "name": "response", - "strict": True, - "schema": schema, - }, - } - else: - # Soft enforcement: add schema to prompt and use json_object mode - if schema is not None: - schema_msg = f"\n\nYou must respond with valid JSON matching this schema:\n{json.dumps(schema, indent=2)}" - - if call_params["messages"] and call_params["messages"][0].get("role") == "system": - first_msg = call_params["messages"][0] - if isinstance(first_msg, dict) and isinstance(first_msg.get("content"), str): - first_msg["content"] += schema_msg - elif call_params["messages"]: - first_msg = call_params["messages"][0] - if isinstance(first_msg, dict) and isinstance(first_msg.get("content"), str): - first_msg["content"] = schema_msg + "\n\n" + first_msg["content"] - if self.provider not in ("lmstudio", "ollama"): - # LM Studio and Ollama don't support json_object response format reliably - # We rely on the schema in the system message instead - call_params["response_format"] = {"type": "json_object"} - - for attempt in range(max_retries + 1): - try: - if response_format is not None: - response = await self._client.chat.completions.create(**call_params) - - content = response.choices[0].message.content - - # Strip reasoning model thinking tags - # Supports: , , , |startthink|/|endthink| - # for reasoning models that embed thinking in their output (e.g., Qwen3, DeepSeek) - if content: - original_len = len(content) - content = re.sub(r".*?", "", content, flags=re.DOTALL) - content = re.sub(r".*?", "", content, flags=re.DOTALL) - content = re.sub(r".*?", "", content, flags=re.DOTALL) - content = re.sub(r"\|startthink\|.*?\|endthink\|", "", content, flags=re.DOTALL) - content = content.strip() - if len(content) < original_len: - logger.debug(f"Stripped {original_len - len(content)} chars of reasoning tokens") - - # For local models, they may wrap JSON in markdown code blocks - if self.provider in ("lmstudio", "ollama"): - clean_content = content - if "```json" in content: - clean_content = content.split("```json")[1].split("```")[0].strip() - elif "```" in content: - clean_content = content.split("```")[1].split("```")[0].strip() - try: - json_data = json.loads(clean_content) - except json.JSONDecodeError: - # Fallback to parsing raw content - json_data = json.loads(content) - else: - # Log raw LLM response for debugging JSON parse issues - try: - json_data = json.loads(content) - except json.JSONDecodeError as json_err: - # Truncate content for logging (first 500 and last 200 chars) - content_preview = content[:500] if content else "" - if content and len(content) > 700: - content_preview = f"{content[:500]}...TRUNCATED...{content[-200:]}" - logger.warning( - f"JSON parse error from LLM response (attempt {attempt + 1}/{max_retries + 1}): {json_err}\n" - f" Model: {self.provider}/{self.model}\n" - f" Content length: {len(content) if content else 0} chars\n" - f" Content preview: {content_preview!r}\n" - f" Finish reason: {response.choices[0].finish_reason if response.choices else 'unknown'}" - ) - # Retry on JSON parse errors - LLM may return valid JSON on next attempt - if attempt < max_retries: - backoff = min(initial_backoff * (2**attempt), max_backoff) - await asyncio.sleep(backoff) - last_exception = json_err - continue - else: - logger.error(f"JSON parse error after {max_retries + 1} attempts, giving up") - raise - - if skip_validation: - result = json_data - else: - result = response_format.model_validate(json_data) - else: - response = await self._client.chat.completions.create(**call_params) - result = response.choices[0].message.content - - # Record token usage metrics - duration = time.time() - start_time - usage = response.usage - input_tokens = usage.prompt_tokens or 0 if usage else 0 - output_tokens = usage.completion_tokens or 0 if usage else 0 - total_tokens = usage.total_tokens or 0 if usage else 0 - - # Record LLM metrics - metrics = get_metrics_collector() - metrics.record_llm_call( - provider=self.provider, - model=self.model, - scope=scope, - duration=duration, - input_tokens=input_tokens, - output_tokens=output_tokens, - success=True, - ) - - # Log slow calls - if duration > 10.0 and usage: - ratio = max(1, output_tokens) / max(1, input_tokens) - cached_tokens = 0 - if hasattr(usage, "prompt_tokens_details") and usage.prompt_tokens_details: - cached_tokens = getattr(usage.prompt_tokens_details, "cached_tokens", 0) or 0 - cache_info = f", cached_tokens={cached_tokens}" if cached_tokens > 0 else "" - wait_info = f", wait={semaphore_wait_time:.3f}s" if semaphore_wait_time > 0.1 else "" - logger.info( - f"slow llm call: scope={scope}, model={self.provider}/{self.model}, " - f"input_tokens={input_tokens}, output_tokens={output_tokens}, " - f"total_tokens={total_tokens}{cache_info}, time={duration:.3f}s{wait_info}, ratio out/in={ratio:.2f}" - ) - - if return_usage: - token_usage = TokenUsage( - input_tokens=input_tokens, - output_tokens=output_tokens, - total_tokens=total_tokens, - ) - return result, token_usage - return result - - except LengthFinishReasonError as e: - logger.warning(f"LLM output exceeded token limits: {str(e)}") - raise OutputTooLongError( - "LLM output exceeded token limits. Input may need to be split into smaller chunks." - ) from e - - except APIConnectionError as e: - last_exception = e - status_code = getattr(e, "status_code", None) or getattr( - getattr(e, "response", None), "status_code", None - ) - logger.warning(f"APIConnectionError (HTTP {status_code}), attempt {attempt + 1}: {str(e)[:200]}") - if attempt < max_retries: - backoff = min(initial_backoff * (2**attempt), max_backoff) - await asyncio.sleep(backoff) - continue - else: - logger.error(f"Connection error after {max_retries + 1} attempts: {str(e)}") - raise - - except APIStatusError as e: - # Fast fail only on 401 (unauthorized) and 403 (forbidden) - these won't recover with retries - if e.status_code in (401, 403): - logger.error(f"Auth error (HTTP {e.status_code}), not retrying: {str(e)}") - raise - - # Handle tool_use_failed error - model outputted in tool call format - # Convert to expected JSON format and continue - if e.status_code == 400 and response_format is not None: - try: - error_body = e.body if hasattr(e, "body") else {} - if isinstance(error_body, dict): - error_info: dict[str, Any] = error_body.get("error") or {} - if error_info.get("code") == "tool_use_failed": - failed_gen = error_info.get("failed_generation", "") - if failed_gen: - # Parse the tool call format and convert to actions format - tool_call = json.loads(failed_gen) - tool_name = tool_call.get("name", "") - tool_args = tool_call.get("arguments", {}) - # Convert to actions format: {"actions": [{"tool": "name", ...args}]} - converted = {"actions": [{"tool": tool_name, **tool_args}]} - if skip_validation: - result = converted - else: - result = response_format.model_validate(converted) - - # Record metrics for this successful recovery - duration = time.time() - start_time - metrics = get_metrics_collector() - metrics.record_llm_call( - provider=self.provider, - model=self.model, - scope=scope, - duration=duration, - input_tokens=0, - output_tokens=0, - success=True, - ) - if return_usage: - return result, TokenUsage(input_tokens=0, output_tokens=0, total_tokens=0) - return result - except (json.JSONDecodeError, KeyError, TypeError): - pass # Failed to parse tool_use_failed, continue with normal retry - - last_exception = e - if attempt < max_retries: - backoff = min(initial_backoff * (2**attempt), max_backoff) - jitter = backoff * 0.2 * (2 * (time.time() % 1) - 1) - sleep_time = backoff + jitter - await asyncio.sleep(sleep_time) - else: - logger.error(f"API error after {max_retries + 1} attempts: {str(e)}") - raise - - except Exception: - raise - - if last_exception: - raise last_exception - raise RuntimeError("LLM call failed after all retries with no exception captured") + return result async def call_with_tools( self, @@ -636,940 +418,122 @@ class LLMProvider: Returns: LLMToolCallResult with content and/or tool_calls. """ - from .response_models import LLMToolCall, LLMToolCallResult - async with _global_llm_semaphore: - start_time = time.time() + # Delegate to provider implementation + result = await self._provider_impl.call_with_tools( + messages=messages, + tools=tools, + max_completion_tokens=max_completion_tokens, + temperature=temperature, + scope=scope, + max_retries=max_retries, + initial_backoff=initial_backoff, + max_backoff=max_backoff, + tool_choice=tool_choice, + ) - # Handle Mock provider + # Backward compatibility: Update mock call tracking for mock provider + # This allows existing tests using LLMProvider._mock_calls to continue working if self.provider == "mock": - return await self._call_with_tools_mock(messages, tools, scope) + from .providers.mock_llm import MockLLM - # Handle Anthropic separately (uses different tool format) - if self.provider == "anthropic": - return await self._call_with_tools_anthropic( - messages, tools, max_completion_tokens, max_retries, initial_backoff, max_backoff, start_time, scope - ) + if isinstance(self._provider_impl, MockLLM): + # Sync the mock calls from provider implementation to wrapper + self._mock_calls = self._provider_impl.get_mock_calls() - # Handle Gemini and Vertex AI (convert to Gemini tool format) - if self.provider in ("gemini", "vertexai"): - return await self._call_with_tools_gemini( - messages, tools, max_retries, initial_backoff, max_backoff, start_time, scope - ) - - # OpenAI-compatible providers (OpenAI, Groq, Ollama, LMStudio) - call_params: dict[str, Any] = { - "model": self.model, - "messages": messages, - "tools": tools, - "tool_choice": tool_choice, - } - - if max_completion_tokens is not None: - call_params["max_completion_tokens"] = max_completion_tokens - if temperature is not None: - call_params["temperature"] = temperature - - # Provider-specific parameters - if self.provider == "groq": - call_params["seed"] = DEFAULT_LLM_SEED - - last_exception = None - - for attempt in range(max_retries + 1): - try: - response = await self._client.chat.completions.create(**call_params) - - message = response.choices[0].message - finish_reason = response.choices[0].finish_reason - - # Extract tool calls if present - tool_calls: list[LLMToolCall] = [] - if message.tool_calls: - for tc in message.tool_calls: - try: - args = json.loads(tc.function.arguments) if tc.function.arguments else {} - except json.JSONDecodeError: - args = {"_raw": tc.function.arguments} - tool_calls.append(LLMToolCall(id=tc.id, name=tc.function.name, arguments=args)) - - content = message.content - - # Record metrics - duration = time.time() - start_time - usage = response.usage - input_tokens = usage.prompt_tokens or 0 if usage else 0 - output_tokens = usage.completion_tokens or 0 if usage else 0 - - metrics = get_metrics_collector() - metrics.record_llm_call( - provider=self.provider, - model=self.model, - scope=scope, - duration=duration, - input_tokens=input_tokens, - output_tokens=output_tokens, - success=True, - ) - - return LLMToolCallResult( - content=content, - tool_calls=tool_calls, - finish_reason=finish_reason, - input_tokens=input_tokens, - output_tokens=output_tokens, - ) - - except APIConnectionError as e: - last_exception = e - if attempt < max_retries: - await asyncio.sleep(min(initial_backoff * (2**attempt), max_backoff)) - continue - raise - - except APIStatusError as e: - if e.status_code in (401, 403): - raise - last_exception = e - if attempt < max_retries: - await asyncio.sleep(min(initial_backoff * (2**attempt), max_backoff)) - continue - raise - - except Exception: - raise - - if last_exception: - raise last_exception - raise RuntimeError("Tool call failed after all retries") - - async def _call_with_tools_mock( - self, - messages: list[dict[str, Any]], - tools: list[dict[str, Any]], - scope: str, - ) -> "LLMToolCallResult": - """Handle mock tool calls for testing.""" - from .response_models import LLMToolCallResult - - call_record = { - "provider": self.provider, - "model": self.model, - "messages": messages, - "tools": [t.get("function", {}).get("name") for t in tools], - "scope": scope, - } - self._mock_calls.append(call_record) - - if self._mock_response is not None: - if isinstance(self._mock_response, LLMToolCallResult): - return self._mock_response - # Allow setting just tool calls as a list - if isinstance(self._mock_response, list): - from .response_models import LLMToolCall - - return LLMToolCallResult( - tool_calls=[ - LLMToolCall(id=f"mock_{i}", name=tc["name"], arguments=tc.get("arguments", {})) - for i, tc in enumerate(self._mock_response) - ], - finish_reason="tool_calls", - ) - - return LLMToolCallResult(content="mock response", finish_reason="stop") - - async def _call_with_tools_anthropic( - self, - messages: list[dict[str, Any]], - tools: list[dict[str, Any]], - max_completion_tokens: int | None, - max_retries: int, - initial_backoff: float, - max_backoff: float, - start_time: float, - scope: str, - ) -> "LLMToolCallResult": - """Handle Anthropic tool calling.""" - from anthropic import APIConnectionError, APIStatusError - - from .response_models import LLMToolCall, LLMToolCallResult - - # Convert OpenAI tool format to Anthropic format - anthropic_tools = [] - for tool in tools: - func = tool.get("function", {}) - anthropic_tools.append( - { - "name": func.get("name", ""), - "description": func.get("description", ""), - "input_schema": func.get("parameters", {"type": "object", "properties": {}}), - } - ) - - # Convert messages - handle tool results - system_prompt = None - anthropic_messages = [] - for msg in messages: - role = msg.get("role", "user") - content = msg.get("content", "") - - if role == "system": - system_prompt = (system_prompt + "\n\n" + content) if system_prompt else content - elif role == "tool": - # Anthropic uses tool_result blocks - anthropic_messages.append( - { - "role": "user", - "content": [ - {"type": "tool_result", "tool_use_id": msg.get("tool_call_id", ""), "content": content} - ], - } - ) - elif role == "assistant" and msg.get("tool_calls"): - # Convert assistant tool calls - tool_use_blocks = [] - for tc in msg["tool_calls"]: - tool_use_blocks.append( - { - "type": "tool_use", - "id": tc.get("id", ""), - "name": tc.get("function", {}).get("name", ""), - "input": json.loads(tc.get("function", {}).get("arguments", "{}")), - } - ) - anthropic_messages.append({"role": "assistant", "content": tool_use_blocks}) - else: - anthropic_messages.append({"role": role, "content": content}) - - call_params: dict[str, Any] = { - "model": self.model, - "messages": anthropic_messages, - "tools": anthropic_tools, - "max_tokens": max_completion_tokens or 4096, - } - if system_prompt: - call_params["system"] = system_prompt - - last_exception = None - for attempt in range(max_retries + 1): - try: - response = await self._anthropic_client.messages.create(**call_params) - - # Extract content and tool calls - content_parts = [] - tool_calls: list[LLMToolCall] = [] - - for block in response.content: - if block.type == "text": - content_parts.append(block.text) - elif block.type == "tool_use": - tool_calls.append(LLMToolCall(id=block.id, name=block.name, arguments=block.input or {})) - - content = "".join(content_parts) if content_parts else None - finish_reason = "tool_calls" if tool_calls else "stop" - - # Extract token usage - input_tokens = response.usage.input_tokens or 0 - output_tokens = response.usage.output_tokens or 0 - - # Record metrics - metrics = get_metrics_collector() - metrics.record_llm_call( - provider=self.provider, - model=self.model, - scope=scope, - duration=time.time() - start_time, - input_tokens=input_tokens, - output_tokens=output_tokens, - success=True, - ) - - return LLMToolCallResult( - content=content, - tool_calls=tool_calls, - finish_reason=finish_reason, - input_tokens=input_tokens, - output_tokens=output_tokens, - ) - - except (APIConnectionError, APIStatusError) as e: - if isinstance(e, APIStatusError) and e.status_code in (401, 403): - raise - last_exception = e - if attempt < max_retries: - await asyncio.sleep(min(initial_backoff * (2**attempt), max_backoff)) - continue - raise - - if last_exception: - raise last_exception - raise RuntimeError("Anthropic tool call failed") - - async def _call_with_tools_gemini( - self, - messages: list[dict[str, Any]], - tools: list[dict[str, Any]], - max_retries: int, - initial_backoff: float, - max_backoff: float, - start_time: float, - scope: str, - ) -> "LLMToolCallResult": - """Handle Gemini tool calling.""" - from .response_models import LLMToolCall, LLMToolCallResult - - # Convert tools to Gemini format - gemini_tools = [] - for tool in tools: - func = tool.get("function", {}) - gemini_tools.append( - genai_types.Tool( - function_declarations=[ - genai_types.FunctionDeclaration( - name=func.get("name", ""), - description=func.get("description", ""), - parameters=func.get("parameters"), - ) - ] - ) - ) - - # Convert messages - system_instruction = None - gemini_contents = [] - for msg in messages: - role = msg.get("role", "user") - content = msg.get("content", "") - - if role == "system": - system_instruction = (system_instruction + "\n\n" + content) if system_instruction else content - elif role == "tool": - # Gemini uses function_response - gemini_contents.append( - genai_types.Content( - role="user", - parts=[ - genai_types.Part( - function_response=genai_types.FunctionResponse( - name=msg.get("name", ""), - response={"result": content}, - ) - ) - ], - ) - ) - elif role == "assistant": - gemini_contents.append(genai_types.Content(role="model", parts=[genai_types.Part(text=content)])) - else: - gemini_contents.append(genai_types.Content(role="user", parts=[genai_types.Part(text=content)])) - - config = genai_types.GenerateContentConfig( - system_instruction=system_instruction, - tools=gemini_tools, - ) - - last_exception = None - for attempt in range(max_retries + 1): - try: - response = await self._gemini_client.aio.models.generate_content( - model=self.model, - contents=gemini_contents, - config=config, - ) - - # Extract content and tool calls - content = None - tool_calls: list[LLMToolCall] = [] - - if response.candidates and response.candidates[0].content: - parts = response.candidates[0].content.parts - if parts: - for part in parts: - if hasattr(part, "text") and part.text: - content = part.text - if hasattr(part, "function_call") and part.function_call: - fc = part.function_call - tool_calls.append( - LLMToolCall( - id=f"gemini_{len(tool_calls)}", - name=fc.name, - arguments=dict(fc.args) if fc.args else {}, - ) - ) - - finish_reason = "tool_calls" if tool_calls else "stop" - - # Record metrics - metrics = get_metrics_collector() - input_tokens = response.usage_metadata.prompt_token_count if response.usage_metadata else 0 - output_tokens = response.usage_metadata.candidates_token_count if response.usage_metadata else 0 - metrics.record_llm_call( - provider=self.provider, - model=self.model, - scope=scope, - duration=time.time() - start_time, - input_tokens=input_tokens, - output_tokens=output_tokens, - success=True, - ) - - return LLMToolCallResult( - content=content, - tool_calls=tool_calls, - finish_reason=finish_reason, - input_tokens=input_tokens, - output_tokens=output_tokens, - ) - - except genai_errors.APIError as e: - if e.code in (401, 403): - raise - last_exception = e - if attempt < max_retries: - await asyncio.sleep(min(initial_backoff * (2**attempt), max_backoff)) - continue - raise - - if last_exception: - raise last_exception - raise RuntimeError("Gemini tool call failed") - - async def _call_anthropic( - self, - messages: list[dict[str, str]], - response_format: Any | None, - max_completion_tokens: int | None, - max_retries: int, - initial_backoff: float, - max_backoff: float, - skip_validation: bool, - start_time: float, - scope: str = "memory", - return_usage: bool = False, - semaphore_wait_time: float = 0.0, - ) -> Any: - """Handle Anthropic-specific API calls.""" - from anthropic import APIConnectionError, APIStatusError, RateLimitError - - # Convert OpenAI-style messages to Anthropic format - system_prompt = None - anthropic_messages = [] - - for msg in messages: - role = msg.get("role", "user") - content = msg.get("content", "") - - if role == "system": - if system_prompt: - system_prompt += "\n\n" + content - else: - system_prompt = content - else: - anthropic_messages.append({"role": role, "content": content}) - - # Add JSON schema instruction if response_format is provided - if response_format is not None and hasattr(response_format, "model_json_schema"): - schema = response_format.model_json_schema() - schema_msg = f"\n\nYou must respond with valid JSON matching this schema:\n{json.dumps(schema, indent=2)}" - if system_prompt: - system_prompt += schema_msg - else: - system_prompt = schema_msg - - # Prepare parameters - call_params = { - "model": self.model, - "messages": anthropic_messages, - "max_tokens": max_completion_tokens if max_completion_tokens is not None else 4096, - } - - if system_prompt: - call_params["system"] = system_prompt - - last_exception = None - - for attempt in range(max_retries + 1): - try: - response = await self._anthropic_client.messages.create(**call_params) - - # Anthropic response content is a list of blocks - content = "" - for block in response.content: - if block.type == "text": - content += block.text - - if response_format is not None: - # Models may wrap JSON in markdown code blocks - clean_content = content - if "```json" in content: - clean_content = content.split("```json")[1].split("```")[0].strip() - elif "```" in content: - clean_content = content.split("```")[1].split("```")[0].strip() - - try: - json_data = json.loads(clean_content) - except json.JSONDecodeError: - # Fallback to parsing raw content if markdown stripping failed - json_data = json.loads(content) - - if skip_validation: - result = json_data - else: - result = response_format.model_validate(json_data) - else: - result = content - - # Record metrics and log slow calls - duration = time.time() - start_time - input_tokens = response.usage.input_tokens or 0 if response.usage else 0 - output_tokens = response.usage.output_tokens or 0 if response.usage else 0 - total_tokens = input_tokens + output_tokens - - # Record LLM metrics - metrics = get_metrics_collector() - metrics.record_llm_call( - provider=self.provider, - model=self.model, - scope=scope, - duration=duration, - input_tokens=input_tokens, - output_tokens=output_tokens, - success=True, - ) - - # Log slow calls - if duration > 10.0: - wait_info = f", wait={semaphore_wait_time:.3f}s" if semaphore_wait_time > 0.1 else "" - logger.info( - f"slow llm call: scope={scope}, model={self.provider}/{self.model}, " - f"input_tokens={input_tokens}, output_tokens={output_tokens}, " - f"time={duration:.3f}s{wait_info}" - ) - - if return_usage: - token_usage = TokenUsage( - input_tokens=input_tokens, - output_tokens=output_tokens, - total_tokens=total_tokens, - ) - return result, token_usage - return result - - except json.JSONDecodeError as e: - last_exception = e - if attempt < max_retries: - logger.warning("Anthropic returned invalid JSON, retrying...") - backoff = min(initial_backoff * (2**attempt), max_backoff) - await asyncio.sleep(backoff) - continue - else: - logger.error(f"Anthropic returned invalid JSON after {max_retries + 1} attempts") - raise - - except (APIConnectionError, RateLimitError, APIStatusError) as e: - # Fast fail on 401/403 - if isinstance(e, APIStatusError) and e.status_code in (401, 403): - logger.error(f"Anthropic auth error (HTTP {e.status_code}), not retrying: {str(e)}") - raise - - last_exception = e - if attempt < max_retries: - # Check if it's a rate limit or server error - should_retry = isinstance(e, (APIConnectionError, RateLimitError)) or ( - isinstance(e, APIStatusError) and e.status_code >= 500 - ) - - if should_retry: - backoff = min(initial_backoff * (2**attempt), max_backoff) - jitter = backoff * 0.2 * (2 * (time.time() % 1) - 1) - await asyncio.sleep(backoff + jitter) - continue - - logger.error(f"Anthropic API error after {max_retries + 1} attempts: {str(e)}") - raise - - except Exception as e: - logger.error(f"Unexpected error during Anthropic call: {type(e).__name__}: {str(e)}") - raise - - if last_exception: - raise last_exception - raise RuntimeError("Anthropic call failed after all retries") - - async def _call_ollama_native( - self, - messages: list[dict[str, str]], - response_format: Any, - max_completion_tokens: int | None, - temperature: float | None, - max_retries: int, - initial_backoff: float, - max_backoff: float, - skip_validation: bool, - start_time: float, - scope: str = "memory", - return_usage: bool = False, - semaphore_wait_time: float = 0.0, - ) -> Any: - """ - Call Ollama using native API with JSON schema enforcement. - - Ollama's native API supports passing a full JSON schema in the 'format' parameter, - which provides better structured output control than the OpenAI-compatible API. - """ - # Get the JSON schema from the Pydantic model - schema = response_format.model_json_schema() if hasattr(response_format, "model_json_schema") else None - - # Build the base URL for Ollama's native API - # Default OpenAI-compatible URL is http://localhost:11434/v1 - # Native API is at http://localhost:11434/api/chat - base_url = self.base_url or "http://localhost:11434/v1" - if base_url.endswith("/v1"): - native_url = base_url[:-3] + "/api/chat" - else: - native_url = base_url.rstrip("/") + "/api/chat" - - # Build request payload - payload = { - "model": self.model, - "messages": messages, - "stream": False, - } - - # Add schema as format parameter for structured output - if schema: - payload["format"] = schema - - # Add optional parameters with optimized defaults for Ollama - # Benchmarking shows num_ctx=16384 + num_batch=512 is optimal - options = { - "num_ctx": 16384, # 16k context window for larger prompts - "num_batch": 512, # Optimal batch size for prompt processing - } - if max_completion_tokens: - options["num_predict"] = max_completion_tokens - if temperature is not None: - options["temperature"] = temperature - payload["options"] = options - - last_exception = None - - async with httpx.AsyncClient(timeout=300.0) as client: - for attempt in range(max_retries + 1): - try: - response = await client.post(native_url, json=payload) - response.raise_for_status() - - result = response.json() - content = result.get("message", {}).get("content", "") - - # Parse JSON response - try: - json_data = json.loads(content) - except json.JSONDecodeError as json_err: - content_preview = content[:500] if content else "" - if content and len(content) > 700: - content_preview = f"{content[:500]}...TRUNCATED...{content[-200:]}" - logger.warning( - f"Ollama JSON parse error (attempt {attempt + 1}/{max_retries + 1}): {json_err}\n" - f" Model: ollama/{self.model}\n" - f" Content length: {len(content) if content else 0} chars\n" - f" Content preview: {content_preview!r}" - ) - if attempt < max_retries: - backoff = min(initial_backoff * (2**attempt), max_backoff) - await asyncio.sleep(backoff) - last_exception = json_err - continue - else: - raise - - # Extract token usage from Ollama response - # Ollama returns prompt_eval_count (input) and eval_count (output) - duration = time.time() - start_time - input_tokens = result.get("prompt_eval_count", 0) or 0 - output_tokens = result.get("eval_count", 0) or 0 - total_tokens = input_tokens + output_tokens - - # Record LLM metrics - metrics = get_metrics_collector() - metrics.record_llm_call( - provider=self.provider, - model=self.model, - scope=scope, - duration=duration, - input_tokens=input_tokens, - output_tokens=output_tokens, - success=True, - ) - - # Validate against Pydantic model or return raw JSON - if skip_validation: - validated_result = json_data - else: - validated_result = response_format.model_validate(json_data) - - if return_usage: - token_usage = TokenUsage( - input_tokens=input_tokens, - output_tokens=output_tokens, - total_tokens=total_tokens, - ) - return validated_result, token_usage - return validated_result - - except httpx.HTTPStatusError as e: - last_exception = e - if attempt < max_retries: - logger.warning( - f"Ollama HTTP error (attempt {attempt + 1}/{max_retries + 1}): {e.response.status_code}" - ) - backoff = min(initial_backoff * (2**attempt), max_backoff) - await asyncio.sleep(backoff) - continue - else: - logger.error(f"Ollama HTTP error after {max_retries + 1} attempts: {e}") - raise - - except httpx.RequestError as e: - last_exception = e - if attempt < max_retries: - logger.warning(f"Ollama connection error (attempt {attempt + 1}/{max_retries + 1}): {e}") - backoff = min(initial_backoff * (2**attempt), max_backoff) - await asyncio.sleep(backoff) - continue - else: - logger.error(f"Ollama connection error after {max_retries + 1} attempts: {e}") - raise - - except Exception as e: - logger.error(f"Unexpected error during Ollama call: {type(e).__name__}: {e}") - raise - - if last_exception: - raise last_exception - raise RuntimeError("Ollama call failed after all retries") - - async def _call_gemini( - self, - messages: list[dict[str, str]], - response_format: Any | None, - max_retries: int, - initial_backoff: float, - max_backoff: float, - skip_validation: bool, - start_time: float, - scope: str = "memory", - return_usage: bool = False, - semaphore_wait_time: float = 0.0, - ) -> Any: - """Handle Gemini-specific API calls.""" - # Convert OpenAI-style messages to Gemini format - system_instruction = None - gemini_contents = [] - - for msg in messages: - role = msg.get("role", "user") - content = msg.get("content", "") - - if role == "system": - if system_instruction: - system_instruction += "\n\n" + content - else: - system_instruction = content - elif role == "assistant": - gemini_contents.append(genai_types.Content(role="model", parts=[genai_types.Part(text=content)])) - else: - gemini_contents.append(genai_types.Content(role="user", parts=[genai_types.Part(text=content)])) - - # Add JSON schema instruction if response_format is provided - if response_format is not None and hasattr(response_format, "model_json_schema"): - schema = response_format.model_json_schema() - schema_msg = f"\n\nYou must respond with valid JSON matching this schema:\n{json.dumps(schema, indent=2)}" - if system_instruction: - system_instruction += schema_msg - else: - system_instruction = schema_msg - - # Build generation config - config_kwargs = {} - if system_instruction: - config_kwargs["system_instruction"] = system_instruction - if response_format is not None: - config_kwargs["response_mime_type"] = "application/json" - config_kwargs["response_schema"] = response_format - - generation_config = genai_types.GenerateContentConfig(**config_kwargs) if config_kwargs else None - - last_exception = None - - for attempt in range(max_retries + 1): - try: - response = await self._gemini_client.aio.models.generate_content( - model=self.model, - contents=gemini_contents, - config=generation_config, - ) - - content = response.text - - # Handle empty response - if content is None: - block_reason = None - if hasattr(response, "candidates") and response.candidates: - candidate = response.candidates[0] - if hasattr(candidate, "finish_reason"): - block_reason = candidate.finish_reason - - if attempt < max_retries: - logger.warning(f"Gemini returned empty response (reason: {block_reason}), retrying...") - backoff = min(initial_backoff * (2**attempt), max_backoff) - await asyncio.sleep(backoff) - continue - else: - raise RuntimeError(f"Gemini returned empty response after {max_retries + 1} attempts") - - if response_format is not None: - json_data = json.loads(content) - if skip_validation: - result = json_data - else: - result = response_format.model_validate(json_data) - else: - result = content - - # Record metrics and log slow calls - duration = time.time() - start_time - input_tokens = 0 - output_tokens = 0 - if hasattr(response, "usage_metadata") and response.usage_metadata: - usage = response.usage_metadata - input_tokens = usage.prompt_token_count or 0 - output_tokens = usage.candidates_token_count or 0 - - # Record LLM metrics - metrics = get_metrics_collector() - metrics.record_llm_call( - provider=self.provider, - model=self.model, - scope=scope, - duration=duration, - input_tokens=input_tokens, - output_tokens=output_tokens, - success=True, - ) - - # Log slow calls - if duration > 10.0 and input_tokens > 0: - wait_info = f", wait={semaphore_wait_time:.3f}s" if semaphore_wait_time > 0.1 else "" - logger.info( - f"slow llm call: scope={scope}, model={self.provider}/{self.model}, " - f"input_tokens={input_tokens}, output_tokens={output_tokens}, " - f"time={duration:.3f}s{wait_info}" - ) - - if return_usage: - token_usage = TokenUsage( - input_tokens=input_tokens, - output_tokens=output_tokens, - total_tokens=input_tokens + output_tokens, - ) - return result, token_usage - return result - - except json.JSONDecodeError as e: - last_exception = e - if attempt < max_retries: - logger.warning("Gemini returned invalid JSON, retrying...") - backoff = min(initial_backoff * (2**attempt), max_backoff) - await asyncio.sleep(backoff) - continue - else: - logger.error(f"Gemini returned invalid JSON after {max_retries + 1} attempts") - raise - - except genai_errors.APIError as e: - # Fast fail only on 401 (unauthorized) and 403 (forbidden) - these won't recover with retries - if e.code in (401, 403): - logger.error(f"Gemini auth error (HTTP {e.code}), not retrying: {str(e)}") - raise - - # Retry on retryable errors (rate limits, server errors, and other client errors like 400) - if e.code in (400, 429, 500, 502, 503, 504) or (e.code and e.code >= 500): - last_exception = e - if attempt < max_retries: - backoff = min(initial_backoff * (2**attempt), max_backoff) - jitter = backoff * 0.2 * (2 * (time.time() % 1) - 1) - await asyncio.sleep(backoff + jitter) - else: - logger.error(f"Gemini API error after {max_retries + 1} attempts: {str(e)}") - raise - else: - logger.error(f"Gemini API error: {type(e).__name__}: {str(e)}") - raise - - except Exception as e: - logger.error(f"Unexpected error during Gemini call: {type(e).__name__}: {str(e)}") - raise - - if last_exception: - raise last_exception - raise RuntimeError("Gemini call failed after all retries") - - async def _call_mock( - self, - messages: list[dict[str, str]], - response_format: Any | None, - scope: str, - return_usage: bool, - ) -> Any: - """ - Handle mock provider calls for testing. - - Records the call and returns a configurable mock response. - """ - # Record the call for test verification - call_record = { - "provider": self.provider, - "model": self.model, - "messages": messages, - "response_format": response_format.__name__ - if response_format and hasattr(response_format, "__name__") - else str(response_format), - "scope": scope, - } - self._mock_calls.append(call_record) - logger.debug(f"Mock LLM call recorded: scope={scope}, model={self.model}") - - # Return mock response - if self._mock_response is not None: - result = self._mock_response - elif response_format is not None: - # Try to create a minimal valid instance of the response format - try: - # For Pydantic models, try to create with minimal valid data - result = {"mock": True} - except Exception: - result = {"mock": True} - else: - result = "mock response" - - if return_usage: - token_usage = TokenUsage(input_tokens=10, output_tokens=5, total_tokens=15) - return result, token_usage - return result + return result def set_mock_response(self, response: Any) -> None: """Set the response to return from mock calls.""" + # Backward compatibility: Store in both wrapper and provider implementation self._mock_response = response + if self.provider == "mock": + from .providers.mock_llm import MockLLM + + if isinstance(self._provider_impl, MockLLM): + self._provider_impl.set_mock_response(response) def get_mock_calls(self) -> list[dict]: """Get the list of recorded mock calls.""" + # Backward compatibility: Read from provider implementation if mock provider + if self.provider == "mock": + from .providers.mock_llm import MockLLM + + if isinstance(self._provider_impl, MockLLM): + return self._provider_impl.get_mock_calls() return self._mock_calls def clear_mock_calls(self) -> None: """Clear the recorded mock calls.""" + # Backward compatibility: Clear in both wrapper and provider implementation self._mock_calls = [] + if self.provider == "mock": + from .providers.mock_llm import MockLLM + + if isinstance(self._provider_impl, MockLLM): + self._provider_impl.clear_mock_calls() + + def _load_codex_auth(self) -> tuple[str, str]: + """ + Load OAuth credentials from ~/.codex/auth.json. + + Returns: + Tuple of (access_token, account_id). + + Raises: + FileNotFoundError: If auth file doesn't exist. + ValueError: If auth file is invalid. + """ + auth_file = Path.home() / ".codex" / "auth.json" + + if not auth_file.exists(): + raise FileNotFoundError( + f"Codex auth file not found: {auth_file}\nRun 'codex auth login' to authenticate with ChatGPT Plus/Pro." + ) + + with open(auth_file) as f: + data = json.load(f) + + # Validate auth structure + auth_mode = data.get("auth_mode") + if auth_mode != "chatgpt": + raise ValueError(f"Expected auth_mode='chatgpt', got: {auth_mode}") + + tokens = data.get("tokens", {}) + access_token = tokens.get("access_token") + account_id = tokens.get("account_id") + + if not access_token: + raise ValueError("No access_token found in Codex auth file. Run 'codex auth login' again.") + + return access_token, account_id + + def _verify_claude_code_available(self) -> None: + """ + Verify that Claude Agent SDK can be imported and is properly configured. + + Raises: + ImportError: If Claude Agent SDK is not installed. + RuntimeError: If Claude Code is not authenticated. + """ + try: + # Import Claude Agent SDK + # Reduce Claude Agent SDK logging verbosity + import logging as sdk_logging + + from claude_agent_sdk import query # noqa: F401 + + sdk_logging.getLogger("claude_agent_sdk").setLevel(sdk_logging.WARNING) + sdk_logging.getLogger("claude_agent_sdk._internal").setLevel(sdk_logging.WARNING) + + logger.debug("Claude Agent SDK imported successfully") + except ImportError as e: + raise ImportError( + "Claude Agent SDK not installed. Run: uv add claude-agent-sdk or pip install claude-agent-sdk" + ) from e + + # SDK will automatically check for authentication when first used + # No need to verify here - let it fail gracefully on first call with helpful error async def cleanup(self) -> None: """Clean up resources.""" @@ -1579,9 +543,14 @@ class LLMProvider: def for_memory(cls) -> "LLMProvider": """Create provider for memory operations from environment variables.""" provider = os.getenv("HINDSIGHT_API_LLM_PROVIDER", "groq") - api_key = os.getenv("HINDSIGHT_API_LLM_API_KEY") - if not api_key: - raise ValueError("HINDSIGHT_API_LLM_API_KEY environment variable is required") + api_key = os.getenv("HINDSIGHT_API_LLM_API_KEY", "") + + # API key not needed for openai-codex (uses OAuth) or claude-code (uses Keychain OAuth) + if not api_key and provider not in ("openai-codex", "claude-code"): + raise ValueError( + "HINDSIGHT_API_LLM_API_KEY environment variable is required (unless using openai-codex or claude-code)" + ) + base_url = os.getenv("HINDSIGHT_API_LLM_BASE_URL", "") model = os.getenv("HINDSIGHT_API_LLM_MODEL", "openai/gpt-oss-120b") @@ -1591,11 +560,15 @@ class LLMProvider: def for_answer_generation(cls) -> "LLMProvider": """Create provider for answer generation. Falls back to memory config if not set.""" provider = os.getenv("HINDSIGHT_API_ANSWER_LLM_PROVIDER", os.getenv("HINDSIGHT_API_LLM_PROVIDER", "groq")) - api_key = os.getenv("HINDSIGHT_API_ANSWER_LLM_API_KEY", os.getenv("HINDSIGHT_API_LLM_API_KEY")) - if not api_key: + api_key = os.getenv("HINDSIGHT_API_ANSWER_LLM_API_KEY", os.getenv("HINDSIGHT_API_LLM_API_KEY", "")) + + # API key not needed for openai-codex (uses OAuth) or claude-code (uses Keychain OAuth) + if not api_key and provider not in ("openai-codex", "claude-code"): raise ValueError( - "HINDSIGHT_API_LLM_API_KEY or HINDSIGHT_API_ANSWER_LLM_API_KEY environment variable is required" + "HINDSIGHT_API_LLM_API_KEY or HINDSIGHT_API_ANSWER_LLM_API_KEY environment variable is required " + "(unless using openai-codex or claude-code)" ) + base_url = os.getenv("HINDSIGHT_API_ANSWER_LLM_BASE_URL", os.getenv("HINDSIGHT_API_LLM_BASE_URL", "")) model = os.getenv("HINDSIGHT_API_ANSWER_LLM_MODEL", os.getenv("HINDSIGHT_API_LLM_MODEL", "openai/gpt-oss-120b")) @@ -1605,11 +578,15 @@ class LLMProvider: def for_judge(cls) -> "LLMProvider": """Create provider for judge/evaluator operations. Falls back to memory config if not set.""" provider = os.getenv("HINDSIGHT_API_JUDGE_LLM_PROVIDER", os.getenv("HINDSIGHT_API_LLM_PROVIDER", "groq")) - api_key = os.getenv("HINDSIGHT_API_JUDGE_LLM_API_KEY", os.getenv("HINDSIGHT_API_LLM_API_KEY")) - if not api_key: + api_key = os.getenv("HINDSIGHT_API_JUDGE_LLM_API_KEY", os.getenv("HINDSIGHT_API_LLM_API_KEY", "")) + + # API key not needed for openai-codex (uses OAuth) or claude-code (uses Keychain OAuth) + if not api_key and provider not in ("openai-codex", "claude-code"): raise ValueError( - "HINDSIGHT_API_LLM_API_KEY or HINDSIGHT_API_JUDGE_LLM_API_KEY environment variable is required" + "HINDSIGHT_API_LLM_API_KEY or HINDSIGHT_API_JUDGE_LLM_API_KEY environment variable is required " + "(unless using openai-codex or claude-code)" ) + base_url = os.getenv("HINDSIGHT_API_JUDGE_LLM_BASE_URL", os.getenv("HINDSIGHT_API_LLM_BASE_URL", "")) model = os.getenv("HINDSIGHT_API_JUDGE_LLM_MODEL", os.getenv("HINDSIGHT_API_LLM_MODEL", "openai/gpt-oss-120b")) diff --git a/hindsight-api/hindsight_api/engine/memory_engine.py b/hindsight-api/hindsight_api/engine/memory_engine.py index 2f8e4680..1df27b42 100644 --- a/hindsight-api/hindsight_api/engine/memory_engine.py +++ b/hindsight-api/hindsight_api/engine/memory_engine.py @@ -303,8 +303,10 @@ class MemoryEngine(MemoryEngineInterface): db_url = db_url or config.database_url memory_llm_provider = memory_llm_provider or config.llm_provider memory_llm_api_key = memory_llm_api_key or config.llm_api_key - # Ollama and mock don't require an API key - if not memory_llm_api_key and memory_llm_provider not in ("ollama", "mock"): + # Ollama, openai-codex, claude-code, and mock don't require an API key + # openai-codex uses OAuth tokens from ~/.codex/auth.json + # claude-code uses OAuth tokens from macOS Keychain + if not memory_llm_api_key and memory_llm_provider not in ("ollama", "openai-codex", "claude-code", "mock"): raise ValueError("LLM API key is required. Set HINDSIGHT_API_LLM_API_KEY environment variable.") memory_llm_model = memory_llm_model or config.llm_model memory_llm_base_url = memory_llm_base_url or config.get_llm_base_url() or None diff --git a/hindsight-api/hindsight_api/engine/providers/__init__.py b/hindsight-api/hindsight_api/engine/providers/__init__.py new file mode 100644 index 00000000..29a5effe --- /dev/null +++ b/hindsight-api/hindsight_api/engine/providers/__init__.py @@ -0,0 +1,14 @@ +""" +LLM provider implementations. + +This package contains concrete implementations of the LLMInterface for various providers. +""" + +from .anthropic_llm import AnthropicLLM +from .claude_code_llm import ClaudeCodeLLM +from .codex_llm import CodexLLM +from .gemini_llm import GeminiLLM +from .mock_llm import MockLLM +from .openai_compatible_llm import OpenAICompatibleLLM + +__all__ = ["AnthropicLLM", "ClaudeCodeLLM", "CodexLLM", "GeminiLLM", "MockLLM", "OpenAICompatibleLLM"] diff --git a/hindsight-api/hindsight_api/engine/providers/anthropic_llm.py b/hindsight-api/hindsight_api/engine/providers/anthropic_llm.py new file mode 100644 index 00000000..73258f78 --- /dev/null +++ b/hindsight-api/hindsight_api/engine/providers/anthropic_llm.py @@ -0,0 +1,434 @@ +""" +Anthropic LLM provider using the Anthropic Python SDK. + +This provider enables using Claude models from Anthropic with support for: +- Structured JSON output +- Tool/function calling with proper format conversion +- Extended thinking mode +- Retry logic with exponential backoff +""" + +import asyncio +import json +import logging +import time +from typing import Any + +from hindsight_api.engine.llm_interface import LLMInterface, OutputTooLongError +from hindsight_api.engine.response_models import LLMToolCall, LLMToolCallResult, TokenUsage +from hindsight_api.metrics import get_metrics_collector + +logger = logging.getLogger(__name__) + + +class AnthropicLLM(LLMInterface): + """ + LLM provider using Anthropic's Claude models. + + Supports structured output, tool calling, and extended thinking mode. + Handles format conversion between OpenAI-style messages and Anthropic's format. + """ + + def __init__( + self, + provider: str, + api_key: str, + base_url: str, + model: str, + reasoning_effort: str = "low", + timeout: float = 300.0, + **kwargs: Any, + ): + """ + Initialize Anthropic LLM provider. + + Args: + provider: Provider name (should be "anthropic"). + api_key: Anthropic API key. + base_url: Base URL for the API (optional, uses Anthropic default if empty). + model: Model name (e.g., "claude-sonnet-4-20250514"). + reasoning_effort: Reasoning effort level (not used by Anthropic). + timeout: Request timeout in seconds. + **kwargs: Additional provider-specific parameters. + """ + super().__init__(provider, api_key, base_url, model, reasoning_effort, **kwargs) + + if not self.api_key: + raise ValueError("API key is required for Anthropic provider") + + # Import and initialize Anthropic client + try: + from anthropic import AsyncAnthropic + + client_kwargs: dict[str, Any] = {"api_key": self.api_key} + if self.base_url: + client_kwargs["base_url"] = self.base_url + if timeout: + client_kwargs["timeout"] = timeout + + self._client = AsyncAnthropic(**client_kwargs) + logger.info(f"Anthropic client initialized for model: {self.model}") + except ImportError as e: + raise RuntimeError("Anthropic SDK not installed. Run: uv add anthropic or pip install anthropic") from e + + async def verify_connection(self) -> None: + """ + Verify that the Anthropic provider is configured correctly by making a simple test call. + + Raises: + RuntimeError: If the connection test fails. + """ + try: + test_messages = [{"role": "user", "content": "test"}] + await self.call( + messages=test_messages, + max_completion_tokens=10, + temperature=0.0, + scope="test", + max_retries=0, + ) + logger.info("Anthropic connection verified successfully") + except Exception as e: + logger.error(f"Anthropic connection verification failed: {e}") + raise RuntimeError(f"Failed to verify Anthropic connection: {e}") from e + + async def call( + self, + messages: list[dict[str, str]], + response_format: Any | None = None, + max_completion_tokens: int | None = None, + temperature: float | None = None, + scope: str = "memory", + max_retries: int = 10, + initial_backoff: float = 1.0, + max_backoff: float = 60.0, + skip_validation: bool = False, + strict_schema: bool = False, + return_usage: bool = False, + ) -> Any: + """ + Make an LLM API call with retry logic. + + Args: + messages: List of message dicts with 'role' and 'content'. + response_format: Optional Pydantic model for structured output. + max_completion_tokens: Maximum tokens in response. + temperature: Sampling temperature (0.0-2.0). + scope: Scope identifier for tracking. + max_retries: Maximum retry attempts. + initial_backoff: Initial backoff time in seconds. + max_backoff: Maximum backoff time in seconds. + skip_validation: Return raw JSON without Pydantic validation. + strict_schema: Use strict JSON schema enforcement (not supported by Anthropic). + return_usage: If True, return tuple (result, TokenUsage) instead of just result. + + Returns: + If return_usage=False: Parsed response if response_format is provided, otherwise text content. + If return_usage=True: Tuple of (result, TokenUsage) with token counts. + + Raises: + OutputTooLongError: If output exceeds token limits. + Exception: Re-raises API errors after retries exhausted. + """ + from anthropic import APIConnectionError, APIStatusError, RateLimitError + + start_time = time.time() + + # Convert OpenAI-style messages to Anthropic format + system_prompt = None + anthropic_messages = [] + + for msg in messages: + role = msg.get("role", "user") + content = msg.get("content", "") + + if role == "system": + if system_prompt: + system_prompt += "\n\n" + content + else: + system_prompt = content + else: + anthropic_messages.append({"role": role, "content": content}) + + # Add JSON schema instruction if response_format is provided + if response_format is not None and hasattr(response_format, "model_json_schema"): + schema = response_format.model_json_schema() + schema_msg = f"\n\nYou must respond with valid JSON matching this schema:\n{json.dumps(schema, indent=2)}" + if system_prompt: + system_prompt += schema_msg + else: + system_prompt = schema_msg + + # Prepare parameters + call_params: dict[str, Any] = { + "model": self.model, + "messages": anthropic_messages, + "max_tokens": max_completion_tokens if max_completion_tokens is not None else 4096, + } + + if system_prompt: + call_params["system"] = system_prompt + + if temperature is not None: + call_params["temperature"] = temperature + + last_exception = None + + for attempt in range(max_retries + 1): + try: + response = await self._client.messages.create(**call_params) + + # Anthropic response content is a list of blocks + content = "" + for block in response.content: + if block.type == "text": + content += block.text + + if response_format is not None: + # Models may wrap JSON in markdown code blocks + clean_content = content + if "```json" in content: + clean_content = content.split("```json")[1].split("```")[0].strip() + elif "```" in content: + clean_content = content.split("```")[1].split("```")[0].strip() + + try: + json_data = json.loads(clean_content) + except json.JSONDecodeError: + # Fallback to parsing raw content if markdown stripping failed + json_data = json.loads(content) + + if skip_validation: + result = json_data + else: + result = response_format.model_validate(json_data) + else: + result = content + + # Record metrics and log slow calls + duration = time.time() - start_time + input_tokens = response.usage.input_tokens or 0 if response.usage else 0 + output_tokens = response.usage.output_tokens or 0 if response.usage else 0 + total_tokens = input_tokens + output_tokens + + # Record LLM metrics + metrics = get_metrics_collector() + metrics.record_llm_call( + provider=self.provider, + model=self.model, + scope=scope, + duration=duration, + input_tokens=input_tokens, + output_tokens=output_tokens, + success=True, + ) + + # Log slow calls + if duration > 10.0: + logger.info( + f"slow llm call: scope={scope}, model={self.provider}/{self.model}, " + f"input_tokens={input_tokens}, output_tokens={output_tokens}, " + f"time={duration:.3f}s" + ) + + if return_usage: + token_usage = TokenUsage( + input_tokens=input_tokens, + output_tokens=output_tokens, + total_tokens=total_tokens, + ) + return result, token_usage + return result + + except json.JSONDecodeError as e: + last_exception = e + if attempt < max_retries: + logger.warning("Anthropic returned invalid JSON, retrying...") + backoff = min(initial_backoff * (2**attempt), max_backoff) + await asyncio.sleep(backoff) + continue + else: + logger.error(f"Anthropic returned invalid JSON after {max_retries + 1} attempts") + raise + + except (APIConnectionError, RateLimitError, APIStatusError) as e: + # Fast fail on 401/403 + if isinstance(e, APIStatusError) and e.status_code in (401, 403): + logger.error(f"Anthropic auth error (HTTP {e.status_code}), not retrying: {str(e)}") + raise + + last_exception = e + if attempt < max_retries: + # Check if it's a rate limit or server error + should_retry = isinstance(e, (APIConnectionError, RateLimitError)) or ( + isinstance(e, APIStatusError) and e.status_code >= 500 + ) + + if should_retry: + backoff = min(initial_backoff * (2**attempt), max_backoff) + jitter = backoff * 0.2 * (2 * (time.time() % 1) - 1) + await asyncio.sleep(backoff + jitter) + continue + + logger.error(f"Anthropic API error after {max_retries + 1} attempts: {str(e)}") + raise + + except Exception as e: + logger.error(f"Unexpected error during Anthropic call: {type(e).__name__}: {str(e)}") + raise + + if last_exception: + raise last_exception + raise RuntimeError("Anthropic call failed after all retries") + + async def call_with_tools( + self, + messages: list[dict[str, Any]], + tools: list[dict[str, Any]], + max_completion_tokens: int | None = None, + temperature: float | None = None, + scope: str = "tools", + max_retries: int = 5, + initial_backoff: float = 1.0, + max_backoff: float = 30.0, + tool_choice: str | dict[str, Any] = "auto", + ) -> LLMToolCallResult: + """ + Make an LLM API call with tool/function calling support. + + Args: + messages: List of message dicts. Can include tool results with role='tool'. + tools: List of tool definitions in OpenAI format. + max_completion_tokens: Maximum tokens in response. + temperature: Sampling temperature (0.0-2.0). + scope: Scope identifier for tracking. + max_retries: Maximum retry attempts. + initial_backoff: Initial backoff time in seconds. + max_backoff: Maximum backoff time in seconds. + tool_choice: How to choose tools - "auto", "none", "required", or specific function. + + Returns: + LLMToolCallResult with content and/or tool_calls. + """ + from anthropic import APIConnectionError, APIStatusError + + start_time = time.time() + + # Convert OpenAI tool format to Anthropic format + anthropic_tools = [] + for tool in tools: + func = tool.get("function", {}) + anthropic_tools.append( + { + "name": func.get("name", ""), + "description": func.get("description", ""), + "input_schema": func.get("parameters", {"type": "object", "properties": {}}), + } + ) + + # Convert messages - handle tool results + system_prompt = None + anthropic_messages = [] + for msg in messages: + role = msg.get("role", "user") + content = msg.get("content", "") + + if role == "system": + system_prompt = (system_prompt + "\n\n" + content) if system_prompt else content + elif role == "tool": + # Anthropic uses tool_result blocks + anthropic_messages.append( + { + "role": "user", + "content": [ + {"type": "tool_result", "tool_use_id": msg.get("tool_call_id", ""), "content": content} + ], + } + ) + elif role == "assistant" and msg.get("tool_calls"): + # Convert assistant tool calls + tool_use_blocks = [] + for tc in msg["tool_calls"]: + tool_use_blocks.append( + { + "type": "tool_use", + "id": tc.get("id", ""), + "name": tc.get("function", {}).get("name", ""), + "input": json.loads(tc.get("function", {}).get("arguments", "{}")), + } + ) + anthropic_messages.append({"role": "assistant", "content": tool_use_blocks}) + else: + anthropic_messages.append({"role": role, "content": content}) + + call_params: dict[str, Any] = { + "model": self.model, + "messages": anthropic_messages, + "tools": anthropic_tools, + "max_tokens": max_completion_tokens or 4096, + } + if system_prompt: + call_params["system"] = system_prompt + + if temperature is not None: + call_params["temperature"] = temperature + + last_exception = None + for attempt in range(max_retries + 1): + try: + response = await self._client.messages.create(**call_params) + + # Extract content and tool calls + content_parts = [] + tool_calls: list[LLMToolCall] = [] + + for block in response.content: + if block.type == "text": + content_parts.append(block.text) + elif block.type == "tool_use": + tool_calls.append(LLMToolCall(id=block.id, name=block.name, arguments=block.input or {})) + + content = "".join(content_parts) if content_parts else None + finish_reason = "tool_calls" if tool_calls else "stop" + + # Extract token usage + input_tokens = response.usage.input_tokens or 0 + output_tokens = response.usage.output_tokens or 0 + + # Record metrics + metrics = get_metrics_collector() + metrics.record_llm_call( + provider=self.provider, + model=self.model, + scope=scope, + duration=time.time() - start_time, + input_tokens=input_tokens, + output_tokens=output_tokens, + success=True, + ) + + return LLMToolCallResult( + content=content, + tool_calls=tool_calls, + finish_reason=finish_reason, + input_tokens=input_tokens, + output_tokens=output_tokens, + ) + + except (APIConnectionError, APIStatusError) as e: + if isinstance(e, APIStatusError) and e.status_code in (401, 403): + raise + last_exception = e + if attempt < max_retries: + await asyncio.sleep(min(initial_backoff * (2**attempt), max_backoff)) + continue + raise + + if last_exception: + raise last_exception + raise RuntimeError("Anthropic tool call failed") + + async def cleanup(self) -> None: + """Clean up resources (close Anthropic client connections).""" + if hasattr(self, "_client") and self._client: + await self._client.close() diff --git a/hindsight-api/hindsight_api/engine/providers/claude_code_llm.py b/hindsight-api/hindsight_api/engine/providers/claude_code_llm.py new file mode 100644 index 00000000..bf8f24fc --- /dev/null +++ b/hindsight-api/hindsight_api/engine/providers/claude_code_llm.py @@ -0,0 +1,352 @@ +""" +Claude Code LLM provider using Claude Agent SDK. + +This provider enables using Claude Pro/Max subscriptions for API calls +via the Claude CLI authentication. It uses the Claude Agent SDK which +automatically handles authentication via `claude auth login` credentials. +""" + +import asyncio +import json +import logging +import time +from typing import Any + +from hindsight_api.engine.llm_interface import LLMInterface, OutputTooLongError +from hindsight_api.engine.response_models import LLMToolCall, LLMToolCallResult, TokenUsage +from hindsight_api.metrics import get_metrics_collector + +logger = logging.getLogger(__name__) + + +class ClaudeCodeLLM(LLMInterface): + """ + LLM provider using Claude Code authentication. + + Authenticates using Claude Pro/Max credentials via `claude auth login` + and makes API calls through the Claude Agent SDK. + """ + + def __init__( + self, + provider: str, + api_key: str, # Will be ignored, uses CLI auth + base_url: str, + model: str, + reasoning_effort: str = "low", + **kwargs: Any, + ): + """Initialize Claude Code LLM provider.""" + super().__init__(provider, api_key, base_url, model, reasoning_effort, **kwargs) + + # Verify Claude Agent SDK is available + try: + self._verify_claude_code_available() + logger.info("Claude Code: Using Claude Agent SDK (authentication via claude auth login)") + except Exception as e: + raise RuntimeError( + f"Failed to initialize Claude Code provider: {e}\n\n" + "To set up Claude Code authentication:\n" + "1. Install Claude Code CLI: npm install -g @anthropics/claude-code\n" + "2. Login with your Pro/Max plan: claude auth login\n" + "3. Verify authentication: claude --version\n\n" + "Or use a different provider (anthropic, openai, gemini) with API keys." + ) from e + + # Metrics collector is imported at module level + + def _verify_claude_code_available(self) -> None: + """ + Verify that Claude Agent SDK can be imported and is properly configured. + + Raises: + ImportError: If Claude Agent SDK is not installed. + RuntimeError: If Claude Code is not authenticated. + """ + try: + # Import Claude Agent SDK + # Reduce Claude Agent SDK logging verbosity + import logging as sdk_logging + + from claude_agent_sdk import query # noqa: F401 + + sdk_logging.getLogger("claude_agent_sdk").setLevel(sdk_logging.WARNING) + sdk_logging.getLogger("claude_agent_sdk._internal").setLevel(sdk_logging.WARNING) + + logger.debug("Claude Agent SDK imported successfully") + except ImportError as e: + raise ImportError( + "Claude Agent SDK not installed. Run: uv add claude-agent-sdk or pip install claude-agent-sdk" + ) from e + + # SDK will automatically check for authentication when first used + # No need to verify here - let it fail gracefully on first call with helpful error + + async def verify_connection(self) -> None: + """ + Verify that the Claude Code provider is configured correctly by making a simple test call. + + Raises: + RuntimeError: If the connection test fails. + """ + try: + test_messages = [{"role": "user", "content": "test"}] + await self.call( + messages=test_messages, + max_completion_tokens=10, + temperature=0.0, + scope="test", + max_retries=0, + ) + logger.info("Claude Code connection verified successfully") + except Exception as e: + logger.error(f"Claude Code connection verification failed: {e}") + raise RuntimeError(f"Failed to verify Claude Code connection: {e}") from e + + async def call( + self, + messages: list[dict[str, str]], + response_format: Any | None = None, + max_completion_tokens: int | None = None, + temperature: float | None = None, + scope: str = "memory", + max_retries: int = 10, + initial_backoff: float = 1.0, + max_backoff: float = 60.0, + skip_validation: bool = False, + strict_schema: bool = False, + return_usage: bool = False, + ) -> Any: + """ + Make an LLM API call with retry logic. + + Args: + messages: List of message dicts with 'role' and 'content'. + response_format: Optional Pydantic model for structured output. + max_completion_tokens: Maximum tokens in response (ignored by Claude Agent SDK). + temperature: Sampling temperature (ignored by Claude Agent SDK). + scope: Scope identifier for tracking. + max_retries: Maximum retry attempts. + initial_backoff: Initial backoff time in seconds. + max_backoff: Maximum backoff time in seconds. + skip_validation: Return raw JSON without Pydantic validation. + strict_schema: Use strict JSON schema enforcement (not supported). + return_usage: If True, return tuple (result, TokenUsage) instead of just result. + + Returns: + If return_usage=False: Parsed response if response_format is provided, otherwise text content. + If return_usage=True: Tuple of (result, TokenUsage) with estimated token counts. + + Raises: + OutputTooLongError: If output exceeds token limits (not supported by Claude Agent SDK). + Exception: Re-raises API errors after retries exhausted. + """ + from claude_agent_sdk import AssistantMessage, ClaudeAgentOptions, TextBlock, query + + start_time = time.time() + + # Build system prompt + system_prompt = "" + user_content = "" + + for msg in messages: + role = msg.get("role", "user") + content = msg.get("content", "") + + if role == "system": + system_prompt += ("\n\n" + content) if system_prompt else content + elif role == "user": + user_content += ("\n\n" + content) if user_content else content + elif role == "assistant": + # Claude Agent SDK doesn't support multi-turn easily in query() + # For now, prepend assistant messages to user content + user_content += f"\n\n[Previous assistant response: {content}]" + + # Add JSON schema instruction if response_format is provided + if response_format is not None and hasattr(response_format, "model_json_schema"): + schema = response_format.model_json_schema() + schema_instruction = ( + f"\n\nYou must respond with valid JSON matching this schema:\n{json.dumps(schema, indent=2)}\n\n" + "Respond with ONLY the JSON, no markdown formatting." + ) + user_content += schema_instruction + + # Configure SDK options + options = ClaudeAgentOptions( + system_prompt=system_prompt if system_prompt else None, + max_turns=1, # Single-turn for API-style interactions + allowed_tools=[], # Disable tools for standard LLM calls + ) + + # Call Claude Agent SDK + last_exception = None + for attempt in range(max_retries + 1): + try: + # Collect streaming response + full_text = "" + + async for message in query(prompt=user_content, options=options): + if isinstance(message, AssistantMessage): + for block in message.content: + if isinstance(block, TextBlock): + full_text += block.text + + # Handle structured output + if response_format is not None: + # Models may wrap JSON in markdown + clean_text = full_text + if "```json" in full_text: + clean_text = full_text.split("```json")[1].split("```")[0].strip() + elif "```" in full_text: + clean_text = full_text.split("```")[1].split("```")[0].strip() + + try: + json_data = json.loads(clean_text) + except json.JSONDecodeError as e: + logger.warning(f"Claude Code JSON parse error (attempt {attempt + 1}/{max_retries + 1}): {e}") + if attempt < max_retries: + backoff = min(initial_backoff * (2**attempt), max_backoff) + await asyncio.sleep(backoff) + last_exception = e + continue + raise + + if skip_validation: + result = json_data + else: + result = response_format.model_validate(json_data) + else: + result = full_text + + # Record metrics + duration = time.time() - start_time + metrics = get_metrics_collector() + + # Estimate token usage (Claude Agent SDK doesn't report exact counts) + # Use character count / 4 as rough estimate (1 token ≈ 4 characters) + estimated_input = sum(len(m.get("content", "")) for m in messages) // 4 + estimated_output = len(full_text) // 4 + + metrics.record_llm_call( + provider=self.provider, + model=self.model, + scope=scope, + duration=duration, + input_tokens=estimated_input, + output_tokens=estimated_output, + success=True, + ) + + # Log slow calls + if duration > 10.0: + logger.info( + f"slow llm call: scope={scope}, model={self.provider}/{self.model}, time={duration:.3f}s" + ) + + if return_usage: + token_usage = TokenUsage( + input_tokens=estimated_input, + output_tokens=estimated_output, + total_tokens=estimated_input + estimated_output, + ) + return result, token_usage + + return result + + except Exception as e: + last_exception = e + + # Check for authentication errors + error_str = str(e).lower() + if "auth" in error_str or "login" in error_str or "credential" in error_str: + logger.error(f"Claude Code authentication error: {e}") + raise RuntimeError( + f"Claude Code authentication failed: {e}\n\n" + "Run 'claude auth login' to authenticate with Claude Pro/Max." + ) from e + + if attempt < max_retries: + backoff = min(initial_backoff * (2**attempt), max_backoff) + logger.warning(f"Claude Code error (attempt {attempt + 1}/{max_retries + 1}): {e}") + await asyncio.sleep(backoff) + continue + else: + logger.error(f"Claude Code error after {max_retries + 1} attempts: {e}") + raise + + if last_exception: + raise last_exception + raise RuntimeError("Claude Code call failed after all retries") + + async def call_with_tools( + self, + messages: list[dict[str, Any]], + tools: list[dict[str, Any]], + max_completion_tokens: int | None = None, + temperature: float | None = None, + scope: str = "tools", + max_retries: int = 5, + initial_backoff: float = 1.0, + max_backoff: float = 30.0, + tool_choice: str | dict[str, Any] = "auto", + ) -> LLMToolCallResult: + """ + Make an LLM API call with tool/function calling support. + + Note: This is a simplified implementation. Full tool support would require + integrating with Claude Agent SDK's tool system. + + Args: + messages: List of message dicts. Can include tool results with role='tool'. + tools: List of tool definitions in OpenAI format. + max_completion_tokens: Maximum tokens in response. + temperature: Sampling temperature. + scope: Scope identifier for tracking. + max_retries: Maximum retry attempts. + initial_backoff: Initial backoff time in seconds. + max_backoff: Maximum backoff time in seconds. + tool_choice: How to choose tools - "auto", "none", "required", or specific function. + + Returns: + LLMToolCallResult with content and/or tool_calls. + """ + # For now, use regular call without tools + # Full implementation would require mapping OpenAI tool format to Claude Agent SDK tools + logger.warning( + "Claude Code provider does not fully support tool calling yet. Falling back to regular text completion." + ) + + result = await self.call( + messages=messages, + response_format=None, + max_completion_tokens=max_completion_tokens, + temperature=temperature, + scope=scope, + max_retries=max_retries, + initial_backoff=initial_backoff, + max_backoff=max_backoff, + return_usage=True, + ) + + if isinstance(result, tuple): + text, usage = result + return LLMToolCallResult( + content=text, + tool_calls=[], + finish_reason="stop", + input_tokens=usage.input_tokens, + output_tokens=usage.output_tokens, + ) + else: + # Fallback if return_usage didn't work as expected + return LLMToolCallResult( + content=str(result), + tool_calls=[], + finish_reason="stop", + input_tokens=0, + output_tokens=0, + ) + + async def cleanup(self) -> None: + """Clean up resources (no HTTP client to close for Claude Agent SDK).""" + pass diff --git a/hindsight-api/hindsight_api/engine/providers/codex_llm.py b/hindsight-api/hindsight_api/engine/providers/codex_llm.py new file mode 100644 index 00000000..7775e51d --- /dev/null +++ b/hindsight-api/hindsight_api/engine/providers/codex_llm.py @@ -0,0 +1,527 @@ +""" +OpenAI Codex LLM provider using ChatGPT Plus/Pro OAuth authentication. + +This provider enables using ChatGPT Plus/Pro subscriptions for API calls +without separate OpenAI Platform API credits. It uses OAuth tokens from +~/.codex/auth.json and communicates with the ChatGPT backend API. +""" + +import asyncio +import json +import logging +import os +import time +import uuid +from pathlib import Path +from typing import Any + +import httpx + +from hindsight_api.engine.llm_interface import LLMInterface, OutputTooLongError +from hindsight_api.engine.response_models import LLMToolCall, LLMToolCallResult, TokenUsage +from hindsight_api.metrics import get_metrics_collector + +logger = logging.getLogger(__name__) + + +class CodexLLM(LLMInterface): + """ + LLM provider using OpenAI Codex OAuth authentication. + + Authenticates using ChatGPT Plus/Pro credentials stored in ~/.codex/auth.json + and makes API calls to chatgpt.com/backend-api/codex/responses. + """ + + def __init__( + self, + provider: str, + api_key: str, # Will be ignored, reads from ~/.codex/auth.json + base_url: str, + model: str, + reasoning_effort: str = "low", + **kwargs: Any, + ): + """Initialize Codex LLM provider.""" + super().__init__(provider, api_key, base_url, model, reasoning_effort, **kwargs) + + # Load Codex OAuth credentials + try: + self.access_token, self.account_id = self._load_codex_auth() + logger.info(f"Loaded Codex OAuth credentials for account: {self.account_id}") + except Exception as e: + raise RuntimeError( + f"Failed to load Codex OAuth credentials from ~/.codex/auth.json: {e}\n\n" + "To set up Codex authentication:\n" + "1. Install Codex CLI: npm install -g @openai/codex\n" + "2. Login: codex auth login\n" + "3. Verify: ls ~/.codex/auth.json\n\n" + "Or use a different provider (openai, anthropic, gemini) with API keys." + ) from e + + # Use ChatGPT backend API endpoint + if not self.base_url: + self.base_url = "https://chatgpt.com/backend-api" + + # Normalize model name (strip openai/ prefix if present) + if self.model.startswith("openai/"): + self.model = self.model[len("openai/") :] + + # Map reasoning effort to Codex reasoning summary format + # Codex supports: "auto", "concise", "detailed" + self.reasoning_summary = self._map_reasoning_effort(reasoning_effort) + + # HTTP client for SSE streaming + self._client = httpx.AsyncClient(timeout=120.0) + + def _load_codex_auth(self) -> tuple[str, str]: + """ + Load OAuth credentials from ~/.codex/auth.json. + + Returns: + Tuple of (access_token, account_id). + + Raises: + FileNotFoundError: If auth file doesn't exist. + ValueError: If auth file is invalid. + """ + auth_file = Path.home() / ".codex" / "auth.json" + + if not auth_file.exists(): + raise FileNotFoundError( + f"Codex auth file not found: {auth_file}\nRun 'codex auth login' to authenticate with ChatGPT Plus/Pro." + ) + + with open(auth_file) as f: + data = json.load(f) + + # Validate auth structure + auth_mode = data.get("auth_mode") + if auth_mode != "chatgpt": + raise ValueError(f"Expected auth_mode='chatgpt', got: {auth_mode}") + + tokens = data.get("tokens", {}) + access_token = tokens.get("access_token") + account_id = tokens.get("account_id") + + if not access_token: + raise ValueError("No access_token found in Codex auth file. Run 'codex auth login' again.") + + return access_token, account_id + + def _map_reasoning_effort(self, effort: str) -> str: + """ + Map standard reasoning effort to Codex reasoning summary format. + + Args: + effort: Standard effort level ("low", "medium", "high", "xhigh"). + + Returns: + Codex reasoning summary: "concise", "detailed", or "auto". + """ + mapping = { + "low": "concise", + "medium": "auto", + "high": "detailed", + "xhigh": "detailed", + } + return mapping.get(effort.lower(), "auto") + + async def verify_connection(self) -> None: + """Verify Codex connection by making a simple test call.""" + try: + logger.info(f"Verifying Codex LLM: model={self.model}, account={self.account_id}...") + await self.call( + messages=[{"role": "user", "content": "Say 'ok'"}], + max_completion_tokens=10, + max_retries=2, + initial_backoff=0.5, + max_backoff=2.0, + ) + logger.info(f"Codex LLM verified: {self.model}") + except Exception as e: + raise RuntimeError(f"Codex LLM connection verification failed for {self.model}: {e}") from e + + async def call( + self, + messages: list[dict[str, str]], + response_format: Any | None = None, + max_completion_tokens: int | None = None, + temperature: float | None = None, + scope: str = "memory", + max_retries: int = 10, + initial_backoff: float = 1.0, + max_backoff: float = 60.0, + skip_validation: bool = False, + strict_schema: bool = False, + return_usage: bool = False, + ) -> Any: + """Make API call to Codex backend with SSE streaming.""" + start_time = time.time() + + # Prepare system instructions + system_instruction = "" + user_messages = [] + + for msg in messages: + role = msg.get("role", "user") + content = msg.get("content", "") + + if role == "system": + system_instruction += ("\n\n" + content) if system_instruction else content + else: + user_messages.append(msg) + + # Add JSON schema instruction if response_format is provided + if response_format is not None and hasattr(response_format, "model_json_schema"): + schema = response_format.model_json_schema() + schema_msg = f"\n\nYou must respond with valid JSON matching this schema:\n{json.dumps(schema, indent=2)}" + system_instruction += schema_msg + + # Build Codex request payload + payload = { + "model": self.model, + "instructions": system_instruction, + "input": [ + { + "type": "message", + "role": msg.get("role", "user"), + "content": msg.get("content", ""), + } + for msg in user_messages + ], + "tools": [], + "tool_choice": "auto", + "parallel_tool_calls": True, + "reasoning": {"summary": self.reasoning_summary}, + "store": False, # Codex uses stateless mode + "stream": True, # SSE streaming + "include": ["reasoning.encrypted_content"], + "prompt_cache_key": str(uuid.uuid4()), + } + + headers = { + "Authorization": f"Bearer {self.access_token}", + "Content-Type": "application/json", + "OpenAI-Account-ID": self.account_id, + "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7)", + "Origin": "https://chatgpt.com", + } + + url = f"{self.base_url}/codex/responses" + last_exception = None + + for attempt in range(max_retries + 1): + try: + response = await self._client.post(url, json=payload, headers=headers, timeout=120.0) + response.raise_for_status() + + # Parse SSE stream + content = await self._parse_sse_stream(response) + + # Handle structured output + if response_format is not None: + # Models may wrap JSON in markdown + clean_content = content + if "```json" in content: + clean_content = content.split("```json")[1].split("```")[0].strip() + elif "```" in content: + clean_content = content.split("```")[1].split("```")[0].strip() + + try: + json_data = json.loads(clean_content) + except json.JSONDecodeError as e: + logger.warning(f"Codex JSON parse error (attempt {attempt + 1}/{max_retries + 1}): {e}") + if attempt < max_retries: + backoff = min(initial_backoff * (2**attempt), max_backoff) + await asyncio.sleep(backoff) + last_exception = e + continue + raise + + if skip_validation: + result = json_data + else: + result = response_format.model_validate(json_data) + else: + result = content + + # Record metrics + duration = time.time() - start_time + metrics = get_metrics_collector() + metrics.record_llm_call( + provider=self.provider, + model=self.model, + scope=scope, + duration=duration, + input_tokens=0, # Codex doesn't report token counts in SSE + output_tokens=0, + success=True, + ) + + if return_usage: + # Codex doesn't provide token counts, estimate based on content + estimated_input = sum(len(m.get("content", "")) for m in messages) // 4 + estimated_output = len(content) // 4 + token_usage = TokenUsage( + input_tokens=estimated_input, + output_tokens=estimated_output, + total_tokens=estimated_input + estimated_output, + ) + return result, token_usage + + return result + + except httpx.HTTPStatusError as e: + last_exception = e + status_code = e.response.status_code + + # Fast fail on auth errors + if status_code in (401, 403): + logger.error(f"Codex auth error (HTTP {status_code}): {e.response.text[:200]}") + raise RuntimeError( + "Codex authentication failed. Your OAuth token may have expired.\n" + "Run 'codex auth login' to re-authenticate." + ) from e + + if attempt < max_retries: + backoff = min(initial_backoff * (2**attempt), max_backoff) + logger.warning(f"Codex HTTP error {status_code} (attempt {attempt + 1}/{max_retries + 1})") + await asyncio.sleep(backoff) + continue + else: + logger.error(f"Codex HTTP error after {max_retries + 1} attempts: {e}") + raise + + except httpx.RequestError as e: + last_exception = e + if attempt < max_retries: + backoff = min(initial_backoff * (2**attempt), max_backoff) + logger.warning(f"Codex connection error (attempt {attempt + 1}/{max_retries + 1}): {e}") + await asyncio.sleep(backoff) + continue + else: + logger.error(f"Codex connection error after {max_retries + 1} attempts: {e}") + raise + + except Exception as e: + logger.error(f"Unexpected Codex error: {type(e).__name__}: {e}") + raise + + if last_exception: + raise last_exception + raise RuntimeError("Codex call failed after all retries") + + async def _parse_sse_stream(self, response: httpx.Response) -> str: + """ + Parse Server-Sent Events (SSE) stream from Codex API. + + Args: + response: HTTP response with SSE stream. + + Returns: + Extracted text content from stream. + """ + full_text = "" + event_type = None + + async for line in response.aiter_lines(): + if not line: + continue + + # Track event type + if line.startswith("event: "): + event_type = line[7:] + + # Parse data + elif line.startswith("data: "): + data_str = line[6:] + if data_str == "[DONE]": + break + + try: + data = json.loads(data_str) + + # Extract content based on event type + if event_type == "response.text.delta" and "delta" in data: + full_text += data["delta"] + elif event_type == "response.content_part.delta" and "delta" in data: + full_text += data["delta"] + # Check for item content + elif "item" in data: + item = data["item"] + if "content" in item: + content = item["content"] + if isinstance(content, list): + for part in content: + if isinstance(part, dict) and "text" in part: + full_text += part["text"] + elif isinstance(content, str): + full_text += content + + except json.JSONDecodeError: + # Skip malformed JSON events + pass + + return full_text + + async def call_with_tools( + self, + messages: list[dict[str, Any]], + tools: list[dict[str, Any]], + max_completion_tokens: int | None = None, + temperature: float | None = None, + scope: str = "tools", + max_retries: int = 5, + initial_backoff: float = 1.0, + max_backoff: float = 30.0, + tool_choice: str | dict[str, Any] = "auto", + ) -> LLMToolCallResult: + """ + Make API call with tool calling support. + + Note: This is a basic implementation. Full tool calling support for Codex + may require additional SSE event parsing. + """ + start_time = time.time() + + # Prepare system instructions + system_instruction = "" + user_messages = [] + + for msg in messages: + role = msg.get("role", "user") + content = msg.get("content", "") + + if role == "system": + system_instruction += ("\n\n" + content) if system_instruction else content + elif role == "tool": + # Handle tool results + user_messages.append( + { + "type": "message", + "role": "user", + "content": f"Tool result: {content}", + } + ) + else: + user_messages.append( + { + "type": "message", + "role": role, + "content": content, + } + ) + + # Convert tools to Codex format + codex_tools = [] + for tool in tools: + func = tool.get("function", {}) + codex_tools.append( + { + "type": "function", + "function": { + "name": func.get("name", ""), + "description": func.get("description", ""), + "parameters": func.get("parameters", {}), + }, + } + ) + + payload = { + "model": self.model, + "instructions": system_instruction, + "input": user_messages, + "tools": codex_tools, + "tool_choice": tool_choice, + "parallel_tool_calls": True, + "reasoning": {"summary": self.reasoning_summary}, + "store": False, + "stream": True, + "include": ["reasoning.encrypted_content"], + "prompt_cache_key": str(uuid.uuid4()), + } + + headers = { + "Authorization": f"Bearer {self.access_token}", + "Content-Type": "application/json", + "OpenAI-Account-ID": self.account_id, + "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7)", + "Origin": "https://chatgpt.com", + } + + url = f"{self.base_url}/codex/responses" + + try: + response = await self._client.post(url, json=payload, headers=headers, timeout=120.0) + response.raise_for_status() + + # Parse SSE for tool calls and content + content, tool_calls = await self._parse_sse_tool_stream(response) + + duration = time.time() - start_time + metrics = get_metrics_collector() + metrics.record_llm_call( + provider=self.provider, + model=self.model, + scope=scope, + duration=duration, + input_tokens=0, + output_tokens=0, + success=True, + ) + + return LLMToolCallResult( + content=content, + tool_calls=tool_calls, + finish_reason="tool_calls" if tool_calls else "stop", + input_tokens=0, + output_tokens=0, + ) + + except Exception as e: + logger.error(f"Codex tool call error: {e}") + raise + + async def _parse_sse_tool_stream(self, response: httpx.Response) -> tuple[str | None, list[LLMToolCall]]: + """ + Parse SSE stream for tool calls and content. + + Returns: + Tuple of (content, tool_calls). + """ + content = "" + tool_calls: list[LLMToolCall] = [] + event_type = None + + async for line in response.aiter_lines(): + if not line: + continue + + if line.startswith("event: "): + event_type = line[7:] + + elif line.startswith("data: "): + data_str = line[6:] + if data_str == "[DONE]": + break + + try: + data = json.loads(data_str) + + # Extract text content + if event_type == "response.text.delta" and "delta" in data: + content += data["delta"] + + # Extract tool calls + elif event_type == "response.function_call_arguments.delta": + # Handle tool call events (implementation depends on actual Codex SSE format) + pass + + except json.JSONDecodeError: + pass + + return content if content else None, tool_calls + + async def cleanup(self) -> None: + """Clean up HTTP client.""" + await self._client.aclose() diff --git a/hindsight-api/hindsight_api/engine/providers/gemini_llm.py b/hindsight-api/hindsight_api/engine/providers/gemini_llm.py new file mode 100644 index 00000000..840efdba --- /dev/null +++ b/hindsight-api/hindsight_api/engine/providers/gemini_llm.py @@ -0,0 +1,502 @@ +""" +Google Gemini/VertexAI LLM provider. + +This provider supports both: +1. Gemini API (api.generativeai.google.com) with API key authentication +2. Vertex AI with service account or Application Default Credentials (ADC) +""" + +import asyncio +import json +import logging +import os +import time +from typing import Any + +from google import genai +from google.genai import errors as genai_errors +from google.genai import types as genai_types + +from hindsight_api.engine.llm_interface import LLMInterface, OutputTooLongError +from hindsight_api.engine.response_models import LLMToolCall, LLMToolCallResult, TokenUsage +from hindsight_api.metrics import get_metrics_collector + +logger = logging.getLogger(__name__) + +# Vertex AI imports (optional) +try: + import google.auth + from google.oauth2 import service_account + + VERTEXAI_AVAILABLE = True +except ImportError: + VERTEXAI_AVAILABLE = False + + +class GeminiLLM(LLMInterface): + """ + LLM provider for Google Gemini and Vertex AI. + + Supports: + - Gemini API: provider="gemini", requires api_key + - Vertex AI: provider="vertexai", requires project_id and region, uses ADC or service account + """ + + def __init__( + self, + provider: str, + api_key: str, + base_url: str, + model: str, + reasoning_effort: str = "low", + **kwargs: Any, + ): + """Initialize Gemini/VertexAI LLM provider.""" + super().__init__(provider, api_key, base_url, model, reasoning_effort, **kwargs) + + self._client = None + self._is_vertexai = self.provider == "vertexai" + + if self._is_vertexai: + self._init_vertexai(**kwargs) + else: + self._init_gemini() + + def _init_gemini(self) -> None: + """Initialize Gemini API client.""" + if not self.api_key: + raise ValueError("Gemini provider requires api_key") + + self._client = genai.Client(api_key=self.api_key) + logger.info(f"Gemini API: model={self.model}") + + def _init_vertexai(self, **kwargs: Any) -> None: + """Initialize Vertex AI client with project, region, and credentials.""" + # Extract Vertex AI config from kwargs + project_id = kwargs.get("vertexai_project_id") + region = kwargs.get("vertexai_region", "us-central1") + service_account_key = kwargs.get("vertexai_service_account_key") + credentials = kwargs.get("vertexai_credentials") # Pre-loaded credentials object + + if not project_id: + raise ValueError( + "HINDSIGHT_API_LLM_VERTEXAI_PROJECT_ID is required for Vertex AI provider. " + "Set it to your GCP project ID." + ) + + auth_method = "ADC" + + # Use pre-loaded credentials if provided (passed from LLMProvider) + if credentials is not None: + auth_method = "service_account" + # Otherwise, load explicit service account credentials if path provided + elif service_account_key: + if not VERTEXAI_AVAILABLE: + raise ValueError( + "Vertex AI service account auth requires 'google-auth' package. " + "Install with: pip install google-auth" + ) + credentials = service_account.Credentials.from_service_account_file( + service_account_key, + scopes=["https://www.googleapis.com/auth/cloud-platform"], + ) + auth_method = "service_account" + logger.info(f"Vertex AI: Using service account key: {service_account_key}") + + # Strip google/ prefix from model name — native SDK uses bare names + # e.g. "google/gemini-2.0-flash-lite-001" -> "gemini-2.0-flash-lite-001" + if self.model.startswith("google/"): + self.model = self.model[len("google/") :] + + # Create Vertex AI client + client_kwargs: dict[str, Any] = { + "vertexai": True, + "project": project_id, + "location": region, + } + if credentials is not None: + client_kwargs["credentials"] = credentials + + self._client = genai.Client(**client_kwargs) + + logger.info(f"Vertex AI: project={project_id}, region={region}, model={self.model}, auth={auth_method}") + + async def verify_connection(self) -> None: + """ + Verify that the Gemini/VertexAI provider is configured correctly. + + Raises: + RuntimeError: If the connection test fails. + """ + try: + logger.info(f"Verifying {self.provider.upper()}: model={self.model}...") + await self.call( + messages=[{"role": "user", "content": "Say 'ok'"}], + max_completion_tokens=100, + max_retries=2, + initial_backoff=0.5, + max_backoff=2.0, + ) + logger.info(f"{self.provider.upper()} connection verified successfully") + except Exception as e: + raise RuntimeError(f"Failed to verify {self.provider.upper()} connection: {e}") from e + + async def call( + self, + messages: list[dict[str, str]], + response_format: Any | None = None, + max_completion_tokens: int | None = None, + temperature: float | None = None, + scope: str = "memory", + max_retries: int = 10, + initial_backoff: float = 1.0, + max_backoff: float = 60.0, + skip_validation: bool = False, + strict_schema: bool = False, + return_usage: bool = False, + ) -> Any: + """ + Make a Gemini/VertexAI API call with retry logic. + + Args: + messages: List of message dicts with 'role' and 'content'. + response_format: Optional Pydantic model for structured output. + max_completion_tokens: Maximum tokens in response (not supported by Gemini). + temperature: Sampling temperature (0.0-2.0). + scope: Scope identifier for tracking. + max_retries: Maximum retry attempts. + initial_backoff: Initial backoff time in seconds. + max_backoff: Maximum backoff time in seconds. + skip_validation: Return raw JSON without Pydantic validation. + strict_schema: Use strict JSON schema enforcement (not supported by Gemini). + return_usage: If True, return tuple (result, TokenUsage). + + Returns: + If return_usage=False: Parsed response if response_format provided, else text. + If return_usage=True: Tuple of (result, TokenUsage). + """ + start_time = time.time() + + # Convert OpenAI-style messages to Gemini format + system_instruction = None + gemini_contents = [] + + for msg in messages: + role = msg.get("role", "user") + content = msg.get("content", "") + + if role == "system": + if system_instruction: + system_instruction += "\n\n" + content + else: + system_instruction = content + elif role == "assistant": + gemini_contents.append(genai_types.Content(role="model", parts=[genai_types.Part(text=content)])) + else: + gemini_contents.append(genai_types.Content(role="user", parts=[genai_types.Part(text=content)])) + + # Add JSON schema instruction if response_format is provided + if response_format is not None and hasattr(response_format, "model_json_schema"): + schema = response_format.model_json_schema() + schema_msg = f"\n\nYou must respond with valid JSON matching this schema:\n{json.dumps(schema, indent=2)}" + if system_instruction: + system_instruction += schema_msg + else: + system_instruction = schema_msg + + # Build generation config + config_kwargs: dict[str, Any] = {} + if system_instruction: + config_kwargs["system_instruction"] = system_instruction + if response_format is not None: + config_kwargs["response_mime_type"] = "application/json" + config_kwargs["response_schema"] = response_format + if temperature is not None: + config_kwargs["temperature"] = temperature + + generation_config = genai_types.GenerateContentConfig(**config_kwargs) if config_kwargs else None + + last_exception = None + + for attempt in range(max_retries + 1): + try: + response = await self._client.aio.models.generate_content( + model=self.model, + contents=gemini_contents, + config=generation_config, + ) + + content = response.text + + # Handle empty response + if content is None: + block_reason = None + if hasattr(response, "candidates") and response.candidates: + candidate = response.candidates[0] + if hasattr(candidate, "finish_reason"): + block_reason = candidate.finish_reason + + if attempt < max_retries: + logger.warning(f"Gemini returned empty response (reason: {block_reason}), retrying...") + backoff = min(initial_backoff * (2**attempt), max_backoff) + await asyncio.sleep(backoff) + continue + else: + raise RuntimeError(f"Gemini returned empty response after {max_retries + 1} attempts") + + # Parse structured output if requested + if response_format is not None: + json_data = json.loads(content) + if skip_validation: + result = json_data + else: + result = response_format.model_validate(json_data) + else: + result = content + + # Extract token usage + input_tokens = 0 + output_tokens = 0 + if hasattr(response, "usage_metadata") and response.usage_metadata: + usage = response.usage_metadata + input_tokens = usage.prompt_token_count or 0 + output_tokens = usage.candidates_token_count or 0 + + # Record metrics + duration = time.time() - start_time + metrics = get_metrics_collector() + metrics.record_llm_call( + provider=self.provider, + model=self.model, + scope=scope, + duration=duration, + input_tokens=input_tokens, + output_tokens=output_tokens, + success=True, + ) + + # Log slow calls + if duration > 10.0 and input_tokens > 0: + logger.info( + f"slow llm call: scope={scope}, model={self.provider}/{self.model}, " + f"input_tokens={input_tokens}, output_tokens={output_tokens}, " + f"time={duration:.3f}s" + ) + + if return_usage: + token_usage = TokenUsage( + input_tokens=input_tokens, + output_tokens=output_tokens, + total_tokens=input_tokens + output_tokens, + ) + return result, token_usage + return result + + except json.JSONDecodeError as e: + last_exception = e + if attempt < max_retries: + logger.warning("Gemini returned invalid JSON, retrying...") + backoff = min(initial_backoff * (2**attempt), max_backoff) + await asyncio.sleep(backoff) + continue + else: + logger.error(f"Gemini returned invalid JSON after {max_retries + 1} attempts") + raise + + except genai_errors.APIError as e: + # Fast fail on auth errors - these won't recover with retries + if e.code in (401, 403): + logger.error(f"Gemini auth error (HTTP {e.code}), not retrying: {str(e)}") + raise + + # Retry on retryable errors (rate limits, server errors, client errors) + if e.code in (400, 429, 500, 502, 503, 504) or (e.code and e.code >= 500): + last_exception = e + if attempt < max_retries: + backoff = min(initial_backoff * (2**attempt), max_backoff) + jitter = backoff * 0.2 * (2 * (time.time() % 1) - 1) + await asyncio.sleep(backoff + jitter) + else: + logger.error(f"Gemini API error after {max_retries + 1} attempts: {str(e)}") + raise + else: + logger.error(f"Gemini API error: {type(e).__name__}: {str(e)}") + raise + + except Exception as e: + logger.error(f"Unexpected error during Gemini call: {type(e).__name__}: {str(e)}") + raise + + if last_exception: + raise last_exception + raise RuntimeError("Gemini call failed after all retries") + + async def call_with_tools( + self, + messages: list[dict[str, Any]], + tools: list[dict[str, Any]], + max_completion_tokens: int | None = None, + temperature: float | None = None, + scope: str = "tools", + max_retries: int = 5, + initial_backoff: float = 1.0, + max_backoff: float = 30.0, + tool_choice: str | dict[str, Any] = "auto", + ) -> LLMToolCallResult: + """ + Make a Gemini/VertexAI API call with tool/function calling support. + + Args: + messages: List of message dicts. Can include tool results with role='tool'. + tools: List of tool definitions in OpenAI format. + max_completion_tokens: Maximum tokens (not supported by Gemini). + temperature: Sampling temperature. + scope: Scope identifier for tracking. + max_retries: Maximum retry attempts. + initial_backoff: Initial backoff time in seconds. + max_backoff: Maximum backoff time in seconds. + tool_choice: How to choose tools (Gemini uses "auto" only). + + Returns: + LLMToolCallResult with content and/or tool_calls. + """ + start_time = time.time() + + # Convert tools to Gemini format + gemini_tools = [] + for tool in tools: + func = tool.get("function", {}) + gemini_tools.append( + genai_types.Tool( + function_declarations=[ + genai_types.FunctionDeclaration( + name=func.get("name", ""), + description=func.get("description", ""), + parameters=func.get("parameters"), + ) + ] + ) + ) + + # Convert messages + system_instruction = None + gemini_contents = [] + for msg in messages: + role = msg.get("role", "user") + content = msg.get("content", "") + + if role == "system": + system_instruction = (system_instruction + "\n\n" + content) if system_instruction else content + elif role == "tool": + # Gemini uses function_response + gemini_contents.append( + genai_types.Content( + role="user", + parts=[ + genai_types.Part( + function_response=genai_types.FunctionResponse( + name=msg.get("name", ""), + response={"result": content}, + ) + ) + ], + ) + ) + elif role == "assistant": + gemini_contents.append(genai_types.Content(role="model", parts=[genai_types.Part(text=content)])) + else: + gemini_contents.append(genai_types.Content(role="user", parts=[genai_types.Part(text=content)])) + + config_kwargs: dict[str, Any] = {"tools": gemini_tools} + if system_instruction: + config_kwargs["system_instruction"] = system_instruction + if temperature is not None: + config_kwargs["temperature"] = temperature + + config = genai_types.GenerateContentConfig(**config_kwargs) + + last_exception = None + for attempt in range(max_retries + 1): + try: + response = await self._client.aio.models.generate_content( + model=self.model, + contents=gemini_contents, + config=config, + ) + + # Extract content and tool calls + content = None + tool_calls: list[LLMToolCall] = [] + + if response.candidates and response.candidates[0].content: + parts = response.candidates[0].content.parts + if parts: + for part in parts: + if hasattr(part, "text") and part.text: + content = part.text + if hasattr(part, "function_call") and part.function_call: + fc = part.function_call + tool_calls.append( + LLMToolCall( + id=f"gemini_{len(tool_calls)}", + name=fc.name, + arguments=dict(fc.args) if fc.args else {}, + ) + ) + + finish_reason = "tool_calls" if tool_calls else "stop" + + # Extract token usage + input_tokens = 0 + output_tokens = 0 + if response.usage_metadata: + input_tokens = response.usage_metadata.prompt_token_count or 0 + output_tokens = response.usage_metadata.candidates_token_count or 0 + + # Record metrics + duration = time.time() - start_time + metrics = get_metrics_collector() + metrics.record_llm_call( + provider=self.provider, + model=self.model, + scope=scope, + duration=duration, + input_tokens=input_tokens, + output_tokens=output_tokens, + success=True, + ) + + return LLMToolCallResult( + content=content, + tool_calls=tool_calls, + finish_reason=finish_reason, + input_tokens=input_tokens, + output_tokens=output_tokens, + ) + + except genai_errors.APIError as e: + # Fast fail on auth errors + if e.code in (401, 403): + logger.error(f"Gemini auth error (HTTP {e.code}), not retrying: {str(e)}") + raise + + # Retry on retryable errors + last_exception = e + if attempt < max_retries: + backoff = min(initial_backoff * (2**attempt), max_backoff) + await asyncio.sleep(backoff) + continue + raise + + except Exception as e: + logger.error(f"Unexpected error during Gemini tool call: {type(e).__name__}: {str(e)}") + raise + + if last_exception: + raise last_exception + raise RuntimeError("Gemini tool call failed") + + async def cleanup(self) -> None: + """Clean up resources (close connections, etc.).""" + # Gemini client doesn't require explicit cleanup + pass diff --git a/hindsight-api/hindsight_api/engine/providers/mock_llm.py b/hindsight-api/hindsight_api/engine/providers/mock_llm.py new file mode 100644 index 00000000..7d02b346 --- /dev/null +++ b/hindsight-api/hindsight_api/engine/providers/mock_llm.py @@ -0,0 +1,234 @@ +""" +Mock LLM provider for testing. + +This provider allows tests to record LLM calls and return configurable mock responses +without making actual API calls to external LLM services. +""" + +import logging +from typing import Any + +from ..llm_interface import LLMInterface +from ..response_models import LLMToolCall, LLMToolCallResult, TokenUsage + +logger = logging.getLogger(__name__) + + +class MockLLM(LLMInterface): + """ + Mock LLM provider for testing. + + This provider records all calls and returns configurable mock responses, + enabling tests to verify LLM interactions without making real API calls. + + Example: + # Create mock provider + mock_llm = MockLLM(provider="mock", api_key="", base_url="", model="mock-model") + + # Set mock response + mock_llm.set_mock_response({"answer": "test"}) + + # Make calls + result = await mock_llm.call( + messages=[{"role": "user", "content": "test"}], + response_format=MyResponseModel + ) + + # Verify calls + calls = mock_llm.get_mock_calls() + assert len(calls) == 1 + assert calls[0]["scope"] == "memory" + """ + + def __init__( + self, + provider: str, + api_key: str, + base_url: str, + model: str, + reasoning_effort: str = "low", + **kwargs: Any, + ): + """ + Initialize mock LLM provider. + + Args: + provider: Provider name (should be "mock"). + api_key: Not used for mock provider. + base_url: Not used for mock provider. + model: Model name for tracking. + reasoning_effort: Not used for mock provider. + **kwargs: Additional parameters (not used). + """ + super().__init__(provider, api_key, base_url, model, reasoning_effort, **kwargs) + + # Storage for test verification + self._mock_calls: list[dict] = [] + self._mock_response: Any = None + + async def verify_connection(self) -> None: + """ + Verify mock provider (always succeeds). + + Mock provider doesn't need connection verification since it doesn't + make real API calls. + """ + logger.debug("Mock LLM: connection verification (always succeeds)") + + async def call( + self, + messages: list[dict[str, str]], + response_format: Any | None = None, + max_completion_tokens: int | None = None, + temperature: float | None = None, + scope: str = "memory", + max_retries: int = 10, + initial_backoff: float = 1.0, + max_backoff: float = 60.0, + skip_validation: bool = False, + strict_schema: bool = False, + return_usage: bool = False, + ) -> Any: + """ + Make a mock LLM API call. + + Records the call for test verification and returns the configured mock response. + + Args: + messages: List of message dicts with 'role' and 'content'. + response_format: Optional Pydantic model for structured output. + max_completion_tokens: Not used in mock. + temperature: Not used in mock. + scope: Scope identifier for tracking. + max_retries: Not used in mock. + initial_backoff: Not used in mock. + max_backoff: Not used in mock. + skip_validation: Return raw JSON without Pydantic validation. + strict_schema: Not used in mock. + return_usage: If True, return tuple (result, TokenUsage) instead of just result. + + Returns: + If return_usage=False: Parsed response if response_format is provided, otherwise text content. + If return_usage=True: Tuple of (result, TokenUsage) with mock token counts. + """ + # Record the call for test verification + call_record = { + "provider": self.provider, + "model": self.model, + "messages": messages, + "response_format": response_format.__name__ + if response_format and hasattr(response_format, "__name__") + else str(response_format), + "scope": scope, + } + self._mock_calls.append(call_record) + logger.debug(f"Mock LLM call recorded: scope={scope}, model={self.model}") + + # Return mock response + if self._mock_response is not None: + result = self._mock_response + elif response_format is not None: + # Try to create a minimal valid instance of the response format + try: + # For Pydantic models, try to create with minimal valid data + result = {"mock": True} + except Exception: + result = {"mock": True} + else: + result = "mock response" + + if return_usage: + token_usage = TokenUsage(input_tokens=10, output_tokens=5, total_tokens=15) + return result, token_usage + return result + + async def call_with_tools( + self, + messages: list[dict[str, Any]], + tools: list[dict[str, Any]], + max_completion_tokens: int | None = None, + temperature: float | None = None, + scope: str = "tools", + max_retries: int = 5, + initial_backoff: float = 1.0, + max_backoff: float = 30.0, + tool_choice: str | dict[str, Any] = "auto", + ) -> LLMToolCallResult: + """ + Make a mock LLM API call with tool/function calling support. + + Records the call for test verification and returns the configured mock response. + + Args: + messages: List of message dicts. Can include tool results with role='tool'. + tools: List of tool definitions in OpenAI format. + max_completion_tokens: Not used in mock. + temperature: Not used in mock. + scope: Scope identifier for tracking. + max_retries: Not used in mock. + initial_backoff: Not used in mock. + max_backoff: Not used in mock. + tool_choice: Not used in mock. + + Returns: + LLMToolCallResult with content and/or tool_calls. + """ + # Record the call for test verification + call_record = { + "provider": self.provider, + "model": self.model, + "messages": messages, + "tools": [t.get("function", {}).get("name") for t in tools], + "scope": scope, + } + self._mock_calls.append(call_record) + + if self._mock_response is not None: + if isinstance(self._mock_response, LLMToolCallResult): + return self._mock_response + # Allow setting just tool calls as a list + if isinstance(self._mock_response, list): + return LLMToolCallResult( + tool_calls=[ + LLMToolCall(id=f"mock_{i}", name=tc["name"], arguments=tc.get("arguments", {})) + for i, tc in enumerate(self._mock_response) + ], + finish_reason="tool_calls", + ) + + return LLMToolCallResult(content="mock response", finish_reason="stop") + + async def cleanup(self) -> None: + """Clean up resources (no-op for mock provider).""" + pass + + def set_mock_response(self, response: Any) -> None: + """ + Set the response to return from mock calls. + + Args: + response: The response to return. Can be: + - A dict/Pydantic model for regular calls + - An LLMToolCallResult for tool calls + - A list of tool call dicts for tool calls + - Any other value to return as-is + """ + self._mock_response = response + + def get_mock_calls(self) -> list[dict]: + """ + Get the list of recorded mock calls. + + Returns: + List of call records, each containing: + - provider: Provider name + - model: Model name + - messages: Messages sent + - response_format/tools: Format or tools used + - scope: Call scope + """ + return self._mock_calls + + def clear_mock_calls(self) -> None: + """Clear the recorded mock calls.""" + self._mock_calls = [] diff --git a/hindsight-api/hindsight_api/engine/providers/openai_compatible_llm.py b/hindsight-api/hindsight_api/engine/providers/openai_compatible_llm.py new file mode 100644 index 00000000..554f6af6 --- /dev/null +++ b/hindsight-api/hindsight_api/engine/providers/openai_compatible_llm.py @@ -0,0 +1,745 @@ +""" +OpenAI-compatible LLM provider supporting OpenAI, Groq, Ollama, and LMStudio. + +This provider handles all OpenAI API-compatible models including: +- OpenAI: GPT-4, GPT-4o, GPT-5, o1, o3 (reasoning models) +- Groq: Fast inference with seed control and service tiers +- Ollama: Local models with native streaming API support +- LMStudio: Local models with OpenAI-compatible API + +Features: +- Reasoning models with extended thinking (o1, o3, GPT-5 families) +- Strict JSON schema enforcement (OpenAI) +- Provider-specific parameters (Groq seed, service tier) +- Native Ollama streaming for better structured output +- Automatic token limit handling per model family +""" + +import asyncio +import json +import logging +import os +import re +import time +from typing import Any + +import httpx +from openai import APIConnectionError, APIStatusError, AsyncOpenAI, LengthFinishReasonError + +from hindsight_api.config import DEFAULT_LLM_TIMEOUT, ENV_LLM_TIMEOUT +from hindsight_api.engine.llm_interface import LLMInterface, OutputTooLongError +from hindsight_api.engine.response_models import LLMToolCall, LLMToolCallResult, TokenUsage +from hindsight_api.metrics import get_metrics_collector + +logger = logging.getLogger(__name__) + +# Seed applied to every Groq request for deterministic behavior +DEFAULT_LLM_SEED = 4242 + + +class OpenAICompatibleLLM(LLMInterface): + """ + LLM provider for OpenAI-compatible APIs. + + Supports: + - OpenAI: Standard models (GPT-4, GPT-4o) and reasoning models (o1, o3, GPT-5) + - Groq: Fast inference with seed control and service tiers + - Ollama: Local models with native streaming API for better structured output + - LMStudio: Local models with OpenAI-compatible API + """ + + def __init__( + self, + provider: str, + api_key: str, + base_url: str, + model: str, + reasoning_effort: str = "low", + timeout: float | None = None, + groq_service_tier: str | None = None, + **kwargs: Any, + ): + """ + Initialize OpenAI-compatible LLM provider. + + Args: + provider: Provider name ("openai", "groq", "ollama", "lmstudio"). + api_key: API key (optional for ollama/lmstudio). + base_url: Base URL for the API (uses defaults for groq/ollama/lmstudio if empty). + model: Model name. + reasoning_effort: Reasoning effort level for supported models ("low", "medium", "high"). + timeout: Request timeout in seconds (uses env var or 300s default). + groq_service_tier: Groq service tier ("on_demand", "flex", "auto"). + **kwargs: Additional provider-specific parameters. + """ + super().__init__(provider, api_key, base_url, model, reasoning_effort, **kwargs) + + # Validate provider + valid_providers = ["openai", "groq", "ollama", "lmstudio"] + if self.provider not in valid_providers: + raise ValueError(f"OpenAICompatibleLLM only supports: {', '.join(valid_providers)}. Got: {self.provider}") + + # Set default base URLs + if not self.base_url: + if self.provider == "groq": + self.base_url = "https://api.groq.com/openai/v1" + elif self.provider == "ollama": + self.base_url = "http://localhost:11434/v1" + elif self.provider == "lmstudio": + self.base_url = "http://localhost:1234/v1" + + # For ollama/lmstudio, use dummy key if not provided + if self.provider in ("ollama", "lmstudio") and not self.api_key: + self.api_key = "local" + + # Validate API key for cloud providers + if self.provider in ("openai", "groq") and not self.api_key: + raise ValueError(f"API key is required for {self.provider}") + + # Groq service tier configuration + self.groq_service_tier = groq_service_tier or os.getenv("HINDSIGHT_API_LLM_GROQ_SERVICE_TIER", "auto") + + # Get timeout config + self.timeout = timeout or float(os.getenv(ENV_LLM_TIMEOUT, str(DEFAULT_LLM_TIMEOUT))) + + # Create OpenAI client + client_kwargs: dict[str, Any] = {"api_key": self.api_key, "max_retries": 0} + if self.base_url: + client_kwargs["base_url"] = self.base_url + if self.timeout: + client_kwargs["timeout"] = self.timeout + + self._client = AsyncOpenAI(**client_kwargs) + logger.info( + f"OpenAI-compatible client initialized: provider={self.provider}, model={self.model}, " + f"base_url={self.base_url or 'default'}" + ) + + async def verify_connection(self) -> None: + """ + Verify that the provider is configured correctly by making a simple test call. + + Raises: + RuntimeError: If the connection test fails. + """ + try: + logger.info(f"Verifying connection: {self.provider}/{self.model}") + await self.call( + messages=[{"role": "user", "content": "Say 'ok'"}], + max_completion_tokens=100, + max_retries=2, + initial_backoff=0.5, + max_backoff=2.0, + ) + logger.info(f"Connection verified: {self.provider}/{self.model}") + except Exception as e: + raise RuntimeError(f"Connection verification failed for {self.provider}/{self.model}: {e}") from e + + def _supports_reasoning_model(self) -> bool: + """Check if the current model is a reasoning model (o1, o3, GPT-5, DeepSeek).""" + model_lower = self.model.lower() + return any(x in model_lower for x in ["gpt-5", "o1", "o3", "deepseek"]) + + def _get_max_reasoning_tokens(self) -> int | None: + """Get max reasoning tokens for reasoning models.""" + model_lower = self.model.lower() + + # GPT-4 and GPT-4.1 models have different caps + if any(x in model_lower for x in ["gpt-4.1", "gpt-4-"]): + return 32000 + elif "gpt-4o" in model_lower: + return 16384 + + return None + + async def call( + self, + messages: list[dict[str, str]], + response_format: Any | None = None, + max_completion_tokens: int | None = None, + temperature: float | None = None, + scope: str = "memory", + max_retries: int = 10, + initial_backoff: float = 1.0, + max_backoff: float = 60.0, + skip_validation: bool = False, + strict_schema: bool = False, + return_usage: bool = False, + ) -> Any: + """ + Make an LLM API call with retry logic. + + Args: + messages: List of message dicts with 'role' and 'content'. + response_format: Optional Pydantic model for structured output. + max_completion_tokens: Maximum tokens in response. + temperature: Sampling temperature (0.0-2.0). + scope: Scope identifier for tracking. + max_retries: Maximum retry attempts. + initial_backoff: Initial backoff time in seconds. + max_backoff: Maximum backoff time in seconds. + skip_validation: Return raw JSON without Pydantic validation. + strict_schema: Use strict JSON schema enforcement (OpenAI only). + return_usage: If True, return tuple (result, TokenUsage) instead of just result. + + Returns: + If return_usage=False: Parsed response if response_format is provided, otherwise text content. + If return_usage=True: Tuple of (result, TokenUsage) with token counts. + + Raises: + OutputTooLongError: If output exceeds token limits. + Exception: Re-raises API errors after retries exhausted. + """ + # Handle Ollama with native API for structured output (better schema enforcement) + if self.provider == "ollama" and response_format is not None: + return await self._call_ollama_native( + messages=messages, + response_format=response_format, + max_completion_tokens=max_completion_tokens, + temperature=temperature, + max_retries=max_retries, + initial_backoff=initial_backoff, + max_backoff=max_backoff, + skip_validation=skip_validation, + scope=scope, + return_usage=return_usage, + ) + + start_time = time.time() + + # Build call parameters + call_params: dict[str, Any] = { + "model": self.model, + "messages": messages, + } + + # Check if model supports reasoning parameter + is_reasoning_model = self._supports_reasoning_model() + + # Apply model-specific token limits + if max_completion_tokens is not None: + max_tokens_cap = self._get_max_reasoning_tokens() + if max_tokens_cap and max_completion_tokens > max_tokens_cap: + max_completion_tokens = max_tokens_cap + # For reasoning models, enforce minimum to ensure space for reasoning + output + if is_reasoning_model and max_completion_tokens < 16000: + max_completion_tokens = 16000 + call_params["max_completion_tokens"] = max_completion_tokens + + # Temperature - reasoning models don't support custom temperature + if temperature is not None and not is_reasoning_model: + call_params["temperature"] = temperature + + # Set reasoning_effort for reasoning models + if is_reasoning_model: + call_params["reasoning_effort"] = self.reasoning_effort + + # Provider-specific parameters + if self.provider == "groq": + call_params["seed"] = DEFAULT_LLM_SEED + extra_body: dict[str, Any] = {} + # Add service_tier if configured + if self.groq_service_tier: + extra_body["service_tier"] = self.groq_service_tier + # Add reasoning parameters for reasoning models + if is_reasoning_model: + extra_body["include_reasoning"] = False + if extra_body: + call_params["extra_body"] = extra_body + + # Prepare response format ONCE before retry loop + if response_format is not None: + schema = None + if hasattr(response_format, "model_json_schema"): + schema = response_format.model_json_schema() + + if strict_schema and schema is not None: + # Use OpenAI's strict JSON schema enforcement + call_params["response_format"] = { + "type": "json_schema", + "json_schema": { + "name": "response", + "strict": True, + "schema": schema, + }, + } + else: + # Soft enforcement: add schema to prompt and use json_object mode + if schema is not None: + schema_msg = ( + f"\n\nYou must respond with valid JSON matching this schema:\n{json.dumps(schema, indent=2)}" + ) + + if call_params["messages"] and call_params["messages"][0].get("role") == "system": + first_msg = call_params["messages"][0] + if isinstance(first_msg, dict) and isinstance(first_msg.get("content"), str): + first_msg["content"] += schema_msg + elif call_params["messages"]: + first_msg = call_params["messages"][0] + if isinstance(first_msg, dict) and isinstance(first_msg.get("content"), str): + first_msg["content"] = schema_msg + "\n\n" + first_msg["content"] + if self.provider not in ("lmstudio", "ollama"): + # LM Studio and Ollama don't support json_object response format reliably + call_params["response_format"] = {"type": "json_object"} + + last_exception = None + + for attempt in range(max_retries + 1): + try: + if response_format is not None: + response = await self._client.chat.completions.create(**call_params) + + content = response.choices[0].message.content + + # Strip reasoning model thinking tags + # Supports: , , , |startthink|/|endthink| + if content: + original_len = len(content) + content = re.sub(r".*?", "", content, flags=re.DOTALL) + content = re.sub(r".*?", "", content, flags=re.DOTALL) + content = re.sub(r".*?", "", content, flags=re.DOTALL) + content = re.sub(r"\|startthink\|.*?\|endthink\|", "", content, flags=re.DOTALL) + content = content.strip() + if len(content) < original_len: + logger.debug(f"Stripped {original_len - len(content)} chars of reasoning tokens") + + # For local models, they may wrap JSON in markdown code blocks + if self.provider in ("lmstudio", "ollama"): + clean_content = content + if "```json" in content: + clean_content = content.split("```json")[1].split("```")[0].strip() + elif "```" in content: + clean_content = content.split("```")[1].split("```")[0].strip() + try: + json_data = json.loads(clean_content) + except json.JSONDecodeError: + # Fallback to parsing raw content + json_data = json.loads(content) + else: + # Log raw LLM response for debugging JSON parse issues + try: + json_data = json.loads(content) + except json.JSONDecodeError as json_err: + # Truncate content for logging + content_preview = content[:500] if content else "" + if content and len(content) > 700: + content_preview = f"{content[:500]}...TRUNCATED...{content[-200:]}" + logger.warning( + f"JSON parse error from LLM response (attempt {attempt + 1}/{max_retries + 1}): {json_err}\n" + f" Model: {self.provider}/{self.model}\n" + f" Content length: {len(content) if content else 0} chars\n" + f" Content preview: {content_preview!r}\n" + f" Finish reason: {response.choices[0].finish_reason if response.choices else 'unknown'}" + ) + # Retry on JSON parse errors + if attempt < max_retries: + backoff = min(initial_backoff * (2**attempt), max_backoff) + await asyncio.sleep(backoff) + last_exception = json_err + continue + else: + logger.error(f"JSON parse error after {max_retries + 1} attempts, giving up") + raise + + if skip_validation: + result = json_data + else: + result = response_format.model_validate(json_data) + else: + response = await self._client.chat.completions.create(**call_params) + result = response.choices[0].message.content + + # Record token usage metrics + duration = time.time() - start_time + usage = response.usage + input_tokens = usage.prompt_tokens or 0 if usage else 0 + output_tokens = usage.completion_tokens or 0 if usage else 0 + total_tokens = usage.total_tokens or 0 if usage else 0 + + # Record LLM metrics + metrics = get_metrics_collector() + metrics.record_llm_call( + provider=self.provider, + model=self.model, + scope=scope, + duration=duration, + input_tokens=input_tokens, + output_tokens=output_tokens, + success=True, + ) + + # Log slow calls + if duration > 10.0 and usage: + ratio = max(1, output_tokens) / max(1, input_tokens) + cached_tokens = 0 + if hasattr(usage, "prompt_tokens_details") and usage.prompt_tokens_details: + cached_tokens = getattr(usage.prompt_tokens_details, "cached_tokens", 0) or 0 + cache_info = f", cached_tokens={cached_tokens}" if cached_tokens > 0 else "" + logger.info( + f"slow llm call: scope={scope}, model={self.provider}/{self.model}, " + f"input_tokens={input_tokens}, output_tokens={output_tokens}, " + f"total_tokens={total_tokens}{cache_info}, time={duration:.3f}s, ratio out/in={ratio:.2f}" + ) + + if return_usage: + token_usage = TokenUsage( + input_tokens=input_tokens, + output_tokens=output_tokens, + total_tokens=total_tokens, + ) + return result, token_usage + return result + + except LengthFinishReasonError as e: + logger.warning(f"LLM output exceeded token limits: {str(e)}") + raise OutputTooLongError( + "LLM output exceeded token limits. Input may need to be split into smaller chunks." + ) from e + + except APIConnectionError as e: + last_exception = e + status_code = getattr(e, "status_code", None) or getattr( + getattr(e, "response", None), "status_code", None + ) + logger.warning(f"APIConnectionError (HTTP {status_code}), attempt {attempt + 1}: {str(e)[:200]}") + if attempt < max_retries: + backoff = min(initial_backoff * (2**attempt), max_backoff) + await asyncio.sleep(backoff) + continue + else: + logger.error(f"Connection error after {max_retries + 1} attempts: {str(e)}") + raise + + except APIStatusError as e: + # Fast fail only on 401 (unauthorized) and 403 (forbidden) + if e.status_code in (401, 403): + logger.error(f"Auth error (HTTP {e.status_code}), not retrying: {str(e)}") + raise + + # Handle tool_use_failed error - model outputted in tool call format + if e.status_code == 400 and response_format is not None: + try: + error_body = e.body if hasattr(e, "body") else {} + if isinstance(error_body, dict): + error_info: dict[str, Any] = error_body.get("error") or {} + if error_info.get("code") == "tool_use_failed": + failed_gen = error_info.get("failed_generation", "") + if failed_gen: + # Parse tool call format and convert to expected format + tool_call = json.loads(failed_gen) + tool_name = tool_call.get("name", "") + tool_args = tool_call.get("arguments", {}) + converted = {"actions": [{"tool": tool_name, **tool_args}]} + if skip_validation: + result = converted + else: + result = response_format.model_validate(converted) + + # Record metrics + duration = time.time() - start_time + metrics = get_metrics_collector() + metrics.record_llm_call( + provider=self.provider, + model=self.model, + scope=scope, + duration=duration, + input_tokens=0, + output_tokens=0, + success=True, + ) + if return_usage: + return result, TokenUsage(input_tokens=0, output_tokens=0, total_tokens=0) + return result + except (json.JSONDecodeError, KeyError, TypeError): + pass # Failed to parse tool_use_failed, continue with normal retry + + last_exception = e + if attempt < max_retries: + backoff = min(initial_backoff * (2**attempt), max_backoff) + jitter = backoff * 0.2 * (2 * (time.time() % 1) - 1) + sleep_time = backoff + jitter + await asyncio.sleep(sleep_time) + else: + logger.error(f"API error after {max_retries + 1} attempts: {str(e)}") + raise + + except Exception: + raise + + if last_exception: + raise last_exception + raise RuntimeError("LLM call failed after all retries with no exception captured") + + async def call_with_tools( + self, + messages: list[dict[str, Any]], + tools: list[dict[str, Any]], + max_completion_tokens: int | None = None, + temperature: float | None = None, + scope: str = "tools", + max_retries: int = 5, + initial_backoff: float = 1.0, + max_backoff: float = 30.0, + tool_choice: str | dict[str, Any] = "auto", + ) -> LLMToolCallResult: + """ + Make an LLM API call with tool/function calling support. + + Args: + messages: List of message dicts. Can include tool results with role='tool'. + tools: List of tool definitions in OpenAI format. + max_completion_tokens: Maximum tokens in response. + temperature: Sampling temperature (0.0-2.0). + scope: Scope identifier for tracking. + max_retries: Maximum retry attempts. + initial_backoff: Initial backoff time in seconds. + max_backoff: Maximum backoff time in seconds. + tool_choice: How to choose tools - "auto", "none", "required", or specific function. + + Returns: + LLMToolCallResult with content and/or tool_calls. + """ + start_time = time.time() + + # Build call parameters + call_params: dict[str, Any] = { + "model": self.model, + "messages": messages, + "tools": tools, + "tool_choice": tool_choice, + } + + if max_completion_tokens is not None: + call_params["max_completion_tokens"] = max_completion_tokens + if temperature is not None: + call_params["temperature"] = temperature + + # Provider-specific parameters + if self.provider == "groq": + call_params["seed"] = DEFAULT_LLM_SEED + + last_exception = None + + for attempt in range(max_retries + 1): + try: + response = await self._client.chat.completions.create(**call_params) + + message = response.choices[0].message + finish_reason = response.choices[0].finish_reason + + # Extract tool calls if present + tool_calls: list[LLMToolCall] = [] + if message.tool_calls: + for tc in message.tool_calls: + try: + args = json.loads(tc.function.arguments) if tc.function.arguments else {} + except json.JSONDecodeError: + args = {"_raw": tc.function.arguments} + tool_calls.append(LLMToolCall(id=tc.id, name=tc.function.name, arguments=args)) + + content = message.content + + # Record metrics + duration = time.time() - start_time + usage = response.usage + input_tokens = usage.prompt_tokens or 0 if usage else 0 + output_tokens = usage.completion_tokens or 0 if usage else 0 + + metrics = get_metrics_collector() + metrics.record_llm_call( + provider=self.provider, + model=self.model, + scope=scope, + duration=duration, + input_tokens=input_tokens, + output_tokens=output_tokens, + success=True, + ) + + return LLMToolCallResult( + content=content, + tool_calls=tool_calls, + finish_reason=finish_reason, + input_tokens=input_tokens, + output_tokens=output_tokens, + ) + + except APIConnectionError as e: + last_exception = e + if attempt < max_retries: + await asyncio.sleep(min(initial_backoff * (2**attempt), max_backoff)) + continue + raise + + except APIStatusError as e: + if e.status_code in (401, 403): + raise + last_exception = e + if attempt < max_retries: + await asyncio.sleep(min(initial_backoff * (2**attempt), max_backoff)) + continue + raise + + except Exception: + raise + + if last_exception: + raise last_exception + raise RuntimeError("Tool call failed after all retries") + + async def _call_ollama_native( + self, + messages: list[dict[str, str]], + response_format: Any, + max_completion_tokens: int | None, + temperature: float | None, + max_retries: int, + initial_backoff: float, + max_backoff: float, + skip_validation: bool, + scope: str = "memory", + return_usage: bool = False, + ) -> Any: + """ + Call Ollama using native API with JSON schema enforcement. + + Ollama's native API supports passing a full JSON schema in the 'format' parameter, + which provides better structured output control than the OpenAI-compatible API. + """ + start_time = time.time() + + # Get the JSON schema from the Pydantic model + schema = response_format.model_json_schema() if hasattr(response_format, "model_json_schema") else None + + # Build the base URL for Ollama's native API + # Default OpenAI-compatible URL is http://localhost:11434/v1 + # Native API is at http://localhost:11434/api/chat + base_url = self.base_url or "http://localhost:11434/v1" + if base_url.endswith("/v1"): + native_url = base_url[:-3] + "/api/chat" + else: + native_url = base_url.rstrip("/") + "/api/chat" + + # Build request payload + payload: dict[str, Any] = { + "model": self.model, + "messages": messages, + "stream": False, + } + + # Add schema as format parameter for structured output + if schema: + payload["format"] = schema + + # Add optional parameters with optimized defaults for Ollama + options: dict[str, Any] = { + "num_ctx": 16384, # 16k context window for larger prompts + "num_batch": 512, # Optimal batch size for prompt processing + } + if max_completion_tokens: + options["num_predict"] = max_completion_tokens + if temperature is not None: + options["temperature"] = temperature + payload["options"] = options + + last_exception = None + + async with httpx.AsyncClient(timeout=300.0) as client: + for attempt in range(max_retries + 1): + try: + response = await client.post(native_url, json=payload) + response.raise_for_status() + + result = response.json() + content = result.get("message", {}).get("content", "") + + # Parse JSON response + try: + json_data = json.loads(content) + except json.JSONDecodeError as json_err: + content_preview = content[:500] if content else "" + if content and len(content) > 700: + content_preview = f"{content[:500]}...TRUNCATED...{content[-200:]}" + logger.warning( + f"Ollama JSON parse error (attempt {attempt + 1}/{max_retries + 1}): {json_err}\n" + f" Model: ollama/{self.model}\n" + f" Content length: {len(content) if content else 0} chars\n" + f" Content preview: {content_preview!r}" + ) + if attempt < max_retries: + backoff = min(initial_backoff * (2**attempt), max_backoff) + await asyncio.sleep(backoff) + last_exception = json_err + continue + else: + raise + + # Extract token usage from Ollama response + duration = time.time() - start_time + input_tokens = result.get("prompt_eval_count", 0) or 0 + output_tokens = result.get("eval_count", 0) or 0 + total_tokens = input_tokens + output_tokens + + # Record LLM metrics + metrics = get_metrics_collector() + metrics.record_llm_call( + provider=self.provider, + model=self.model, + scope=scope, + duration=duration, + input_tokens=input_tokens, + output_tokens=output_tokens, + success=True, + ) + + # Validate against Pydantic model or return raw JSON + if skip_validation: + validated_result = json_data + else: + validated_result = response_format.model_validate(json_data) + + if return_usage: + token_usage = TokenUsage( + input_tokens=input_tokens, + output_tokens=output_tokens, + total_tokens=total_tokens, + ) + return validated_result, token_usage + return validated_result + + except httpx.HTTPStatusError as e: + last_exception = e + if attempt < max_retries: + logger.warning( + f"Ollama HTTP error (attempt {attempt + 1}/{max_retries + 1}): {e.response.status_code}" + ) + backoff = min(initial_backoff * (2**attempt), max_backoff) + await asyncio.sleep(backoff) + continue + else: + logger.error(f"Ollama HTTP error after {max_retries + 1} attempts: {e}") + raise + + except httpx.RequestError as e: + last_exception = e + if attempt < max_retries: + logger.warning(f"Ollama connection error (attempt {attempt + 1}/{max_retries + 1}): {e}") + backoff = min(initial_backoff * (2**attempt), max_backoff) + await asyncio.sleep(backoff) + continue + else: + logger.error(f"Ollama connection error after {max_retries + 1} attempts: {e}") + raise + + except Exception as e: + logger.error(f"Unexpected error during Ollama call: {type(e).__name__}: {e}") + raise + + if last_exception: + raise last_exception + raise RuntimeError("Ollama call failed after all retries") + + async def cleanup(self) -> None: + """Clean up resources (close OpenAI client connections).""" + if hasattr(self, "_client") and self._client: + await self._client.close() diff --git a/hindsight-api/pyproject.toml b/hindsight-api/pyproject.toml index cb1fc579..ec8d8e3f 100644 --- a/hindsight-api/pyproject.toml +++ b/hindsight-api/pyproject.toml @@ -25,7 +25,7 @@ dependencies = [ "psycopg2-binary>=2.9.11", "tiktoken>=0.12.0", "httpx>=0.27.0", - "fastmcp>=2.14.0", # CVE-2025-66416 + "fastmcp>=2.14.0", # CVE-2025-66416 "pg0-embedded>=0.11.0", "python-dateutil>=2.8.0", "opentelemetry-api>=1.20.0", @@ -41,16 +41,17 @@ dependencies = [ "flashrank>=0.2.0", # Local ML models for embeddings/reranking - can be excluded in Docker with INCLUDE_LOCAL_MODELS=false "sentence-transformers>=3.3.0", - "transformers>=4.53.0", # Security fixes for ReDoS vulnerabilities - "torch>=2.6.0", # CVE fix for remote code execution + "transformers>=4.53.0", # Security fixes for ReDoS vulnerabilities + "torch>=2.6.0", # CVE fix for remote code execution "uvloop>=0.22.1", # Transitive dependency security fixes - "pyasn1>=0.6.2", # DoS vulnerability fix - "urllib3>=2.6.3", # Decompression-bomb safeguards bypass fix - "langchain-core>=1.2.5", # Serialization injection vulnerability fix - "filelock>=3.20.1", # TOCTOU race condition fix - "authlib>=1.6.6", # Account takeover vulnerability fix - "aiohttp>=3.13.3", # Multiple DoS vulnerabilities + "pyasn1>=0.6.2", # DoS vulnerability fix + "urllib3>=2.6.3", # Decompression-bomb safeguards bypass fix + "langchain-core>=1.2.5", # Serialization injection vulnerability fix + "filelock>=3.20.1", # TOCTOU race condition fix + "authlib>=1.6.6", # Account takeover vulnerability fix + "aiohttp>=3.13.3", # Multiple DoS vulnerabilities + "claude-agent-sdk>=0.1.27", ] [project.optional-dependencies] diff --git a/hindsight-api/tests/test_llm_provider.py b/hindsight-api/tests/test_llm_provider.py index 4f26606a..df4f0975 100644 --- a/hindsight-api/tests/test_llm_provider.py +++ b/hindsight-api/tests/test_llm_provider.py @@ -19,6 +19,10 @@ MODEL_MATRIX = [ ("openai", "gpt-5-nano"), ("openai", "gpt-5"), ("openai", "gpt-5.2"), + # Anthropic models + ("anthropic", "claude-sonnet-4-20250514"), + ("anthropic", "claude-opus-4-5-20251101"), + ("anthropic", "claude-haiku-4-20250514"), # Groq models ("groq", "openai/gpt-oss-120b"), ("groq", "openai/gpt-oss-20b"), @@ -36,6 +40,7 @@ def get_api_key_for_provider(provider: str) -> str | None: """Get API key for provider from environment variables.""" provider_key_map = { "openai": "OPENAI_API_KEY", + "anthropic": "ANTHROPIC_API_KEY", "groq": "GROQ_API_KEY", "gemini": "GEMINI_API_KEY", } diff --git a/hindsight-api/tests/test_llm_token_metrics.py b/hindsight-api/tests/test_llm_token_metrics.py index 87a24cd6..54986bd4 100644 --- a/hindsight-api/tests/test_llm_token_metrics.py +++ b/hindsight-api/tests/test_llm_token_metrics.py @@ -30,7 +30,8 @@ async def test_llm_metrics_recorded_for_groq(): # Create a mock metrics collector to track record_llm_call calls mock_collector = MagicMock(spec=MetricsCollector) - with patch("hindsight_api.engine.llm_wrapper.get_metrics_collector", return_value=mock_collector): + # Patch the provider module where get_metrics_collector is actually called + with patch("hindsight_api.engine.providers.openai_compatible_llm.get_metrics_collector", return_value=mock_collector): llm = LLMProvider( provider="groq", api_key=api_key, @@ -90,7 +91,8 @@ async def test_llm_metrics_recorded_for_structured_output(): mock_collector = MagicMock(spec=MetricsCollector) - with patch("hindsight_api.engine.llm_wrapper.get_metrics_collector", return_value=mock_collector): + # Patch the provider module where get_metrics_collector is actually called + with patch("hindsight_api.engine.providers.openai_compatible_llm.get_metrics_collector", return_value=mock_collector): llm = LLMProvider( provider="groq", api_key=api_key, diff --git a/hindsight-api/tests/test_provider_default_models.py b/hindsight-api/tests/test_provider_default_models.py new file mode 100644 index 00000000..433d8b94 --- /dev/null +++ b/hindsight-api/tests/test_provider_default_models.py @@ -0,0 +1,123 @@ +"""Test provider-specific default models in config.""" + +import os + +import pytest + + +def test_provider_default_models(): + """Test that each provider has a default model and it's used when model is not explicitly set.""" + from hindsight_api.config import PROVIDER_DEFAULT_MODELS, HindsightConfig, clear_config_cache + + # Save original env vars + original_provider = os.environ.get("HINDSIGHT_API_LLM_PROVIDER") + original_model = os.environ.get("HINDSIGHT_API_LLM_MODEL") + + try: + # Test each provider has a default + for provider, expected_model in PROVIDER_DEFAULT_MODELS.items(): + clear_config_cache() + os.environ["HINDSIGHT_API_LLM_PROVIDER"] = provider + # Remove explicit model setting to test default + if "HINDSIGHT_API_LLM_MODEL" in os.environ: + del os.environ["HINDSIGHT_API_LLM_MODEL"] + + config = HindsightConfig.from_env() + assert config.llm_provider == provider, f"Provider mismatch for {provider}" + assert config.llm_model == expected_model, f"Expected {expected_model} for {provider}, got {config.llm_model}" + + finally: + # Restore original env vars + clear_config_cache() + if original_provider: + os.environ["HINDSIGHT_API_LLM_PROVIDER"] = original_provider + elif "HINDSIGHT_API_LLM_PROVIDER" in os.environ: + del os.environ["HINDSIGHT_API_LLM_PROVIDER"] + + if original_model: + os.environ["HINDSIGHT_API_LLM_MODEL"] = original_model + elif "HINDSIGHT_API_LLM_MODEL" in os.environ: + del os.environ["HINDSIGHT_API_LLM_MODEL"] + + +def test_explicit_model_overrides_provider_default(): + """Test that explicit model setting overrides provider default.""" + from hindsight_api.config import HindsightConfig, clear_config_cache + + original_provider = os.environ.get("HINDSIGHT_API_LLM_PROVIDER") + original_model = os.environ.get("HINDSIGHT_API_LLM_MODEL") + + try: + clear_config_cache() + os.environ["HINDSIGHT_API_LLM_PROVIDER"] = "anthropic" + os.environ["HINDSIGHT_API_LLM_MODEL"] = "claude-sonnet-4-5-20250929" + + config = HindsightConfig.from_env() + assert config.llm_provider == "anthropic" + assert config.llm_model == "claude-sonnet-4-5-20250929", "Explicit model should override default" + + finally: + clear_config_cache() + if original_provider: + os.environ["HINDSIGHT_API_LLM_PROVIDER"] = original_provider + elif "HINDSIGHT_API_LLM_PROVIDER" in os.environ: + del os.environ["HINDSIGHT_API_LLM_PROVIDER"] + + if original_model: + os.environ["HINDSIGHT_API_LLM_MODEL"] = original_model + elif "HINDSIGHT_API_LLM_MODEL" in os.environ: + del os.environ["HINDSIGHT_API_LLM_MODEL"] + + +def test_per_operation_provider_default_model(): + """Test that per-operation providers use their own default models.""" + from hindsight_api.config import HindsightConfig, clear_config_cache + + original_provider = os.environ.get("HINDSIGHT_API_LLM_PROVIDER") + original_model = os.environ.get("HINDSIGHT_API_LLM_MODEL") + original_retain_provider = os.environ.get("HINDSIGHT_API_RETAIN_LLM_PROVIDER") + original_retain_model = os.environ.get("HINDSIGHT_API_RETAIN_LLM_MODEL") + + try: + clear_config_cache() + os.environ["HINDSIGHT_API_LLM_PROVIDER"] = "openai" + # Remove explicit model to use provider default + if "HINDSIGHT_API_LLM_MODEL" in os.environ: + del os.environ["HINDSIGHT_API_LLM_MODEL"] + + # Set retain-specific provider but not model + os.environ["HINDSIGHT_API_RETAIN_LLM_PROVIDER"] = "anthropic" + if "HINDSIGHT_API_RETAIN_LLM_MODEL" in os.environ: + del os.environ["HINDSIGHT_API_RETAIN_LLM_MODEL"] + + config = HindsightConfig.from_env() + + # Global LLM should use OpenAI default + assert config.llm_model == "o3-mini", f"Expected o3-mini, got {config.llm_model}" + + # Retain should use Anthropic default + assert ( + config.retain_llm_model == "claude-haiku-4-5-20251001" + ), f"Expected claude-haiku-4-5-20251001, got {config.retain_llm_model}" + + finally: + clear_config_cache() + if original_provider: + os.environ["HINDSIGHT_API_LLM_PROVIDER"] = original_provider + elif "HINDSIGHT_API_LLM_PROVIDER" in os.environ: + del os.environ["HINDSIGHT_API_LLM_PROVIDER"] + + if original_model: + os.environ["HINDSIGHT_API_LLM_MODEL"] = original_model + elif "HINDSIGHT_API_LLM_MODEL" in os.environ: + del os.environ["HINDSIGHT_API_LLM_MODEL"] + + if original_retain_provider: + os.environ["HINDSIGHT_API_RETAIN_LLM_PROVIDER"] = original_retain_provider + elif "HINDSIGHT_API_RETAIN_LLM_PROVIDER" in os.environ: + del os.environ["HINDSIGHT_API_RETAIN_LLM_PROVIDER"] + + if original_retain_model: + os.environ["HINDSIGHT_API_RETAIN_LLM_MODEL"] = original_retain_model + elif "HINDSIGHT_API_RETAIN_LLM_MODEL" in os.environ: + del os.environ["HINDSIGHT_API_RETAIN_LLM_MODEL"] diff --git a/hindsight-docs/docs/developer/configuration.md b/hindsight-docs/docs/developer/configuration.md index fa0fad81..b9d68509 100644 --- a/hindsight-docs/docs/developer/configuration.md +++ b/hindsight-docs/docs/developer/configuration.md @@ -61,7 +61,7 @@ hindsight-admin run-db-migration --schema tenant_acme | Variable | Description | Default | |----------|-------------|---------| -| `HINDSIGHT_API_LLM_PROVIDER` | Provider: `openai`, `anthropic`, `gemini`, `groq`, `ollama`, `lmstudio`, `vertexai` | `openai` | +| `HINDSIGHT_API_LLM_PROVIDER` | Provider: `openai`, `openai-codex`, `claude-code`, `anthropic`, `gemini`, `groq`, `ollama`, `lmstudio`, `vertexai` | `openai` | | `HINDSIGHT_API_LLM_API_KEY` | API key for LLM provider | - | | `HINDSIGHT_API_LLM_MODEL` | Model name | `gpt-5-mini` | | `HINDSIGHT_API_LLM_BASE_URL` | Custom LLM endpoint | Provider default | @@ -120,8 +120,22 @@ export HINDSIGHT_API_LLM_PROVIDER=openai export HINDSIGHT_API_LLM_BASE_URL=https://your-endpoint.com/v1 export HINDSIGHT_API_LLM_API_KEY=your-api-key export HINDSIGHT_API_LLM_MODEL=your-model-name + +# OpenAI Codex (ChatGPT Plus/Pro subscription - uses OAuth, no API key needed) +export HINDSIGHT_API_LLM_PROVIDER=openai-codex +export HINDSIGHT_API_LLM_MODEL=gpt-5.2-codex +# No API key needed - uses OAuth tokens from ~/.codex/auth.json + +# Claude Code (Claude Pro/Max subscription - uses OAuth, no API key needed) +export HINDSIGHT_API_LLM_PROVIDER=claude-code +export HINDSIGHT_API_LLM_MODEL=claude-sonnet-4-5-20250929 +# No API key needed - uses claude auth login credentials ``` +:::tip OpenAI Codex & Claude Code Setup +For detailed setup instructions for **OpenAI Codex** (ChatGPT Plus/Pro) and **Claude Code** (Claude Pro/Max), see the [Models documentation](./models#openai-codex-setup-chatgpt-pluspro). +::: + #### Vertex AI Setup Google Cloud's Vertex AI provides access to Gemini models via the native Google GenAI SDK. Hindsight supports two authentication methods: diff --git a/hindsight-docs/docs/developer/models.md b/hindsight-docs/docs/developer/models.md index 2214970e..e9a60907 100644 --- a/hindsight-docs/docs/developer/models.md +++ b/hindsight-docs/docs/developer/models.md @@ -47,6 +47,46 @@ The following models have been tested and verified to work correctly with Hindsi | **Groq** | `openai/gpt-oss-120b` | | **Groq** | `openai/gpt-oss-20b` | +### Provider Default Models + +Each provider has a recommended default model that's used when `HINDSIGHT_API_LLM_MODEL` is not explicitly set. This makes configuration simpler - just specify the provider and get a sensible default: + +| Provider | Default Model | +|----------|--------------| +| `openai` | `o3-mini` | +| `anthropic` | `claude-haiku-4-5-20251001` | +| `gemini` | `gemini-2.5-flash` | +| `groq` | `openai/gpt-oss-120b` | +| `ollama` | `gemma3:12b` | +| `lmstudio` | `local-model` | +| `vertexai` | `gemini-2.0-flash-001` | +| `openai-codex` | `gpt-5.2-codex` | +| `claude-code` | `claude-sonnet-4-5-20250929` | + +**Example:** Setting just the provider uses its default model: +```bash +# Uses claude-haiku-4-5-20251001 automatically +export HINDSIGHT_API_LLM_PROVIDER=anthropic +export HINDSIGHT_API_LLM_API_KEY=sk-ant-xxxxxxxxxxxx +``` + +You can override the default by explicitly setting `HINDSIGHT_API_LLM_MODEL`: +```bash +# Override to use Sonnet instead +export HINDSIGHT_API_LLM_PROVIDER=anthropic +export HINDSIGHT_API_LLM_API_KEY=sk-ant-xxxxxxxxxxxx +export HINDSIGHT_API_LLM_MODEL=claude-sonnet-4-5-20250929 +``` + +This also applies to per-operation overrides: +```bash +# Global: OpenAI o3-mini (default) +export HINDSIGHT_API_LLM_PROVIDER=openai + +# Retain: Anthropic claude-haiku-4-5-20251001 (default) +export HINDSIGHT_API_RETAIN_LLM_PROVIDER=anthropic +``` + ### Using Other Models Other LLM models not listed above may work with Hindsight, but they must support **at least 65,000 output tokens** to ensure reliable fact extraction. If you need support for a specific model that doesn't meet this requirement, please [open an issue](https://github.com/hindsight-ai/hindsight/issues) to request an exception. @@ -89,6 +129,148 @@ export HINDSIGHT_API_LLM_MODEL=your-local-model --- +### OpenAI Codex Setup (ChatGPT Plus/Pro) + +Use your ChatGPT Plus or Pro subscription for Hindsight without separate OpenAI Platform API costs. + +**Prerequisites:** +- Active ChatGPT Plus or Pro subscription +- Node.js/npm installed (for Codex CLI) + +**Setup Steps:** + +1. **Install Codex CLI:** + ```bash + npm install -g @openai/codex + ``` + +2. **Login with ChatGPT credentials:** + ```bash + codex auth login + ``` + This opens a browser window to authenticate with your ChatGPT account and saves OAuth tokens to `~/.codex/auth.json`. + +3. **Verify authentication:** + ```bash + ls ~/.codex/auth.json # Should show the auth file exists + ``` + +4. **Configure Hindsight:** + ```bash + export HINDSIGHT_API_LLM_PROVIDER=openai-codex + export HINDSIGHT_API_LLM_MODEL=gpt-5.2-codex # or gpt-5.1-codex + # No API key needed - reads from ~/.codex/auth.json automatically + ``` + +5. **Start Hindsight:** + ```bash + ./scripts/dev/start-api.sh + ``` + +**Available Models:** +- `gpt-5.2-codex` - Latest frontier agentic coding model (default) +- `gpt-5.2` - Latest frontier model +- `gpt-5.1-codex` - Previous generation coding model +- `gpt-5.1-codex-max` - Maximum context variant +- `gpt-5.1-codex-mini` - Lightweight variant + +**Important Notes:** +- OAuth tokens are stored in `~/.codex/auth.json` +- Tokens refresh automatically when needed +- Usage is billed to your ChatGPT subscription (not separate API costs) +- For personal development use only (see ChatGPT Terms of Service) + +**Troubleshooting:** + +If authentication fails: +```bash +# Re-login to refresh tokens +codex auth login + +# Check auth file exists and has correct format +cat ~/.codex/auth.json | python3 -c "import json, sys; d=json.load(sys.stdin); print('auth_mode:', d.get('auth_mode')); print('has tokens:', 'tokens' in d)" +``` + +--- + +### Claude Code Setup (Claude Pro/Max) + +Use your Claude Pro or Max subscription for Hindsight without separate Anthropic API costs. + +**Prerequisites:** +- Active Claude Pro or Max subscription +- Claude Code CLI installed + +**Setup Steps:** + +1. **Install Claude Code CLI:** + ```bash + npm install -g @anthropics/claude-code + # Or via Homebrew + brew install anthropics/claude-code/claude-code + ``` + +2. **Login with Claude credentials:** + ```bash + claude auth login + ``` + This opens a browser window to authenticate with your Claude account. Authentication is automatically managed by the Claude Agent SDK. + +3. **Verify authentication:** + ```bash + claude --version + # Should show version without errors + ``` + +4. **Configure Hindsight:** + ```bash + export HINDSIGHT_API_LLM_PROVIDER=claude-code + export HINDSIGHT_API_LLM_MODEL=claude-sonnet-4-5-20250929 + # No API key needed - uses claude auth login credentials + ``` + +5. **Start Hindsight:** + ```bash + ./scripts/dev/start-api.sh + ``` + +**Available Models:** +- `claude-sonnet-4-5-20250929` - Latest Claude Sonnet (default) +- `claude-opus-4-20250514` - Claude Opus for complex tasks +- `claude-sonnet-3-5-20241022` - Previous generation Sonnet +- Any model supported by Claude Code CLI + +**Important Notes:** +- Authentication handled by Claude Agent SDK (uses bundled CLI) +- Credentials managed securely by Claude Code +- Usage billed to your Claude subscription (not separate API costs) +- Includes Claude Agent SDK as dependency (auto-installed) +- For personal development use only (see Claude Terms of Service) + +**Troubleshooting:** + +If authentication fails: +```bash +# Re-login to refresh credentials +claude auth login + +# Check Claude CLI is working +claude --version + +# Test authentication directly +claude query "test" +``` + +If the SDK is not found: +```bash +# Install Claude Agent SDK +pip install claude-agent-sdk +# Or with uv +uv add claude-agent-sdk +``` + +--- + ## Embedding Model Converts text into dense vector representations for semantic similarity search. diff --git a/uv.lock b/uv.lock index 9fa15c6a..5b7e3867 100644 --- a/uv.lock +++ b/uv.lock @@ -590,6 +590,22 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/0a/4c/925909008ed5a988ccbb72dcc897407e5d6d3bd72410d69e051fc0c14647/charset_normalizer-3.4.4-py3-none-any.whl", hash = "sha256:7a32c560861a02ff789ad905a2fe94e3f840803362c84fecf1851cb4cf3dc37f", size = 53402 }, ] +[[package]] +name = "claude-agent-sdk" +version = "0.1.27" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "anyio" }, + { name = "mcp" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/ce/ef/0e51909e5a6e39d7c9e4073fdd3e00ff70677f99f8d1b87adef329c34acc/claude_agent_sdk-0.1.27.tar.gz", hash = "sha256:d2f4fc4c5e5c088efbaf66c34efcfd2aa7efafa3fed82f5cb1a95c451df96c38", size = 57216 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/66/fe/52b1e8394428ddafd952f41799bb4c8b0e60627b808ee2d797644da02624/claude_agent_sdk-0.1.27-py3-none-macosx_11_0_arm64.whl", hash = "sha256:eddfe7fa40fdbd0a49fafd5698791bc911bc1e66e6ace2f77c50d5b64e138e93", size = 53901311 }, + { url = "https://files.pythonhosted.org/packages/9f/eb/69dedbb195b69bd4b2ebf127407778e89c56e547e02bbcb74c130e1584c4/claude_agent_sdk-0.1.27-py3-none-manylinux_2_17_aarch64.whl", hash = "sha256:babf796d478a2b7ff75afab61d47bede4afecd6c793b7d540ee3aab42f00d5fb", size = 68107707 }, + { url = "https://files.pythonhosted.org/packages/84/06/886931dcbce8cd586aa38afa3ebdefe7d9eaa4ad389fa795560317c1f891/claude_agent_sdk-0.1.27-py3-none-manylinux_2_17_x86_64.whl", hash = "sha256:de0e22f3408ce7bdf909218e28be0e317b8d7d64b855cefc2cb3dd022f5f887b", size = 69810719 }, + { url = "https://files.pythonhosted.org/packages/d8/ea/c987078f5059f05756886609f3196c8aeebe10f4e79c1f82f58b71eaeb9f/claude_agent_sdk-0.1.27-py3-none-win_amd64.whl", hash = "sha256:23fbb90727cd4dc776ad894a1b2dc040fb9fc2f0277a32b94336665e7c950692", size = 71994821 }, +] + [[package]] name = "click" version = "8.3.0" @@ -1327,6 +1343,7 @@ dependencies = [ { name = "anthropic" }, { name = "asyncpg" }, { name = "authlib" }, + { name = "claude-agent-sdk" }, { name = "cohere" }, { name = "dateparser" }, { name = "fastapi", extra = ["standard"] }, @@ -1392,6 +1409,7 @@ requires-dist = [ { name = "anthropic", specifier = ">=0.40.0" }, { name = "asyncpg", specifier = ">=0.29.0" }, { name = "authlib", specifier = ">=1.6.6" }, + { name = "claude-agent-sdk", specifier = ">=0.1.27" }, { name = "cohere", specifier = ">=5.0.0" }, { name = "dateparser", specifier = ">=1.2.2" }, { name = "fastapi", extras = ["standard"], specifier = ">=0.120.3" },