diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 6881554d..eac11167 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -461,6 +461,9 @@ jobs: OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} COHERE_API_KEY: ${{ secrets.COHERE_API_KEY }} HINDSIGHT_API_EMBEDDINGS_OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + AWS_REGION_NAME: ${{ secrets.AWS_REGION_NAME }} GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} # Prefer CPU-only PyTorch in CI (but keep PyPI for everything else) diff --git a/hindsight-api-slim/hindsight_api/config.py b/hindsight-api-slim/hindsight_api/config.py index 8cb152b0..15572805 100644 --- a/hindsight-api-slim/hindsight_api/config.py +++ b/hindsight-api-slim/hindsight_api/config.py @@ -364,6 +364,8 @@ PROVIDER_DEFAULT_MODELS = { "openai-codex": "gpt-5.2-codex", "claude-code": "claude-sonnet-4-5-20250929", "mock": "mock-model", + "litellm": "gpt-4o-mini", + "bedrock": "us.amazon.nova-2-lite-v1:0", } DEFAULT_LLM_MODEL = "gpt-4o-mini" # Fallback if provider not in table DEFAULT_LLM_MAX_CONCURRENT = 32 diff --git a/hindsight-api-slim/hindsight_api/engine/llm_wrapper.py b/hindsight-api-slim/hindsight_api/engine/llm_wrapper.py index 4fd4c2e1..7a8e46c1 100644 --- a/hindsight-api-slim/hindsight_api/engine/llm_wrapper.py +++ b/hindsight-api-slim/hindsight_api/engine/llm_wrapper.py @@ -126,6 +126,8 @@ _PROVIDERS_WITHOUT_API_KEY = frozenset( "claude-code", "mock", "vertexai", + "litellm", + "bedrock", } ) @@ -172,6 +174,7 @@ def create_llm_provider( ClaudeCodeLLM, CodexLLM, GeminiLLM, + LiteLLMLLM, MockLLM, OpenAICompatibleLLM, ) @@ -227,6 +230,26 @@ def create_llm_provider( reasoning_effort=reasoning_effort, ) + elif provider_lower == "litellm": + return LiteLLMLLM( + provider=provider, + api_key=api_key, + base_url=base_url, + model=model, + reasoning_effort=reasoning_effort, + ) + + elif provider_lower == "bedrock": + # Bedrock is a first-class alias backed by LiteLLM with auto-prefixed model names + bedrock_model = model if model.startswith("bedrock/") else f"bedrock/{model}" + return LiteLLMLLM( + provider=provider, + api_key=api_key, + base_url=base_url, + model=bedrock_model, + reasoning_effort=reasoning_effort, + ) + elif provider_lower in ("openai", "groq", "ollama", "lmstudio", "minimax"): return OpenAICompatibleLLM( provider=provider, @@ -297,6 +320,8 @@ class LLMProvider: "claude-code", "mock", "minimax", + "litellm", + "bedrock", ] if self.provider not in valid_providers: raise ValueError(f"Invalid LLM provider: {self.provider}. Must be one of: {', '.join(valid_providers)}") @@ -675,10 +700,13 @@ class LLMProvider: api_key = os.getenv("HINDSIGHT_API_LLM_API_KEY", "") # API key not needed for openai-codex (uses OAuth), claude-code (uses Keychain OAuth), - # ollama (local), or vertexai (uses GCP service account credentials) - if not api_key and provider not in ("openai-codex", "claude-code", "ollama", "vertexai"): + # ollama (local), vertexai (uses GCP service account credentials), + # or litellm (uses provider-specific auth, e.g. AWS credentials for Bedrock) + if not api_key and not requires_api_key(provider): + pass # Provider handles its own auth + elif not api_key: raise ValueError( - "HINDSIGHT_API_LLM_API_KEY environment variable is required (unless using openai-codex or claude-code)" + "HINDSIGHT_API_LLM_API_KEY environment variable is required (unless using openai-codex, claude-code, or litellm)" ) base_url = os.getenv("HINDSIGHT_API_LLM_BASE_URL", "") @@ -692,12 +720,13 @@ class LLMProvider: provider = os.getenv("HINDSIGHT_API_ANSWER_LLM_PROVIDER", os.getenv("HINDSIGHT_API_LLM_PROVIDER", "groq")) api_key = os.getenv("HINDSIGHT_API_ANSWER_LLM_API_KEY", os.getenv("HINDSIGHT_API_LLM_API_KEY", "")) - # API key not needed for openai-codex (uses OAuth), claude-code (uses Keychain OAuth), - # ollama (local), or vertexai (uses GCP service account credentials) - if not api_key and provider not in ("openai-codex", "claude-code", "ollama", "vertexai"): + # API key not needed for providers with their own auth mechanisms + if not api_key and not requires_api_key(provider): + pass # Provider handles its own auth + elif not api_key: raise ValueError( "HINDSIGHT_API_LLM_API_KEY or HINDSIGHT_API_ANSWER_LLM_API_KEY environment variable is required " - "(unless using openai-codex or claude-code)" + "(unless using openai-codex, claude-code, or litellm)" ) base_url = os.getenv("HINDSIGHT_API_ANSWER_LLM_BASE_URL", os.getenv("HINDSIGHT_API_LLM_BASE_URL", "")) @@ -711,12 +740,13 @@ class LLMProvider: provider = os.getenv("HINDSIGHT_API_JUDGE_LLM_PROVIDER", os.getenv("HINDSIGHT_API_LLM_PROVIDER", "groq")) api_key = os.getenv("HINDSIGHT_API_JUDGE_LLM_API_KEY", os.getenv("HINDSIGHT_API_LLM_API_KEY", "")) - # API key not needed for openai-codex (uses OAuth), claude-code (uses Keychain OAuth), - # ollama (local), or vertexai (uses GCP service account credentials) - if not api_key and provider not in ("openai-codex", "claude-code", "ollama", "vertexai"): + # API key not needed for providers with their own auth mechanisms + if not api_key and not requires_api_key(provider): + pass # Provider handles its own auth + elif not api_key: raise ValueError( "HINDSIGHT_API_LLM_API_KEY or HINDSIGHT_API_JUDGE_LLM_API_KEY environment variable is required " - "(unless using openai-codex or claude-code)" + "(unless using openai-codex, claude-code, or litellm)" ) base_url = os.getenv("HINDSIGHT_API_JUDGE_LLM_BASE_URL", os.getenv("HINDSIGHT_API_LLM_BASE_URL", "")) diff --git a/hindsight-api-slim/hindsight_api/engine/providers/__init__.py b/hindsight-api-slim/hindsight_api/engine/providers/__init__.py index 29a5effe..89ffd3f6 100644 --- a/hindsight-api-slim/hindsight_api/engine/providers/__init__.py +++ b/hindsight-api-slim/hindsight_api/engine/providers/__init__.py @@ -8,7 +8,8 @@ from .anthropic_llm import AnthropicLLM from .claude_code_llm import ClaudeCodeLLM from .codex_llm import CodexLLM from .gemini_llm import GeminiLLM +from .litellm_llm import LiteLLMLLM from .mock_llm import MockLLM from .openai_compatible_llm import OpenAICompatibleLLM -__all__ = ["AnthropicLLM", "ClaudeCodeLLM", "CodexLLM", "GeminiLLM", "MockLLM", "OpenAICompatibleLLM"] +__all__ = ["AnthropicLLM", "ClaudeCodeLLM", "CodexLLM", "GeminiLLM", "LiteLLMLLM", "MockLLM", "OpenAICompatibleLLM"] diff --git a/hindsight-api-slim/hindsight_api/engine/providers/litellm_llm.py b/hindsight-api-slim/hindsight_api/engine/providers/litellm_llm.py new file mode 100644 index 00000000..15175761 --- /dev/null +++ b/hindsight-api-slim/hindsight_api/engine/providers/litellm_llm.py @@ -0,0 +1,380 @@ +""" +LiteLLM LLM provider for universal model support. + +This provider enables using 100+ LLM providers via the LiteLLM SDK, including: +- AWS Bedrock (bedrock/anthropic.claude-3-5-sonnet-...) +- Azure OpenAI (azure/gpt-4o) +- Together AI (together_ai/meta-llama/...) +- Any other LiteLLM-supported provider + +Uses litellm.acompletion() for async chat completions. +Authentication for cloud providers (e.g., AWS Bedrock via boto3 credential chain) +is handled automatically by LiteLLM. +""" + +import asyncio +import json +import logging +import time +from typing import Any + +from hindsight_api.engine.llm_interface import LLMInterface, OutputTooLongError +from hindsight_api.engine.response_models import LLMToolCall, LLMToolCallResult, TokenUsage +from hindsight_api.metrics import get_metrics_collector + +logger = logging.getLogger(__name__) + + +class LiteLLMLLM(LLMInterface): + """ + LLM provider using the LiteLLM SDK for universal model support. + + Supports any model accessible via litellm.acompletion(), including AWS Bedrock, + Azure OpenAI, Together AI, Fireworks AI, and more. + + Model names follow LiteLLM conventions with provider prefixes: + - bedrock/anthropic.claude-3-5-sonnet-20241022-v2:0 + - azure/gpt-4o + - together_ai/meta-llama/Llama-3-70b-chat-hf + - fireworks_ai/accounts/fireworks/models/llama-v3p1-70b-instruct + """ + + def __init__( + self, + provider: str, + api_key: str, + base_url: str, + model: str, + reasoning_effort: str = "low", + timeout: float = 300.0, + **kwargs: Any, + ): + super().__init__(provider, api_key, base_url, model, reasoning_effort, **kwargs) + self.timeout = timeout + self._litellm: Any = None + + try: + import litellm + + self._litellm = litellm + # Suppress LiteLLM's verbose logging + litellm.suppress_debug_info = True # type: ignore[assignment] + # Drop unsupported params instead of raising errors (e.g. tool_choice on some Bedrock models) + litellm.drop_params = True # type: ignore[assignment] + logging.getLogger("LiteLLM").setLevel(logging.WARNING) + logger.info(f"LiteLLM SDK initialized for model: {self.model}") + except ImportError as e: + raise RuntimeError("LiteLLM SDK not installed. Run: uv add litellm or pip install litellm") from e + + async def verify_connection(self) -> None: + try: + test_messages = [{"role": "user", "content": "test"}] + await self.call( + messages=test_messages, + max_completion_tokens=50, + temperature=0.0, + scope="verification", + max_retries=0, + ) + logger.info("LiteLLM connection verified successfully") + except OutputTooLongError: + # Truncation is fine for verification — it means the connection works + logger.info("LiteLLM connection verified successfully (response truncated)") + except Exception as e: + logger.error(f"LiteLLM connection verification failed: {e}") + raise RuntimeError(f"Failed to verify LiteLLM connection: {e}") from e + + def _build_common_kwargs( + self, + messages: list[dict[str, Any]], + max_completion_tokens: int | None = None, + temperature: float | None = None, + ) -> dict[str, Any]: + """Build common kwargs for litellm calls.""" + kwargs: dict[str, Any] = { + "model": self.model, + "messages": messages, + "timeout": self.timeout, + } + + if self.api_key: + kwargs["api_key"] = self.api_key + if self.base_url: + kwargs["api_base"] = self.base_url + if max_completion_tokens is not None: + kwargs["max_completion_tokens"] = max_completion_tokens + if temperature is not None: + kwargs["temperature"] = temperature + + return kwargs + + async def call( + self, + messages: list[dict[str, str]], + response_format: Any | None = None, + max_completion_tokens: int | None = None, + temperature: float | None = None, + scope: str = "memory", + max_retries: int = 10, + initial_backoff: float = 1.0, + max_backoff: float = 60.0, + skip_validation: bool = False, + strict_schema: bool = False, + return_usage: bool = False, + ) -> Any: + start_time = time.time() + + call_kwargs = self._build_common_kwargs(messages, max_completion_tokens, temperature) + + # Add JSON schema response format if provided + if response_format is not None and hasattr(response_format, "model_json_schema"): + schema = response_format.model_json_schema() + call_kwargs["response_format"] = { + "type": "json_schema", + "json_schema": { + "name": response_format.__name__ if hasattr(response_format, "__name__") else "response", + "schema": schema, + "strict": strict_schema, + }, + } + + last_exception = None + + for attempt in range(max_retries + 1): + try: + response = await self._litellm.acompletion(**call_kwargs) + + content = response.choices[0].message.content or "" + finish_reason = response.choices[0].finish_reason + + # Check for length-limited output + if finish_reason == "length": + raise OutputTooLongError("LiteLLM response was truncated due to token limit") + + if response_format is not None: + # Strip markdown code fences if present + clean_content = content + if "```json" in content: + clean_content = content.split("```json")[1].split("```")[0].strip() + elif "```" in content: + clean_content = content.split("```")[1].split("```")[0].strip() + + try: + json_data = json.loads(clean_content) + except json.JSONDecodeError: + json_data = json.loads(content) + + if skip_validation: + result = json_data + else: + result = response_format.model_validate(json_data) + else: + result = content + + # Extract usage + input_tokens = getattr(response.usage, "prompt_tokens", 0) or 0 + output_tokens = getattr(response.usage, "completion_tokens", 0) or 0 + total_tokens = input_tokens + output_tokens + + # Record metrics + duration = time.time() - start_time + metrics = get_metrics_collector() + metrics.record_llm_call( + provider=self.provider, + model=self.model, + scope=scope, + duration=duration, + input_tokens=input_tokens, + output_tokens=output_tokens, + success=True, + ) + + # Record trace span + from hindsight_api.tracing import _serialize_for_span, get_span_recorder + + span_recorder = get_span_recorder() + span_recorder.record_llm_call( + provider=self.provider, + model=self.model, + scope=scope, + messages=messages, + response_content=_serialize_for_span(result), + input_tokens=input_tokens, + output_tokens=output_tokens, + duration=duration, + finish_reason=finish_reason, + error=None, + ) + + if duration > 10.0: + logger.info( + f"slow llm call: scope={scope}, model={self.provider}/{self.model}, " + f"input_tokens={input_tokens}, output_tokens={output_tokens}, " + f"time={duration:.3f}s" + ) + + if return_usage: + token_usage = TokenUsage( + input_tokens=input_tokens, + output_tokens=output_tokens, + total_tokens=total_tokens, + ) + return result, token_usage + return result + + except OutputTooLongError: + raise + + except json.JSONDecodeError as e: + last_exception = e + if attempt < max_retries: + logger.warning("LiteLLM returned invalid JSON, retrying...") + backoff = min(initial_backoff * (2**attempt), max_backoff) + await asyncio.sleep(backoff) + continue + else: + logger.error(f"LiteLLM returned invalid JSON after {max_retries + 1} attempts") + raise + + except Exception as e: + error_str = str(e).lower() + # Fast fail on auth errors + if "401" in error_str or "403" in error_str or "unauthorized" in error_str: + logger.error(f"LiteLLM auth error, not retrying: {e}") + raise + + last_exception = e + if attempt < max_retries: + # Retry on rate limits, connection errors, server errors + is_retryable = any( + keyword in error_str + for keyword in ("rate", "limit", "timeout", "connection", "500", "502", "503", "529") + ) + if is_retryable: + backoff = min(initial_backoff * (2**attempt), max_backoff) + jitter = backoff * 0.2 * (2 * (time.time() % 1) - 1) + await asyncio.sleep(backoff + jitter) + continue + + logger.error(f"LiteLLM API error after {attempt + 1} attempts: {e}") + raise + + if last_exception: + raise last_exception + raise RuntimeError("LiteLLM call failed after all retries") + + async def call_with_tools( + self, + messages: list[dict[str, Any]], + tools: list[dict[str, Any]], + max_completion_tokens: int | None = None, + temperature: float | None = None, + scope: str = "tools", + max_retries: int = 5, + initial_backoff: float = 1.0, + max_backoff: float = 30.0, + tool_choice: str | dict[str, Any] = "auto", + ) -> LLMToolCallResult: + start_time = time.time() + + call_kwargs = self._build_common_kwargs(messages, max_completion_tokens, temperature) + call_kwargs["tools"] = tools + call_kwargs["tool_choice"] = tool_choice + + last_exception = None + for attempt in range(max_retries + 1): + try: + response = await self._litellm.acompletion(**call_kwargs) + + message = response.choices[0].message + content = message.content + finish_reason = response.choices[0].finish_reason + + # Extract tool calls + tool_calls: list[LLMToolCall] = [] + if message.tool_calls: + for tc in message.tool_calls: + arguments = tc.function.arguments + if isinstance(arguments, str): + arguments = json.loads(arguments) + tool_calls.append( + LLMToolCall( + id=tc.id, + name=tc.function.name, + arguments=arguments, + ) + ) + + # Extract usage + input_tokens = getattr(response.usage, "prompt_tokens", 0) or 0 + output_tokens = getattr(response.usage, "completion_tokens", 0) or 0 + + # Record metrics + duration = time.time() - start_time + metrics = get_metrics_collector() + metrics.record_llm_call( + provider=self.provider, + model=self.model, + scope=scope, + duration=duration, + input_tokens=input_tokens, + output_tokens=output_tokens, + success=True, + ) + + # Record trace span + from hindsight_api.tracing import get_span_recorder + + span_recorder = get_span_recorder() + tool_calls_dict = ( + [{"id": tc.id, "name": tc.name, "arguments": tc.arguments} for tc in tool_calls] + if tool_calls + else None + ) + span_recorder.record_llm_call( + provider=self.provider, + model=self.model, + scope=scope, + messages=messages, + response_content=content, + input_tokens=input_tokens, + output_tokens=output_tokens, + duration=duration, + finish_reason=finish_reason, + error=None, + tool_calls=tool_calls_dict, + ) + + return LLMToolCallResult( + content=content, + tool_calls=tool_calls, + finish_reason=finish_reason or ("tool_calls" if tool_calls else "stop"), + input_tokens=input_tokens, + output_tokens=output_tokens, + ) + + except Exception as e: + error_str = str(e).lower() + if "401" in error_str or "403" in error_str or "unauthorized" in error_str: + raise + + last_exception = e + if attempt < max_retries: + is_retryable = any( + keyword in error_str + for keyword in ("rate", "limit", "timeout", "connection", "500", "502", "503", "529") + ) + if is_retryable: + await asyncio.sleep(min(initial_backoff * (2**attempt), max_backoff)) + continue + + logger.error(f"LiteLLM tool call error after {attempt + 1} attempts: {e}") + raise + + if last_exception: + raise last_exception + raise RuntimeError("LiteLLM tool call failed after all retries") + + async def cleanup(self) -> None: + """Clean up resources.""" + pass diff --git a/hindsight-api-slim/pyproject.toml b/hindsight-api-slim/pyproject.toml index e6a5e3d4..767e6d24 100644 --- a/hindsight-api-slim/pyproject.toml +++ b/hindsight-api-slim/pyproject.toml @@ -59,6 +59,7 @@ dependencies = [ "tornado>=6.5.5", # DoS multipart/incomplete cookie validation fix "aiohttp>=3.13.3", # Multiple DoS vulnerabilities "claude-agent-sdk>=0.1.27", + "boto3>=1.42.74", ] [project.optional-dependencies] diff --git a/hindsight-api-slim/tests/test_llm_provider.py b/hindsight-api-slim/tests/test_llm_provider.py index f8d0d8a6..616a2f66 100644 --- a/hindsight-api-slim/tests/test_llm_provider.py +++ b/hindsight-api-slim/tests/test_llm_provider.py @@ -43,6 +43,8 @@ MODEL_MATRIX = [ ("claude-code", "claude-sonnet-4-20250514"), # OpenAI Codex (uses MCP with Codex-specific models) ("openai-codex", "gpt-5.2-codex"), + # Bedrock models (via LiteLLM) + ("bedrock", "us.amazon.nova-2-lite-v1:0"), # Mock provider (for testing) ("mock", "mock"), ] @@ -78,6 +80,12 @@ def should_skip_provider(provider: str, model: str = "") -> tuple[bool, str]: if provider == "ollama" and "gemma" in model.lower(): return True, f"Ollama {model} does not support tool calling" + # Bedrock needs AWS credentials + if provider == "bedrock": + if not os.getenv("AWS_ACCESS_KEY_ID"): + return True, "No AWS credentials available (set AWS_ACCESS_KEY_ID)" + return False, "" + # Other providers need an API key if provider not in ("ollama", "claude-code", "openai-codex", "mock"): api_key = get_api_key_for_provider(provider) @@ -227,7 +235,7 @@ async def test_llm_provider_api_methods(provider: str, model: str): @pytest.mark.parametrize("provider,model", MODEL_MATRIX) @pytest.mark.asyncio -@pytest.mark.timeout(300) +@pytest.mark.timeout(600) # 600s: some providers (e.g., bedrock via litellm) need extra time for fact extraction async def test_llm_provider_memory_operations(provider: str, model: str): """ Test LLM provider with actual memory operations: fact extraction and reflect. diff --git a/hindsight-docs/docs/developer/configuration.md b/hindsight-docs/docs/developer/configuration.md index dda4289d..b2166696 100644 --- a/hindsight-docs/docs/developer/configuration.md +++ b/hindsight-docs/docs/developer/configuration.md @@ -160,7 +160,7 @@ To switch between backends: | Variable | Description | Default | |----------|-------------|---------| -| `HINDSIGHT_API_LLM_PROVIDER` | Provider: `openai`, `openai-codex`, `claude-code`, `anthropic`, `gemini`, `groq`, `minimax`, `ollama`, `lmstudio`, `vertexai` | `openai` | +| `HINDSIGHT_API_LLM_PROVIDER` | Provider: `openai`, `openai-codex`, `claude-code`, `anthropic`, `gemini`, `groq`, `minimax`, `ollama`, `lmstudio`, `vertexai`, `bedrock`, `litellm` | `openai` | | `HINDSIGHT_API_LLM_API_KEY` | API key for LLM provider | - | | `HINDSIGHT_API_LLM_MODEL` | Model name | `gpt-5-mini` | | `HINDSIGHT_API_LLM_BASE_URL` | Custom LLM endpoint | Provider default | @@ -232,6 +232,24 @@ export HINDSIGHT_API_LLM_MODEL=gpt-5.2-codex export HINDSIGHT_API_LLM_PROVIDER=claude-code export HINDSIGHT_API_LLM_MODEL=claude-sonnet-4-5-20250929 # No API key needed - uses claude auth login credentials + +# AWS Bedrock (native support - no API key needed, uses AWS credentials) +export HINDSIGHT_API_LLM_PROVIDER=bedrock +export HINDSIGHT_API_LLM_MODEL=us.amazon.nova-2-lite-v1:0 +export AWS_ACCESS_KEY_ID=your-access-key +export AWS_SECRET_ACCESS_KEY=your-secret-key +export AWS_REGION_NAME=us-east-1 + +# LiteLLM (100+ providers via LiteLLM SDK) +# Azure OpenAI via LiteLLM +export HINDSIGHT_API_LLM_PROVIDER=litellm +export HINDSIGHT_API_LLM_API_KEY=your-azure-api-key +export HINDSIGHT_API_LLM_MODEL=azure/gpt-4o + +# Together AI via LiteLLM +export HINDSIGHT_API_LLM_PROVIDER=litellm +export HINDSIGHT_API_LLM_API_KEY=your-together-api-key +export HINDSIGHT_API_LLM_MODEL=together_ai/meta-llama/Llama-3-70b-chat-hf ``` :::tip OpenAI Codex, Claude Code & Vertex AI Setup diff --git a/hindsight-docs/docs/developer/models.mdx b/hindsight-docs/docs/developer/models.mdx index 076f4520..d22d9355 100644 --- a/hindsight-docs/docs/developer/models.mdx +++ b/hindsight-docs/docs/developer/models.mdx @@ -22,7 +22,7 @@ Used for fact extraction, entity resolution, mental model consolidation, and ans -Also supports **any OpenAI-compatible API** (e.g., Azure OpenAI, Together AI, Fireworks). +Also supports **any OpenAI-compatible API** (e.g., Azure OpenAI, Together AI, Fireworks) and **100+ providers via LiteLLM** (e.g., AWS Bedrock, Azure OpenAI, Together AI). :::tip OpenAI-Compatible Providers Hindsight works with any provider that exposes an OpenAI-compatible API (e.g., Azure OpenAI). Simply set `HINDSIGHT_API_LLM_PROVIDER=openai` and configure `HINDSIGHT_API_LLM_BASE_URL` to point to your provider's endpoint. @@ -30,6 +30,18 @@ Hindsight works with any provider that exposes an OpenAI-compatible API (e.g., A See [Configuration](./configuration#llm-provider) for setup examples. ::: +:::tip AWS Bedrock +Set `HINDSIGHT_API_LLM_PROVIDER=bedrock` to use AWS Bedrock models directly. Model names use Bedrock model IDs (e.g., `us.amazon.nova-2-lite-v1:0`). No API key is required — authentication uses AWS credentials (`AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY`, `AWS_REGION_NAME`) or IAM roles. + +See [Configuration](./configuration#llm-provider) for setup examples. +::: + +:::tip LiteLLM Provider (Azure, Together AI, and more) +Set `HINDSIGHT_API_LLM_PROVIDER=litellm` to use any model supported by [LiteLLM](https://docs.litellm.ai/docs/providers), including **Azure OpenAI**, **Together AI**, **Fireworks AI**, and many more. Model names use LiteLLM's provider prefix format (e.g., `azure/gpt-4o`). + +See [Configuration](./configuration#llm-provider) for setup examples. +::: + ### Benchmarks Not sure which model to use? The **[Model Leaderboard](https://benchmarks.hindsight.vectorize.io/)** benchmarks models across accuracy, speed, cost, and reliability for retain, reflect, and observation consolidation so you can pick the right trade-off for your use case. @@ -73,6 +85,8 @@ Each provider has a recommended default model that's used when `HINDSIGHT_API_LL | `vertexai` | `gemini-2.0-flash-001` | | `openai-codex` | `gpt-5.2-codex` | | `claude-code` | `claude-sonnet-4-5-20250929` | +| `bedrock` | `us.amazon.nova-2-lite-v1:0` | +| `litellm` | `gpt-4o-mini` | **Example:** Setting just the provider uses its default model: ```bash diff --git a/hindsight-docs/docs/developer/monitoring.md b/hindsight-docs/docs/developer/monitoring.md index 83f448a7..d2383646 100644 --- a/hindsight-docs/docs/developer/monitoring.md +++ b/hindsight-docs/docs/developer/monitoring.md @@ -79,7 +79,7 @@ The `source` label allows distinguishing between: | `hindsight.llm.tokens.output` | Counter | provider, model, scope, success, token_bucket | Output tokens from LLM calls | **Labels:** -- `provider`: LLM provider (`openai`, `anthropic`, `gemini`, `groq`, `ollama`, `lmstudio`) +- `provider`: LLM provider (`openai`, `anthropic`, `gemini`, `groq`, `ollama`, `lmstudio`, `bedrock`, `litellm`) - `model`: Model name (e.g., `gpt-4`, `claude-3-sonnet`) - `scope`: What the LLM call is for (`memory`, `reflect`, `consolidation`, `answer`) - `success`: Whether the call succeeded (`true`, `false`) diff --git a/hindsight-docs/src/components/SupportedGrids.tsx b/hindsight-docs/src/components/SupportedGrids.tsx index 61d73aef..f7252a52 100644 --- a/hindsight-docs/src/components/SupportedGrids.tsx +++ b/hindsight-docs/src/components/SupportedGrids.tsx @@ -2,7 +2,7 @@ import React from 'react'; import type {IconType} from 'react-icons'; import {IconGrid} from './IconGrid'; import {SiPython, SiGo, SiOpenai, SiAnthropic, SiGooglegemini, SiOllama} from 'react-icons/si'; -import {LuTerminal, LuPlug, LuZap, LuBrainCog, LuSparkles, LuGlobe} from 'react-icons/lu'; +import {LuTerminal, LuPlug, LuZap, LuBrainCog, LuSparkles, LuGlobe, LuLayers, LuCloud} from 'react-icons/lu'; const OpenAICompatibleIcon: IconType = ({size = 28, ...props}) => ( @@ -39,6 +39,8 @@ export function LLMProvidersGrid() { { label: 'LM Studio', icon: LuBrainCog }, { label: 'MiniMax', icon: LuSparkles }, { label: 'OpenAI Compatible', icon: OpenAICompatibleIcon }, + { label: 'AWS Bedrock', icon: LuCloud }, + { label: 'LiteLLM (100+)', icon: LuLayers }, ]} /> ); } diff --git a/skills/hindsight-docs/references/developer/configuration.md b/skills/hindsight-docs/references/developer/configuration.md index 80129316..01429392 100644 --- a/skills/hindsight-docs/references/developer/configuration.md +++ b/skills/hindsight-docs/references/developer/configuration.md @@ -160,7 +160,7 @@ To switch between backends: | Variable | Description | Default | |----------|-------------|---------| -| `HINDSIGHT_API_LLM_PROVIDER` | Provider: `openai`, `openai-codex`, `claude-code`, `anthropic`, `gemini`, `groq`, `minimax`, `ollama`, `lmstudio`, `vertexai` | `openai` | +| `HINDSIGHT_API_LLM_PROVIDER` | Provider: `openai`, `openai-codex`, `claude-code`, `anthropic`, `gemini`, `groq`, `minimax`, `ollama`, `lmstudio`, `vertexai`, `bedrock`, `litellm` | `openai` | | `HINDSIGHT_API_LLM_API_KEY` | API key for LLM provider | - | | `HINDSIGHT_API_LLM_MODEL` | Model name | `gpt-5-mini` | | `HINDSIGHT_API_LLM_BASE_URL` | Custom LLM endpoint | Provider default | @@ -232,6 +232,24 @@ export HINDSIGHT_API_LLM_MODEL=gpt-5.2-codex export HINDSIGHT_API_LLM_PROVIDER=claude-code export HINDSIGHT_API_LLM_MODEL=claude-sonnet-4-5-20250929 # No API key needed - uses claude auth login credentials + +# AWS Bedrock (native support - no API key needed, uses AWS credentials) +export HINDSIGHT_API_LLM_PROVIDER=bedrock +export HINDSIGHT_API_LLM_MODEL=us.amazon.nova-2-lite-v1:0 +export AWS_ACCESS_KEY_ID=your-access-key +export AWS_SECRET_ACCESS_KEY=your-secret-key +export AWS_REGION_NAME=us-east-1 + +# LiteLLM (100+ providers via LiteLLM SDK) +# Azure OpenAI via LiteLLM +export HINDSIGHT_API_LLM_PROVIDER=litellm +export HINDSIGHT_API_LLM_API_KEY=your-azure-api-key +export HINDSIGHT_API_LLM_MODEL=azure/gpt-4o + +# Together AI via LiteLLM +export HINDSIGHT_API_LLM_PROVIDER=litellm +export HINDSIGHT_API_LLM_API_KEY=your-together-api-key +export HINDSIGHT_API_LLM_MODEL=together_ai/meta-llama/Llama-3-70b-chat-hf ``` :::tip OpenAI Codex, Claude Code & Vertex AI Setup diff --git a/skills/hindsight-docs/references/developer/models.md b/skills/hindsight-docs/references/developer/models.md index 98e2d01d..33451800 100644 --- a/skills/hindsight-docs/references/developer/models.md +++ b/skills/hindsight-docs/references/developer/models.md @@ -21,12 +21,22 @@ Used for fact extraction, entity resolution, mental model consolidation, and ans -Also supports **any OpenAI-compatible API** (e.g., Azure OpenAI, Together AI, Fireworks). +Also supports **any OpenAI-compatible API** (e.g., Azure OpenAI, Together AI, Fireworks) and **100+ providers via LiteLLM** (e.g., AWS Bedrock, Azure OpenAI, Together AI). > **💡 OpenAI-Compatible Providers** > Hindsight works with any provider that exposes an OpenAI-compatible API (e.g., Azure OpenAI). Simply set `HINDSIGHT_API_LLM_PROVIDER=openai` and configure `HINDSIGHT_API_LLM_BASE_URL` to point to your provider's endpoint. +See [Configuration](./configuration#llm-provider) for setup examples. +> **💡 AWS Bedrock** +> +Set `HINDSIGHT_API_LLM_PROVIDER=bedrock` to use AWS Bedrock models directly. Model names use Bedrock model IDs (e.g., `us.amazon.nova-2-lite-v1:0`). No API key is required — authentication uses AWS credentials (`AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY`, `AWS_REGION_NAME`) or IAM roles. + +See [Configuration](./configuration#llm-provider) for setup examples. +> **💡 LiteLLM Provider (Azure, Together AI, and more)** +> +Set `HINDSIGHT_API_LLM_PROVIDER=litellm` to use any model supported by [LiteLLM](https://docs.litellm.ai/docs/providers), including **Azure OpenAI**, **Together AI**, **Fireworks AI**, and many more. Model names use LiteLLM's provider prefix format (e.g., `azure/gpt-4o`). + See [Configuration](./configuration#llm-provider) for setup examples. ### Benchmarks @@ -71,6 +81,8 @@ Each provider has a recommended default model that's used when `HINDSIGHT_API_LL | `vertexai` | `gemini-2.0-flash-001` | | `openai-codex` | `gpt-5.2-codex` | | `claude-code` | `claude-sonnet-4-5-20250929` | +| `bedrock` | `us.amazon.nova-2-lite-v1:0` | +| `litellm` | `gpt-4o-mini` | **Example:** Setting just the provider uses its default model: ```bash diff --git a/skills/hindsight-docs/references/developer/monitoring.md b/skills/hindsight-docs/references/developer/monitoring.md index 83f448a7..d2383646 100644 --- a/skills/hindsight-docs/references/developer/monitoring.md +++ b/skills/hindsight-docs/references/developer/monitoring.md @@ -79,7 +79,7 @@ The `source` label allows distinguishing between: | `hindsight.llm.tokens.output` | Counter | provider, model, scope, success, token_bucket | Output tokens from LLM calls | **Labels:** -- `provider`: LLM provider (`openai`, `anthropic`, `gemini`, `groq`, `ollama`, `lmstudio`) +- `provider`: LLM provider (`openai`, `anthropic`, `gemini`, `groq`, `ollama`, `lmstudio`, `bedrock`, `litellm`) - `model`: Model name (e.g., `gpt-4`, `claude-3-sonnet`) - `scope`: What the LLM call is for (`memory`, `reflect`, `consolidation`, `answer`) - `success`: Whether the call succeeded (`true`, `false`) diff --git a/uv.lock b/uv.lock index f8a14448..a8e72731 100644 --- a/uv.lock +++ b/uv.lock @@ -441,6 +441,33 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/10/cb/f2ad4230dc2eb1a74edf38f1a38b9b52277f75bef262d8908e60d957e13c/blinker-1.9.0-py3-none-any.whl", hash = "sha256:ba0efaa9080b619ff2f3459d1d500c57bddea4a6b424b60a91141db6fd2f08bc", size = 8458 }, ] +[[package]] +name = "boto3" +version = "1.42.74" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "botocore" }, + { name = "jmespath" }, + { name = "s3transfer" }, +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/ad/16/a264b4da2af99f4a12609b93fea941cce5ec41da14b33ed3fef77a910f0c/boto3-1.42.74-py3-none-any.whl", hash = "sha256:4bf89c044d618fe4435af854ab820f09dd43569c0df15d7beb0398f50b9aa970", size = 140557 }, +] + +[[package]] +name = "botocore" +version = "1.42.74" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "jmespath" }, + { name = "python-dateutil" }, + { name = "urllib3" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/9d/c7/cab8a14f0b69944bd0dd1fd58559163455b347eeda00bf836e93ce2684e4/botocore-1.42.74.tar.gz", hash = "sha256:9cf5cdffc6c90ed87b0fe184676806182588be0d0df9b363e9fe3e2923ac8e80", size = 15014379 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d3/65/75852e04de5423c9b0c5b88241d0bdea33e6c6f454c88b71377d230216f2/botocore-1.42.74-py3-none-any.whl", hash = "sha256:3a76a8af08b5de82e51a0ae132394e226e15dbf21c8146ac3f7c1f881517a7a7", size = 14688218 }, +] + [[package]] name = "cachetools" version = "6.2.1" @@ -1508,6 +1535,7 @@ dependencies = [ { name = "anthropic" }, { name = "asyncpg" }, { name = "authlib" }, + { name = "boto3" }, { name = "claude-agent-sdk" }, { name = "cohere" }, { name = "cryptography" }, @@ -1609,6 +1637,7 @@ requires-dist = [ { name = "anthropic", specifier = ">=0.40.0" }, { name = "asyncpg", specifier = ">=0.29.0" }, { name = "authlib", specifier = ">=1.6.9" }, + { name = "boto3", specifier = ">=1.42.74" }, { name = "claude-agent-sdk", specifier = ">=0.1.27" }, { name = "cohere", specifier = ">=5.0.0" }, { name = "cryptography", specifier = ">=46.0.5" }, @@ -2082,6 +2111,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/2f/9c/6753e6522b8d0ef07d3a3d239426669e984fb0eba15a315cdbc1253904e4/jiter-0.12.0-graalpy312-graalpy250_312_native-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c24e864cb30ab82311c6425655b0cdab0a98c5d973b065c66a3f020740c2324c", size = 346110 }, ] +[[package]] +name = "jmespath" +version = "1.1.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d3/59/322338183ecda247fb5d1763a6cbe46eff7222eaeebafd9fa65d4bf5cb11/jmespath-1.1.0.tar.gz", hash = "sha256:472c87d80f36026ae83c6ddd0f1d05d4e510134ed462851fd5f754c8c3cbb88d", size = 27377 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/14/2f/967ba146e6d58cf6a652da73885f52fc68001525b4197effc174321d70b4/jmespath-1.1.0-py3-none-any.whl", hash = "sha256:a5663118de4908c91729bea0acadca56526eb2698e83de10cd116ae0f4e97c64", size = 20419 }, +] + [[package]] name = "joblib" version = "1.5.2" @@ -4685,6 +4723,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/26/09/7a9520315decd2334afa65ed258fed438f070e31f05a2e43dd480a5e5911/ruff-0.14.9-py3-none-win_arm64.whl", hash = "sha256:8e821c366517a074046d92f0e9213ed1c13dbc5b37a7fc20b07f79b64d62cc84", size = 13744730 }, ] +[[package]] +name = "s3transfer" +version = "0.16.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "botocore" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/05/04/74127fc843314818edfa81b5540e26dd537353b123a4edc563109d8f17dd/s3transfer-0.16.0.tar.gz", hash = "sha256:8e990f13268025792229cd52fa10cb7163744bf56e719e0b9cb925ab79abf920", size = 153827 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/fc/51/727abb13f44c1fcf6d145979e1535a35794db0f6e450a0cb46aa24732fe2/s3transfer-0.16.0-py3-none-any.whl", hash = "sha256:18e25d66fed509e3868dc1572b3f427ff947dd2c56f844a5bf09481ad3f3b2fe", size = 86830 }, +] + [[package]] name = "safetensors" version = "0.6.2"