holetron-lab/fleet-memory
1
0
Fork 0
Code Releases Packages Activity Actions

feat: add LiteLLM LLM provider for Bedrock and 100+ providers (#679)

Browse source 
* feat: add LiteLLM LLM provider for Bedrock and 100+ providers

Add a new `litellm` LLM provider that uses the LiteLLM SDK for chat
completions and tool calling, enabling AWS Bedrock and 100+ other
providers for Hindsight's core engine (retain, recall, reflect).

- New LiteLLMLLM provider in engine/providers/litellm_llm.py
- Registered in factory, valid providers list, and no-api-key set
- Refactored API key validation to use requires_api_key() helper
- Added boto3 dependency for Bedrock auth
- Updated docs: configuration, models, monitoring, providers grid

* feat: add bedrock as first-class LLM provider alias

Add `bedrock` as a dedicated provider name that auto-prepends the
`bedrock/` prefix to model names and delegates to LiteLLMLLM under
the hood. This makes Bedrock support more discoverable — users set
`HINDSIGHT_API_LLM_PROVIDER=bedrock` with plain Bedrock model IDs.

* test: add Bedrock to CI provider tests

- Add bedrock/us.amazon.nova-lite-v1:0 to MODEL_MATRIX in test_llm_provider.py
- Add AWS credential check in should_skip_provider()
- Pass AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, AWS_REGION_NAME secrets to test-api job
- Update default bedrock model to amazon.nova-2-lite-v1:0

* fix: regenerate docs skill files and bump memory test timeout

- Regenerate skills/hindsight-docs references after docs changes
- Bump test_llm_provider_memory_operations timeout to 600s for slower
  providers like Bedrock via LiteLLM

* test: skip bedrock lite models in memory operations test

Nova Lite has a 10K output token limit which is too low for fact
extraction (requires 64K). The api_methods test (completion, tools,
structured output) already validates the provider works correctly.

* test: use Nova Pro for bedrock CI tests to cover full memory pipeline

Nova Lite only supports 10K output tokens, too low for fact extraction.
Switch to Nova Pro which supports the full 64K output needed for
retain/reflect operations. This ensures bedrock is tested on all
Hindsight functionalities, not just basic API methods.

* test: switch bedrock CI to Nova 2 Lite (supports 64K output tokens)

Nova v1 models (Pro, Lite) have a 10K output token limit which is
too low for fact extraction. Nova 2 Lite supports 64K+ output tokens,
enabling full memory pipeline testing (retain + reflect).
This commit is contained in:
Nicolò Boschi 2026-03-25 14:17:38 +01:00 committed by GitHub
parent c5273f5fd4
commit db70fdbe5e
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
15 changed files with 559 additions and 20 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
.github/workflows/test.yml vendored
View file

@ -461,6 +461,9 @@ jobs:
OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
COHERE_API_KEY: ${{ secrets.COHERE_API_KEY }}
HINDSIGHT_API_EMBEDDINGS_OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
AWS_REGION_NAME: ${{ secrets.AWS_REGION_NAME }}
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
# Prefer CPU-only PyTorch in CI (but keep PyPI for everything else)

2
hindsight-api-slim/hindsight_api/config.py
View file

@ -364,6 +364,8 @@ PROVIDER_DEFAULT_MODELS = {
"openai-codex": "gpt-5.2-codex",
"claude-code": "claude-sonnet-4-5-20250929",
"mock": "mock-model",
"litellm": "gpt-4o-mini",
"bedrock": "us.amazon.nova-2-lite-v1:0",
}
DEFAULT_LLM_MODEL = "gpt-4o-mini" # Fallback if provider not in table
DEFAULT_LLM_MAX_CONCURRENT = 32

52
hindsight-api-slim/hindsight_api/engine/llm_wrapper.py
View file

@ -126,6 +126,8 @@ _PROVIDERS_WITHOUT_API_KEY = frozenset(
"claude-code",
"mock",
"vertexai",
"litellm",
"bedrock",
}
)
@ -172,6 +174,7 @@ def create_llm_provider(
ClaudeCodeLLM,
CodexLLM,
GeminiLLM,
LiteLLMLLM,
MockLLM,
OpenAICompatibleLLM,
)
@ -227,6 +230,26 @@ def create_llm_provider(
reasoning_effort=reasoning_effort,
)
elif provider_lower == "litellm":
return LiteLLMLLM(
provider=provider,
api_key=api_key,
base_url=base_url,
model=model,
reasoning_effort=reasoning_effort,
)
elif provider_lower == "bedrock":
# Bedrock is a first-class alias backed by LiteLLM with auto-prefixed model names
bedrock_model = model if model.startswith("bedrock/") else f"bedrock/{model}"
return LiteLLMLLM(
provider=provider,
api_key=api_key,
base_url=base_url,
model=bedrock_model,
reasoning_effort=reasoning_effort,
)
elif provider_lower in ("openai", "groq", "ollama", "lmstudio", "minimax"):
return OpenAICompatibleLLM(
provider=provider,
@ -297,6 +320,8 @@ class LLMProvider:
"claude-code",
"mock",
"minimax",
"litellm",
"bedrock",
]
if self.provider not in valid_providers:
raise ValueError(f"Invalid LLM provider: {self.provider}. Must be one of: {', '.join(valid_providers)}")
@ -675,10 +700,13 @@ class LLMProvider:
api_key = os.getenv("HINDSIGHT_API_LLM_API_KEY", "")
# API key not needed for openai-codex (uses OAuth), claude-code (uses Keychain OAuth),
# ollama (local), or vertexai (uses GCP service account credentials)
if not api_key and provider not in ("openai-codex", "claude-code", "ollama", "vertexai"):
# ollama (local), vertexai (uses GCP service account credentials),
# or litellm (uses provider-specific auth, e.g. AWS credentials for Bedrock)
if not api_key and not requires_api_key(provider):
pass # Provider handles its own auth
elif not api_key:
raise ValueError(
"HINDSIGHT_API_LLM_API_KEY environment variable is required (unless using openai-codex or claude-code)"
"HINDSIGHT_API_LLM_API_KEY environment variable is required (unless using openai-codex, claude-code, or litellm)"
)
base_url = os.getenv("HINDSIGHT_API_LLM_BASE_URL", "")
@ -692,12 +720,13 @@ class LLMProvider:
provider = os.getenv("HINDSIGHT_API_ANSWER_LLM_PROVIDER", os.getenv("HINDSIGHT_API_LLM_PROVIDER", "groq"))
api_key = os.getenv("HINDSIGHT_API_ANSWER_LLM_API_KEY", os.getenv("HINDSIGHT_API_LLM_API_KEY", ""))
# API key not needed for openai-codex (uses OAuth), claude-code (uses Keychain OAuth),
# ollama (local), or vertexai (uses GCP service account credentials)
if not api_key and provider not in ("openai-codex", "claude-code", "ollama", "vertexai"):
# API key not needed for providers with their own auth mechanisms
if not api_key and not requires_api_key(provider):
pass # Provider handles its own auth
elif not api_key:
raise ValueError(
"HINDSIGHT_API_LLM_API_KEY or HINDSIGHT_API_ANSWER_LLM_API_KEY environment variable is required "
"(unless using openai-codex or claude-code)"
"(unless using openai-codex, claude-code, or litellm)"
)
base_url = os.getenv("HINDSIGHT_API_ANSWER_LLM_BASE_URL", os.getenv("HINDSIGHT_API_LLM_BASE_URL", ""))
@ -711,12 +740,13 @@ class LLMProvider:
provider = os.getenv("HINDSIGHT_API_JUDGE_LLM_PROVIDER", os.getenv("HINDSIGHT_API_LLM_PROVIDER", "groq"))
api_key = os.getenv("HINDSIGHT_API_JUDGE_LLM_API_KEY", os.getenv("HINDSIGHT_API_LLM_API_KEY", ""))
# API key not needed for openai-codex (uses OAuth), claude-code (uses Keychain OAuth),
# ollama (local), or vertexai (uses GCP service account credentials)
if not api_key and provider not in ("openai-codex", "claude-code", "ollama", "vertexai"):
# API key not needed for providers with their own auth mechanisms
if not api_key and not requires_api_key(provider):
pass # Provider handles its own auth
elif not api_key:
raise ValueError(
"HINDSIGHT_API_LLM_API_KEY or HINDSIGHT_API_JUDGE_LLM_API_KEY environment variable is required "
"(unless using openai-codex or claude-code)"
"(unless using openai-codex, claude-code, or litellm)"
)
base_url = os.getenv("HINDSIGHT_API_JUDGE_LLM_BASE_URL", os.getenv("HINDSIGHT_API_LLM_BASE_URL", ""))

3
hindsight-api-slim/hindsight_api/engine/providers/__init__.py
View file

@ -8,7 +8,8 @@ from .anthropic_llm import AnthropicLLM
from .claude_code_llm import ClaudeCodeLLM
from .codex_llm import CodexLLM
from .gemini_llm import GeminiLLM
from .litellm_llm import LiteLLMLLM
from .mock_llm import MockLLM
from .openai_compatible_llm import OpenAICompatibleLLM
__all__ = ["AnthropicLLM", "ClaudeCodeLLM", "CodexLLM", "GeminiLLM", "MockLLM", "OpenAICompatibleLLM"]
__all__ = ["AnthropicLLM", "ClaudeCodeLLM", "CodexLLM", "GeminiLLM", "LiteLLMLLM", "MockLLM", "OpenAICompatibleLLM"]

380
hindsight-api-slim/hindsight_api/engine/providers/litellm_llm.py Normal file
View file

@ -0,0 +1,380 @@
"""
LiteLLM LLM provider for universal model support.
This provider enables using 100+ LLM providers via the LiteLLM SDK, including:
- AWS Bedrock (bedrock/anthropic.claude-3-5-sonnet-...)
- Azure OpenAI (azure/gpt-4o)
- Together AI (together_ai/meta-llama/...)
- Any other LiteLLM-supported provider
Uses litellm.acompletion() for async chat completions.
Authentication for cloud providers (e.g., AWS Bedrock via boto3 credential chain)
is handled automatically by LiteLLM.
"""
import asyncio
import json
import logging
import time
from typing import Any
from hindsight_api.engine.llm_interface import LLMInterface, OutputTooLongError
from hindsight_api.engine.response_models import LLMToolCall, LLMToolCallResult, TokenUsage
from hindsight_api.metrics import get_metrics_collector
logger = logging.getLogger(__name__)
class LiteLLMLLM(LLMInterface):
"""
LLM provider using the LiteLLM SDK for universal model support.
Supports any model accessible via litellm.acompletion(), including AWS Bedrock,
Azure OpenAI, Together AI, Fireworks AI, and more.
Model names follow LiteLLM conventions with provider prefixes:
- bedrock/anthropic.claude-3-5-sonnet-20241022-v2:0
- azure/gpt-4o
- together_ai/meta-llama/Llama-3-70b-chat-hf
- fireworks_ai/accounts/fireworks/models/llama-v3p1-70b-instruct
"""
def __init__(
self,
provider: str,
api_key: str,
base_url: str,
model: str,
reasoning_effort: str = "low",
timeout: float = 300.0,
**kwargs: Any,
):
super().__init__(provider, api_key, base_url, model, reasoning_effort, **kwargs)
self.timeout = timeout
self._litellm: Any = None
try:
import litellm
self._litellm = litellm
# Suppress LiteLLM's verbose logging
litellm.suppress_debug_info = True # type: ignore[assignment]
# Drop unsupported params instead of raising errors (e.g. tool_choice on some Bedrock models)
litellm.drop_params = True # type: ignore[assignment]
logging.getLogger("LiteLLM").setLevel(logging.WARNING)
logger.info(f"LiteLLM SDK initialized for model: {self.model}")
except ImportError as e:
raise RuntimeError("LiteLLM SDK not installed. Run: uv add litellm or pip install litellm") from e
async def verify_connection(self) -> None:
try:
test_messages = [{"role": "user", "content": "test"}]
await self.call(
messages=test_messages,
max_completion_tokens=50,
temperature=0.0,
scope="verification",
max_retries=0,
)
logger.info("LiteLLM connection verified successfully")
except OutputTooLongError:
# Truncation is fine for verification — it means the connection works
logger.info("LiteLLM connection verified successfully (response truncated)")
except Exception as e:
logger.error(f"LiteLLM connection verification failed: {e}")
raise RuntimeError(f"Failed to verify LiteLLM connection: {e}") from e
def _build_common_kwargs(
self,
messages: list[dict[str, Any]],
max_completion_tokens: int | None = None,
temperature: float | None = None,
) -> dict[str, Any]:
"""Build common kwargs for litellm calls."""
kwargs: dict[str, Any] = {
"model": self.model,
"messages": messages,
"timeout": self.timeout,
}
if self.api_key:
kwargs["api_key"] = self.api_key
if self.base_url:
kwargs["api_base"] = self.base_url
if max_completion_tokens is not None:
kwargs["max_completion_tokens"] = max_completion_tokens
if temperature is not None:
kwargs["temperature"] = temperature
return kwargs
async def call(
self,
messages: list[dict[str, str]],
response_format: Any | None = None,
max_completion_tokens: int | None = None,
temperature: float | None = None,
scope: str = "memory",
max_retries: int = 10,
initial_backoff: float = 1.0,
max_backoff: float = 60.0,
skip_validation: bool = False,
strict_schema: bool = False,
return_usage: bool = False,
) -> Any:
start_time = time.time()
call_kwargs = self._build_common_kwargs(messages, max_completion_tokens, temperature)
# Add JSON schema response format if provided
if response_format is not None and hasattr(response_format, "model_json_schema"):
schema = response_format.model_json_schema()
call_kwargs["response_format"] = {
"type": "json_schema",
"json_schema": {
"name": response_format.__name__ if hasattr(response_format, "__name__") else "response",
"schema": schema,
"strict": strict_schema,
},
}
last_exception = None
for attempt in range(max_retries + 1):
try:
response = await self._litellm.acompletion(**call_kwargs)
content = response.choices[0].message.content or ""
finish_reason = response.choices[0].finish_reason
# Check for length-limited output
if finish_reason == "length":
raise OutputTooLongError("LiteLLM response was truncated due to token limit")
if response_format is not None:
# Strip markdown code fences if present
clean_content = content
if "```json" in content:
clean_content = content.split("```json")[1].split("```")[0].strip()
elif "```" in content:
clean_content = content.split("```")[1].split("```")[0].strip()
try:
json_data = json.loads(clean_content)
except json.JSONDecodeError:
json_data = json.loads(content)
if skip_validation:
result = json_data
else:
result = response_format.model_validate(json_data)
else:
result = content
# Extract usage
input_tokens = getattr(response.usage, "prompt_tokens", 0) or 0
output_tokens = getattr(response.usage, "completion_tokens", 0) or 0
total_tokens = input_tokens + output_tokens
# Record metrics
duration = time.time() - start_time
metrics = get_metrics_collector()
metrics.record_llm_call(
provider=self.provider,
model=self.model,
scope=scope,
duration=duration,
input_tokens=input_tokens,
output_tokens=output_tokens,
success=True,
)
# Record trace span
from hindsight_api.tracing import _serialize_for_span, get_span_recorder
span_recorder = get_span_recorder()
span_recorder.record_llm_call(
provider=self.provider,
model=self.model,
scope=scope,
messages=messages,
response_content=_serialize_for_span(result),
input_tokens=input_tokens,
output_tokens=output_tokens,
duration=duration,
finish_reason=finish_reason,
error=None,
)
if duration > 10.0:
logger.info(
f"slow llm call: scope={scope}, model={self.provider}/{self.model}, "
f"input_tokens={input_tokens}, output_tokens={output_tokens}, "
f"time={duration:.3f}s"
)
if return_usage:
token_usage = TokenUsage(
input_tokens=input_tokens,
output_tokens=output_tokens,
total_tokens=total_tokens,
)
return result, token_usage
return result
except OutputTooLongError:
raise
except json.JSONDecodeError as e:
last_exception = e
if attempt < max_retries:
logger.warning("LiteLLM returned invalid JSON, retrying...")
backoff = min(initial_backoff * (2**attempt), max_backoff)
await asyncio.sleep(backoff)
continue
else:
logger.error(f"LiteLLM returned invalid JSON after {max_retries + 1} attempts")
raise
except Exception as e:
error_str = str(e).lower()
# Fast fail on auth errors
if "401" in error_str or "403" in error_str or "unauthorized" in error_str:
logger.error(f"LiteLLM auth error, not retrying: {e}")
raise
last_exception = e
if attempt < max_retries:
# Retry on rate limits, connection errors, server errors
is_retryable = any(
keyword in error_str
for keyword in ("rate", "limit", "timeout", "connection", "500", "502", "503", "529")
)
if is_retryable:
backoff = min(initial_backoff * (2**attempt), max_backoff)
jitter = backoff * 0.2 * (2 * (time.time() % 1) - 1)
await asyncio.sleep(backoff + jitter)
continue
logger.error(f"LiteLLM API error after {attempt + 1} attempts: {e}")
raise
if last_exception:
raise last_exception
raise RuntimeError("LiteLLM call failed after all retries")
async def call_with_tools(
self,
messages: list[dict[str, Any]],
tools: list[dict[str, Any]],
max_completion_tokens: int | None = None,
temperature: float | None = None,
scope: str = "tools",
max_retries: int = 5,
initial_backoff: float = 1.0,
max_backoff: float = 30.0,
tool_choice: str | dict[str, Any] = "auto",
) -> LLMToolCallResult:
start_time = time.time()
call_kwargs = self._build_common_kwargs(messages, max_completion_tokens, temperature)
call_kwargs["tools"] = tools
call_kwargs["tool_choice"] = tool_choice
last_exception = None
for attempt in range(max_retries + 1):
try:
response = await self._litellm.acompletion(**call_kwargs)
message = response.choices[0].message
content = message.content
finish_reason = response.choices[0].finish_reason
# Extract tool calls
tool_calls: list[LLMToolCall] = []
if message.tool_calls:
for tc in message.tool_calls:
arguments = tc.function.arguments
if isinstance(arguments, str):
arguments = json.loads(arguments)
tool_calls.append(
LLMToolCall(
id=tc.id,
name=tc.function.name,
arguments=arguments,
)
)
# Extract usage
input_tokens = getattr(response.usage, "prompt_tokens", 0) or 0
output_tokens = getattr(response.usage, "completion_tokens", 0) or 0
# Record metrics
duration = time.time() - start_time
metrics = get_metrics_collector()
metrics.record_llm_call(
provider=self.provider,
model=self.model,
scope=scope,
duration=duration,
input_tokens=input_tokens,
output_tokens=output_tokens,
success=True,
)
# Record trace span
from hindsight_api.tracing import get_span_recorder
span_recorder = get_span_recorder()
tool_calls_dict = (
[{"id": tc.id, "name": tc.name, "arguments": tc.arguments} for tc in tool_calls]
if tool_calls
else None
)
span_recorder.record_llm_call(
provider=self.provider,
model=self.model,
scope=scope,
messages=messages,
response_content=content,
input_tokens=input_tokens,
output_tokens=output_tokens,
duration=duration,
finish_reason=finish_reason,
error=None,
tool_calls=tool_calls_dict,
)
return LLMToolCallResult(
content=content,
tool_calls=tool_calls,
finish_reason=finish_reason or ("tool_calls" if tool_calls else "stop"),
input_tokens=input_tokens,
output_tokens=output_tokens,
)
except Exception as e:
error_str = str(e).lower()
if "401" in error_str or "403" in error_str or "unauthorized" in error_str:
raise
last_exception = e
if attempt < max_retries:
is_retryable = any(
keyword in error_str
for keyword in ("rate", "limit", "timeout", "connection", "500", "502", "503", "529")
)
if is_retryable:
await asyncio.sleep(min(initial_backoff * (2**attempt), max_backoff))
continue
logger.error(f"LiteLLM tool call error after {attempt + 1} attempts: {e}")
raise
if last_exception:
raise last_exception
raise RuntimeError("LiteLLM tool call failed after all retries")
async def cleanup(self) -> None:
"""Clean up resources."""
pass

1
hindsight-api-slim/pyproject.toml
View file

@ -59,6 +59,7 @@ dependencies = [
"tornado>=6.5.5", # DoS multipart/incomplete cookie validation fix
"aiohttp>=3.13.3", # Multiple DoS vulnerabilities
"claude-agent-sdk>=0.1.27",
"boto3>=1.42.74",
]
[project.optional-dependencies]

10
hindsight-api-slim/tests/test_llm_provider.py
View file

@ -43,6 +43,8 @@ MODEL_MATRIX = [
("claude-code", "claude-sonnet-4-20250514"),
# OpenAI Codex (uses MCP with Codex-specific models)
("openai-codex", "gpt-5.2-codex"),
# Bedrock models (via LiteLLM)
("bedrock", "us.amazon.nova-2-lite-v1:0"),
# Mock provider (for testing)
("mock", "mock"),
]
@ -78,6 +80,12 @@ def should_skip_provider(provider: str, model: str = "") -> tuple[bool, str]:
if provider == "ollama" and "gemma" in model.lower():
return True, f"Ollama {model} does not support tool calling"
# Bedrock needs AWS credentials
if provider == "bedrock":
if not os.getenv("AWS_ACCESS_KEY_ID"):
return True, "No AWS credentials available (set AWS_ACCESS_KEY_ID)"
return False, ""
# Other providers need an API key
if provider not in ("ollama", "claude-code", "openai-codex", "mock"):
api_key = get_api_key_for_provider(provider)
@ -227,7 +235,7 @@ async def test_llm_provider_api_methods(provider: str, model: str):
@pytest.mark.parametrize("provider,model", MODEL_MATRIX)
@pytest.mark.asyncio
@pytest.mark.timeout(300)
@pytest.mark.timeout(600) # 600s: some providers (e.g., bedrock via litellm) need extra time for fact extraction
async def test_llm_provider_memory_operations(provider: str, model: str):
"""
Test LLM provider with actual memory operations: fact extraction and reflect.

20
hindsight-docs/docs/developer/configuration.md
View file

@ -160,7 +160,7 @@ To switch between backends:
| Variable | Description | Default |
|----------|-------------|---------|
| `HINDSIGHT_API_LLM_PROVIDER` | Provider: `openai`, `openai-codex`, `claude-code`, `anthropic`, `gemini`, `groq`, `minimax`, `ollama`, `lmstudio`, `vertexai` | `openai` |
| `HINDSIGHT_API_LLM_PROVIDER` | Provider: `openai`, `openai-codex`, `claude-code`, `anthropic`, `gemini`, `groq`, `minimax`, `ollama`, `lmstudio`, `vertexai`, `bedrock`, `litellm` | `openai` |
| `HINDSIGHT_API_LLM_API_KEY` | API key for LLM provider | - |
| `HINDSIGHT_API_LLM_MODEL` | Model name | `gpt-5-mini` |
| `HINDSIGHT_API_LLM_BASE_URL` | Custom LLM endpoint | Provider default |
@ -232,6 +232,24 @@ export HINDSIGHT_API_LLM_MODEL=gpt-5.2-codex
export HINDSIGHT_API_LLM_PROVIDER=claude-code
export HINDSIGHT_API_LLM_MODEL=claude-sonnet-4-5-20250929
# No API key needed - uses claude auth login credentials
# AWS Bedrock (native support - no API key needed, uses AWS credentials)
export HINDSIGHT_API_LLM_PROVIDER=bedrock
export HINDSIGHT_API_LLM_MODEL=us.amazon.nova-2-lite-v1:0
export AWS_ACCESS_KEY_ID=your-access-key
export AWS_SECRET_ACCESS_KEY=your-secret-key
export AWS_REGION_NAME=us-east-1
# LiteLLM (100+ providers via LiteLLM SDK)
# Azure OpenAI via LiteLLM
export HINDSIGHT_API_LLM_PROVIDER=litellm
export HINDSIGHT_API_LLM_API_KEY=your-azure-api-key
export HINDSIGHT_API_LLM_MODEL=azure/gpt-4o
# Together AI via LiteLLM
export HINDSIGHT_API_LLM_PROVIDER=litellm
export HINDSIGHT_API_LLM_API_KEY=your-together-api-key
export HINDSIGHT_API_LLM_MODEL=together_ai/meta-llama/Llama-3-70b-chat-hf
```
:::tip OpenAI Codex, Claude Code & Vertex AI Setup

16
hindsight-docs/docs/developer/models.mdx
View file

@ -22,7 +22,7 @@ Used for fact extraction, entity resolution, mental model consolidation, and ans
<LLMProvidersGrid />
Also supports **any OpenAI-compatible API** (e.g., Azure OpenAI, Together AI, Fireworks).
Also supports **any OpenAI-compatible API** (e.g., Azure OpenAI, Together AI, Fireworks) and **100+ providers via LiteLLM** (e.g., AWS Bedrock, Azure OpenAI, Together AI).
:::tip OpenAI-Compatible Providers
Hindsight works with any provider that exposes an OpenAI-compatible API (e.g., Azure OpenAI). Simply set `HINDSIGHT_API_LLM_PROVIDER=openai` and configure `HINDSIGHT_API_LLM_BASE_URL` to point to your provider's endpoint.
@ -30,6 +30,18 @@ Hindsight works with any provider that exposes an OpenAI-compatible API (e.g., A
See [Configuration](./configuration#llm-provider) for setup examples.
:::
:::tip AWS Bedrock
Set `HINDSIGHT_API_LLM_PROVIDER=bedrock` to use AWS Bedrock models directly. Model names use Bedrock model IDs (e.g., `us.amazon.nova-2-lite-v1:0`). No API key is required — authentication uses AWS credentials (`AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY`, `AWS_REGION_NAME`) or IAM roles.
See [Configuration](./configuration#llm-provider) for setup examples.
:::
:::tip LiteLLM Provider (Azure, Together AI, and more)
Set `HINDSIGHT_API_LLM_PROVIDER=litellm` to use any model supported by [LiteLLM](https://docs.litellm.ai/docs/providers), including **Azure OpenAI**, **Together AI**, **Fireworks AI**, and many more. Model names use LiteLLM's provider prefix format (e.g., `azure/gpt-4o`).
See [Configuration](./configuration#llm-provider) for setup examples.
:::
### Benchmarks
Not sure which model to use? The **[Model Leaderboard](https://benchmarks.hindsight.vectorize.io/)** benchmarks models across accuracy, speed, cost, and reliability for retain, reflect, and observation consolidation so you can pick the right trade-off for your use case.
@ -73,6 +85,8 @@ Each provider has a recommended default model that's used when `HINDSIGHT_API_LL
| `vertexai` | `gemini-2.0-flash-001` |
| `openai-codex` | `gpt-5.2-codex` |
| `claude-code` | `claude-sonnet-4-5-20250929` |
| `bedrock` | `us.amazon.nova-2-lite-v1:0` |
| `litellm` | `gpt-4o-mini` |
**Example:** Setting just the provider uses its default model:
```bash

2
hindsight-docs/docs/developer/monitoring.md
View file

@ -79,7 +79,7 @@ The `source` label allows distinguishing between:
| `hindsight.llm.tokens.output` | Counter | provider, model, scope, success, token_bucket | Output tokens from LLM calls |
**Labels:**
- `provider`: LLM provider (`openai`, `anthropic`, `gemini`, `groq`, `ollama`, `lmstudio`)
- `provider`: LLM provider (`openai`, `anthropic`, `gemini`, `groq`, `ollama`, `lmstudio`, `bedrock`, `litellm`)
- `model`: Model name (e.g., `gpt-4`, `claude-3-sonnet`)
- `scope`: What the LLM call is for (`memory`, `reflect`, `consolidation`, `answer`)
- `success`: Whether the call succeeded (`true`, `false`)

4
hindsight-docs/src/components/SupportedGrids.tsx
View file

@ -2,7 +2,7 @@ import React from 'react';
import type {IconType} from 'react-icons';
import {IconGrid} from './IconGrid';
import {SiPython, SiGo, SiOpenai, SiAnthropic, SiGooglegemini, SiOllama} from 'react-icons/si';
import {LuTerminal, LuPlug, LuZap, LuBrainCog, LuSparkles, LuGlobe} from 'react-icons/lu';
import {LuTerminal, LuPlug, LuZap, LuBrainCog, LuSparkles, LuGlobe, LuLayers, LuCloud} from 'react-icons/lu';
const OpenAICompatibleIcon: IconType = ({size = 28, ...props}) => (
<span style={{position: 'relative', display: 'inline-flex'}}>
@ -39,6 +39,8 @@ export function LLMProvidersGrid() {
{ label: 'LM Studio', icon: LuBrainCog },
{ label: 'MiniMax', icon: LuSparkles },
{ label: 'OpenAI Compatible', icon: OpenAICompatibleIcon },
{ label: 'AWS Bedrock', icon: LuCloud },
{ label: 'LiteLLM (100+)', icon: LuLayers },
]} />
);
}

20
skills/hindsight-docs/references/developer/configuration.md
View file

@ -160,7 +160,7 @@ To switch between backends:
| Variable | Description | Default |
|----------|-------------|---------|
| `HINDSIGHT_API_LLM_PROVIDER` | Provider: `openai`, `openai-codex`, `claude-code`, `anthropic`, `gemini`, `groq`, `minimax`, `ollama`, `lmstudio`, `vertexai` | `openai` |
| `HINDSIGHT_API_LLM_PROVIDER` | Provider: `openai`, `openai-codex`, `claude-code`, `anthropic`, `gemini`, `groq`, `minimax`, `ollama`, `lmstudio`, `vertexai`, `bedrock`, `litellm` | `openai` |
| `HINDSIGHT_API_LLM_API_KEY` | API key for LLM provider | - |
| `HINDSIGHT_API_LLM_MODEL` | Model name | `gpt-5-mini` |
| `HINDSIGHT_API_LLM_BASE_URL` | Custom LLM endpoint | Provider default |
@ -232,6 +232,24 @@ export HINDSIGHT_API_LLM_MODEL=gpt-5.2-codex
export HINDSIGHT_API_LLM_PROVIDER=claude-code
export HINDSIGHT_API_LLM_MODEL=claude-sonnet-4-5-20250929
# No API key needed - uses claude auth login credentials
# AWS Bedrock (native support - no API key needed, uses AWS credentials)
export HINDSIGHT_API_LLM_PROVIDER=bedrock
export HINDSIGHT_API_LLM_MODEL=us.amazon.nova-2-lite-v1:0
export AWS_ACCESS_KEY_ID=your-access-key
export AWS_SECRET_ACCESS_KEY=your-secret-key
export AWS_REGION_NAME=us-east-1
# LiteLLM (100+ providers via LiteLLM SDK)
# Azure OpenAI via LiteLLM
export HINDSIGHT_API_LLM_PROVIDER=litellm
export HINDSIGHT_API_LLM_API_KEY=your-azure-api-key
export HINDSIGHT_API_LLM_MODEL=azure/gpt-4o
# Together AI via LiteLLM
export HINDSIGHT_API_LLM_PROVIDER=litellm
export HINDSIGHT_API_LLM_API_KEY=your-together-api-key
export HINDSIGHT_API_LLM_MODEL=together_ai/meta-llama/Llama-3-70b-chat-hf
```
:::tip OpenAI Codex, Claude Code & Vertex AI Setup

14
skills/hindsight-docs/references/developer/models.md
View file

@ -21,12 +21,22 @@ Used for fact extraction, entity resolution, mental model consolidation, and ans
<LLMProvidersGrid />
Also supports **any OpenAI-compatible API** (e.g., Azure OpenAI, Together AI, Fireworks).
Also supports **any OpenAI-compatible API** (e.g., Azure OpenAI, Together AI, Fireworks) and **100+ providers via LiteLLM** (e.g., AWS Bedrock, Azure OpenAI, Together AI).
> **💡 OpenAI-Compatible Providers**
>
Hindsight works with any provider that exposes an OpenAI-compatible API (e.g., Azure OpenAI). Simply set `HINDSIGHT_API_LLM_PROVIDER=openai` and configure `HINDSIGHT_API_LLM_BASE_URL` to point to your provider's endpoint.
See [Configuration](./configuration#llm-provider) for setup examples.
> **💡 AWS Bedrock**
>
Set `HINDSIGHT_API_LLM_PROVIDER=bedrock` to use AWS Bedrock models directly. Model names use Bedrock model IDs (e.g., `us.amazon.nova-2-lite-v1:0`). No API key is required — authentication uses AWS credentials (`AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY`, `AWS_REGION_NAME`) or IAM roles.
See [Configuration](./configuration#llm-provider) for setup examples.
> **💡 LiteLLM Provider (Azure, Together AI, and more)**
>
Set `HINDSIGHT_API_LLM_PROVIDER=litellm` to use any model supported by [LiteLLM](https://docs.litellm.ai/docs/providers), including **Azure OpenAI**, **Together AI**, **Fireworks AI**, and many more. Model names use LiteLLM's provider prefix format (e.g., `azure/gpt-4o`).
See [Configuration](./configuration#llm-provider) for setup examples.
### Benchmarks
@ -71,6 +81,8 @@ Each provider has a recommended default model that's used when `HINDSIGHT_API_LL
| `vertexai` | `gemini-2.0-flash-001` |
| `openai-codex` | `gpt-5.2-codex` |
| `claude-code` | `claude-sonnet-4-5-20250929` |
| `bedrock` | `us.amazon.nova-2-lite-v1:0` |
| `litellm` | `gpt-4o-mini` |
**Example:** Setting just the provider uses its default model:
```bash

2
skills/hindsight-docs/references/developer/monitoring.md
View file

@ -79,7 +79,7 @@ The `source` label allows distinguishing between:
| `hindsight.llm.tokens.output` | Counter | provider, model, scope, success, token_bucket | Output tokens from LLM calls |
**Labels:**
- `provider`: LLM provider (`openai`, `anthropic`, `gemini`, `groq`, `ollama`, `lmstudio`)
- `provider`: LLM provider (`openai`, `anthropic`, `gemini`, `groq`, `ollama`, `lmstudio`, `bedrock`, `litellm`)
- `model`: Model name (e.g., `gpt-4`, `claude-3-sonnet`)
- `scope`: What the LLM call is for (`memory`, `reflect`, `consolidation`, `answer`)
- `success`: Whether the call succeeded (`true`, `false`)

50
uv.lock
View file

@ -441,6 +441,33 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/10/cb/f2ad4230dc2eb1a74edf38f1a38b9b52277f75bef262d8908e60d957e13c/blinker-1.9.0-py3-none-any.whl", hash = "sha256:ba0efaa9080b619ff2f3459d1d500c57bddea4a6b424b60a91141db6fd2f08bc", size = 8458 },
]
[[package]]
name = "boto3"
version = "1.42.74"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "botocore" },
{ name = "jmespath" },
{ name = "s3transfer" },
]
wheels = [
{ url = "https://files.pythonhosted.org/packages/ad/16/a264b4da2af99f4a12609b93fea941cce5ec41da14b33ed3fef77a910f0c/boto3-1.42.74-py3-none-any.whl", hash = "sha256:4bf89c044d618fe4435af854ab820f09dd43569c0df15d7beb0398f50b9aa970", size = 140557 },
]
[[package]]
name = "botocore"
version = "1.42.74"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "jmespath" },
{ name = "python-dateutil" },
{ name = "urllib3" },
]
sdist = { url = "https://files.pythonhosted.org/packages/9d/c7/cab8a14f0b69944bd0dd1fd58559163455b347eeda00bf836e93ce2684e4/botocore-1.42.74.tar.gz", hash = "sha256:9cf5cdffc6c90ed87b0fe184676806182588be0d0df9b363e9fe3e2923ac8e80", size = 15014379 }
wheels = [
{ url = "https://files.pythonhosted.org/packages/d3/65/75852e04de5423c9b0c5b88241d0bdea33e6c6f454c88b71377d230216f2/botocore-1.42.74-py3-none-any.whl", hash = "sha256:3a76a8af08b5de82e51a0ae132394e226e15dbf21c8146ac3f7c1f881517a7a7", size = 14688218 },
]
[[package]]
name = "cachetools"
version = "6.2.1"
@ -1508,6 +1535,7 @@ dependencies = [
{ name = "anthropic" },
{ name = "asyncpg" },
{ name = "authlib" },
{ name = "boto3" },
{ name = "claude-agent-sdk" },
{ name = "cohere" },
{ name = "cryptography" },
@ -1609,6 +1637,7 @@ requires-dist = [
{ name = "anthropic", specifier = ">=0.40.0" },
{ name = "asyncpg", specifier = ">=0.29.0" },
{ name = "authlib", specifier = ">=1.6.9" },
{ name = "boto3", specifier = ">=1.42.74" },
{ name = "claude-agent-sdk", specifier = ">=0.1.27" },
{ name = "cohere", specifier = ">=5.0.0" },
{ name = "cryptography", specifier = ">=46.0.5" },
@ -2082,6 +2111,15 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/2f/9c/6753e6522b8d0ef07d3a3d239426669e984fb0eba15a315cdbc1253904e4/jiter-0.12.0-graalpy312-graalpy250_312_native-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c24e864cb30ab82311c6425655b0cdab0a98c5d973b065c66a3f020740c2324c", size = 346110 },
]
[[package]]
name = "jmespath"
version = "1.1.0"
source = { registry = "https://pypi.org/simple" }
sdist = { url = "https://files.pythonhosted.org/packages/d3/59/322338183ecda247fb5d1763a6cbe46eff7222eaeebafd9fa65d4bf5cb11/jmespath-1.1.0.tar.gz", hash = "sha256:472c87d80f36026ae83c6ddd0f1d05d4e510134ed462851fd5f754c8c3cbb88d", size = 27377 }
wheels = [
{ url = "https://files.pythonhosted.org/packages/14/2f/967ba146e6d58cf6a652da73885f52fc68001525b4197effc174321d70b4/jmespath-1.1.0-py3-none-any.whl", hash = "sha256:a5663118de4908c91729bea0acadca56526eb2698e83de10cd116ae0f4e97c64", size = 20419 },
]
[[package]]
name = "joblib"
version = "1.5.2"
@ -4685,6 +4723,18 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/26/09/7a9520315decd2334afa65ed258fed438f070e31f05a2e43dd480a5e5911/ruff-0.14.9-py3-none-win_arm64.whl", hash = "sha256:8e821c366517a074046d92f0e9213ed1c13dbc5b37a7fc20b07f79b64d62cc84", size = 13744730 },
]
[[package]]
name = "s3transfer"
version = "0.16.0"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "botocore" },
]
sdist = { url = "https://files.pythonhosted.org/packages/05/04/74127fc843314818edfa81b5540e26dd537353b123a4edc563109d8f17dd/s3transfer-0.16.0.tar.gz", hash = "sha256:8e990f13268025792229cd52fa10cb7163744bf56e719e0b9cb925ab79abf920", size = 153827 }
wheels = [
{ url = "https://files.pythonhosted.org/packages/fc/51/727abb13f44c1fcf6d145979e1535a35794db0f6e450a0cb46aa24732fe2/s3transfer-0.16.0-py3-none-any.whl", hash = "sha256:18e25d66fed509e3868dc1572b3f427ff947dd2c56f844a5bf09481ad3f3b2fe", size = 86830 },
]
[[package]]
name = "safetensors"
version = "0.6.2"