539190b69e
* feat: support for codex and claude-code as llm * Remove refactoring plan file * Consolidate Anthropic tests into main LLM provider test suite - Add Anthropic models (Sonnet, Opus, Haiku) to MODEL_MATRIX - Remove separate test_anthropic_provider.py file - All Anthropic models now tested with standard memory operations * Add provider-specific default models Each LLM provider now has a sensible default model that's used when HINDSIGHT_API_LLM_MODEL is not explicitly set. This simplifies configuration - users can specify just the provider and API key. Changes: - Add PROVIDER_DEFAULT_MODELS mapping in config.py - Update config logic to use provider defaults for both global and per-operation LLM configs - Add comprehensive tests for provider default model selection - Document provider defaults in models.md Example usage: export HINDSIGHT_API_LLM_PROVIDER=anthropic export HINDSIGHT_API_LLM_API_KEY=sk-ant-xxx # Automatically uses claude-sonnet-4-20250514 Provider defaults: - openai: gpt-5-mini - anthropic: claude-sonnet-4-20250514 - gemini: gemini-2.5-flash - groq: openai/gpt-oss-120b - ollama: gemma3:12b - lmstudio: local-model - vertexai: gemini-2.0-flash-001 - openai-codex: o3-mini - claude-code: claude-sonnet-4-20250514 - mock: mock-model * Update provider default models - openai: gpt-5-mini -> o3-mini - anthropic: claude-sonnet-4-20250514 -> claude-haiku-4-5-20251001 - openai-codex: o3-mini -> gpt-5.2-codex - claude-code: claude-sonnet-4-20250514 -> claude-sonnet-4-5-20250929 Updated tests and documentation to reflect new defaults. * Move OpenAI Codex and Claude Code setup to models.md Moved detailed setup instructions for OpenAI Codex and Claude Code from configuration.md to models.md where they better fit with model-specific documentation. Changes: - Move "OpenAI Codex Setup" section from configuration.md to models.md - Move "Claude Code Setup" section from configuration.md to models.md - Add cross-reference tip in configuration.md pointing to models.md - Update default model in Claude Code example to claude-sonnet-4-5-20250929 - Keep basic provider examples in configuration.md for quick reference This makes the configuration.md page more focused on environment variables while models.md contains provider-specific setup details.
123 lines
5 KiB
Python
123 lines
5 KiB
Python
"""Test provider-specific default models in config."""
|
|
|
|
import os
|
|
|
|
import pytest
|
|
|
|
|
|
def test_provider_default_models():
|
|
"""Test that each provider has a default model and it's used when model is not explicitly set."""
|
|
from hindsight_api.config import PROVIDER_DEFAULT_MODELS, HindsightConfig, clear_config_cache
|
|
|
|
# Save original env vars
|
|
original_provider = os.environ.get("HINDSIGHT_API_LLM_PROVIDER")
|
|
original_model = os.environ.get("HINDSIGHT_API_LLM_MODEL")
|
|
|
|
try:
|
|
# Test each provider has a default
|
|
for provider, expected_model in PROVIDER_DEFAULT_MODELS.items():
|
|
clear_config_cache()
|
|
os.environ["HINDSIGHT_API_LLM_PROVIDER"] = provider
|
|
# Remove explicit model setting to test default
|
|
if "HINDSIGHT_API_LLM_MODEL" in os.environ:
|
|
del os.environ["HINDSIGHT_API_LLM_MODEL"]
|
|
|
|
config = HindsightConfig.from_env()
|
|
assert config.llm_provider == provider, f"Provider mismatch for {provider}"
|
|
assert config.llm_model == expected_model, f"Expected {expected_model} for {provider}, got {config.llm_model}"
|
|
|
|
finally:
|
|
# Restore original env vars
|
|
clear_config_cache()
|
|
if original_provider:
|
|
os.environ["HINDSIGHT_API_LLM_PROVIDER"] = original_provider
|
|
elif "HINDSIGHT_API_LLM_PROVIDER" in os.environ:
|
|
del os.environ["HINDSIGHT_API_LLM_PROVIDER"]
|
|
|
|
if original_model:
|
|
os.environ["HINDSIGHT_API_LLM_MODEL"] = original_model
|
|
elif "HINDSIGHT_API_LLM_MODEL" in os.environ:
|
|
del os.environ["HINDSIGHT_API_LLM_MODEL"]
|
|
|
|
|
|
def test_explicit_model_overrides_provider_default():
|
|
"""Test that explicit model setting overrides provider default."""
|
|
from hindsight_api.config import HindsightConfig, clear_config_cache
|
|
|
|
original_provider = os.environ.get("HINDSIGHT_API_LLM_PROVIDER")
|
|
original_model = os.environ.get("HINDSIGHT_API_LLM_MODEL")
|
|
|
|
try:
|
|
clear_config_cache()
|
|
os.environ["HINDSIGHT_API_LLM_PROVIDER"] = "anthropic"
|
|
os.environ["HINDSIGHT_API_LLM_MODEL"] = "claude-sonnet-4-5-20250929"
|
|
|
|
config = HindsightConfig.from_env()
|
|
assert config.llm_provider == "anthropic"
|
|
assert config.llm_model == "claude-sonnet-4-5-20250929", "Explicit model should override default"
|
|
|
|
finally:
|
|
clear_config_cache()
|
|
if original_provider:
|
|
os.environ["HINDSIGHT_API_LLM_PROVIDER"] = original_provider
|
|
elif "HINDSIGHT_API_LLM_PROVIDER" in os.environ:
|
|
del os.environ["HINDSIGHT_API_LLM_PROVIDER"]
|
|
|
|
if original_model:
|
|
os.environ["HINDSIGHT_API_LLM_MODEL"] = original_model
|
|
elif "HINDSIGHT_API_LLM_MODEL" in os.environ:
|
|
del os.environ["HINDSIGHT_API_LLM_MODEL"]
|
|
|
|
|
|
def test_per_operation_provider_default_model():
|
|
"""Test that per-operation providers use their own default models."""
|
|
from hindsight_api.config import HindsightConfig, clear_config_cache
|
|
|
|
original_provider = os.environ.get("HINDSIGHT_API_LLM_PROVIDER")
|
|
original_model = os.environ.get("HINDSIGHT_API_LLM_MODEL")
|
|
original_retain_provider = os.environ.get("HINDSIGHT_API_RETAIN_LLM_PROVIDER")
|
|
original_retain_model = os.environ.get("HINDSIGHT_API_RETAIN_LLM_MODEL")
|
|
|
|
try:
|
|
clear_config_cache()
|
|
os.environ["HINDSIGHT_API_LLM_PROVIDER"] = "openai"
|
|
# Remove explicit model to use provider default
|
|
if "HINDSIGHT_API_LLM_MODEL" in os.environ:
|
|
del os.environ["HINDSIGHT_API_LLM_MODEL"]
|
|
|
|
# Set retain-specific provider but not model
|
|
os.environ["HINDSIGHT_API_RETAIN_LLM_PROVIDER"] = "anthropic"
|
|
if "HINDSIGHT_API_RETAIN_LLM_MODEL" in os.environ:
|
|
del os.environ["HINDSIGHT_API_RETAIN_LLM_MODEL"]
|
|
|
|
config = HindsightConfig.from_env()
|
|
|
|
# Global LLM should use OpenAI default
|
|
assert config.llm_model == "o3-mini", f"Expected o3-mini, got {config.llm_model}"
|
|
|
|
# Retain should use Anthropic default
|
|
assert (
|
|
config.retain_llm_model == "claude-haiku-4-5-20251001"
|
|
), f"Expected claude-haiku-4-5-20251001, got {config.retain_llm_model}"
|
|
|
|
finally:
|
|
clear_config_cache()
|
|
if original_provider:
|
|
os.environ["HINDSIGHT_API_LLM_PROVIDER"] = original_provider
|
|
elif "HINDSIGHT_API_LLM_PROVIDER" in os.environ:
|
|
del os.environ["HINDSIGHT_API_LLM_PROVIDER"]
|
|
|
|
if original_model:
|
|
os.environ["HINDSIGHT_API_LLM_MODEL"] = original_model
|
|
elif "HINDSIGHT_API_LLM_MODEL" in os.environ:
|
|
del os.environ["HINDSIGHT_API_LLM_MODEL"]
|
|
|
|
if original_retain_provider:
|
|
os.environ["HINDSIGHT_API_RETAIN_LLM_PROVIDER"] = original_retain_provider
|
|
elif "HINDSIGHT_API_RETAIN_LLM_PROVIDER" in os.environ:
|
|
del os.environ["HINDSIGHT_API_RETAIN_LLM_PROVIDER"]
|
|
|
|
if original_retain_model:
|
|
os.environ["HINDSIGHT_API_RETAIN_LLM_MODEL"] = original_retain_model
|
|
elif "HINDSIGHT_API_RETAIN_LLM_MODEL" in os.environ:
|
|
del os.environ["HINDSIGHT_API_RETAIN_LLM_MODEL"]
|