15ea23d5d6
* feat: introduce hindsight-api-slim and hindsight-all-slim packages Closes #552 - Move all source code from hindsight-api/ to new hindsight-api-slim/ - hindsight-api-slim has heavy ML deps (torch, sentence-transformers, transformers, einops, flashrank, mlx, mlx-lm, safetensors) and pg0-embedded as optional extras: [local-ml], [embedded-db], [all] - hindsight-api becomes a zero-code meta-package depending on hindsight-api-slim[all] for full backward compatibility - Add hindsight-all-slim meta-package: hindsight-api-slim + client + embed - hindsight-all updated to depend on hindsight-api-slim[all] - pg0.py: lazy-import pg0 with clear ImportError pointing to [embedded-db] - Dockerfile: replace sed hack with proper uv sync --extra flags - Update release.yml, test.yml, lint.sh, release.sh, CLAUDE.md and all path references throughout the repo * refactor: rename hindsight/ directory to hindsight-all/ * docs: document hindsight-api-slim and hindsight-all-slim package variants Add package variants table and extras explanation to installation.md * docs: remove emojis from installation.md, use professional tone * docs: link Docker slim variant to pip package variants section * docs: consolidate Docker image variants into single table * ci: fix working-directory paths after package restructure - Replace all hindsight-api → hindsight-api-slim in test.yml - Replace hindsight → hindsight-all in test.yml - Add --extra embedded-db to test-embed API install step * ci: add local-ml and embedded-db extras to API sync steps These extras were previously implicit in the old hindsight-api package (which bundled everything). Now that hindsight-api-slim uses optional extras, we must explicitly request local-ml and embedded-db in CI. * ci: add API install step with embedded-db to test-embed smoke test The smoke test starts hindsight-api as a daemon, which requires pg0-embedded. Add a dedicated install step for hindsight-api-slim with embedded-db extra so the daemon can start successfully. * ci: remove --no-install-project when using optional extras When --no-install-project is combined with --extra, the optional deps are not installed because extras require the project to be active. Remove --no-install-project from steps that need local-ml or embedded-db. * ci: fix ordering of uv sync steps to preserve optional extras When uv sync runs for a different workspace member, it removes optional extras installed for other members. Fix by always running extra-requiring API sync last, after other workspace member syncs. Also remove --no-install-project from embedded-db sync in test-embed, as --no-install-project prevents optional extras from being active. * ci: add local-ml extra to test-embed API install for smoke test The smoke test starts the full API server which needs sentence-transformers for local embeddings (default provider). Add local-ml extra to the install. * ci: simplify extras with --all-extras and add slim pip smoke test - Replace explicit --extra local-ml --extra embedded-db with --all-extras for cleaner, more maintainable sync steps - Add test-pip-slim job: tests hindsight-api-slim[embedded-db] without local ML models, using Cohere for embeddings/reranking (mirrors Docker slim smoke test approach) * ci: simplify slim smoke test to health check only (mirrors Docker test)
362 lines
13 KiB
Python
362 lines
13 KiB
Python
"""
|
|
Tests for per-operation LLM configuration.
|
|
|
|
Verifies that retain and reflect operations use their respective LLM configs.
|
|
"""
|
|
|
|
import os
|
|
|
|
import pytest
|
|
|
|
|
|
@pytest.fixture(autouse=True)
|
|
def setup_test_env():
|
|
"""Set up environment for each test, restoring original values after."""
|
|
from hindsight_api.config import clear_config_cache
|
|
|
|
# Save original environment values
|
|
env_vars_to_set = {
|
|
"HINDSIGHT_API_SKIP_LLM_VERIFICATION": "true",
|
|
"HINDSIGHT_API_LAZY_RERANKER": "true",
|
|
"HINDSIGHT_API_LLM_PROVIDER": "mock",
|
|
"HINDSIGHT_API_LLM_MODEL": "default-model",
|
|
"HINDSIGHT_API_RETAIN_LLM_PROVIDER": "mock",
|
|
"HINDSIGHT_API_RETAIN_LLM_MODEL": "retain-model",
|
|
"HINDSIGHT_API_REFLECT_LLM_PROVIDER": "mock",
|
|
"HINDSIGHT_API_REFLECT_LLM_MODEL": "reflect-model",
|
|
}
|
|
|
|
# Save original values
|
|
original_values = {}
|
|
for key in env_vars_to_set:
|
|
original_values[key] = os.environ.get(key)
|
|
|
|
# Set test values
|
|
for key, value in env_vars_to_set.items():
|
|
os.environ[key] = value
|
|
|
|
clear_config_cache()
|
|
|
|
yield
|
|
|
|
# Restore original environment
|
|
for key, original_value in original_values.items():
|
|
if original_value is None:
|
|
os.environ.pop(key, None)
|
|
else:
|
|
os.environ[key] = original_value
|
|
|
|
clear_config_cache()
|
|
|
|
|
|
class TestPerOperationLLMConfig:
|
|
"""Test that per-operation LLM configs are correctly applied."""
|
|
|
|
def test_config_loads_per_operation_settings(self):
|
|
"""Test that config correctly loads per-operation LLM settings."""
|
|
from hindsight_api.config import get_config
|
|
|
|
config = get_config()
|
|
|
|
# Default config
|
|
assert config.llm_provider == "mock"
|
|
assert config.llm_model == "default-model"
|
|
|
|
# Retain config
|
|
assert config.retain_llm_provider == "mock"
|
|
assert config.retain_llm_model == "retain-model"
|
|
|
|
# Reflect config
|
|
assert config.reflect_llm_provider == "mock"
|
|
assert config.reflect_llm_model == "reflect-model"
|
|
|
|
def test_memory_engine_creates_separate_llm_configs(self):
|
|
"""Test that MemoryEngine creates separate LLM configs for each operation."""
|
|
from hindsight_api import MemoryEngine
|
|
|
|
engine = MemoryEngine(
|
|
skip_llm_verification=True,
|
|
lazy_reranker=True,
|
|
)
|
|
|
|
# Verify default config
|
|
assert engine._llm_config.provider == "mock"
|
|
assert engine._llm_config.model == "default-model"
|
|
|
|
# Verify retain config
|
|
assert engine._retain_llm_config.provider == "mock"
|
|
assert engine._retain_llm_config.model == "retain-model"
|
|
|
|
# Verify reflect config
|
|
assert engine._reflect_llm_config.provider == "mock"
|
|
assert engine._reflect_llm_config.model == "reflect-model"
|
|
|
|
def test_memory_engine_with_explicit_params(self):
|
|
"""Test that explicit params override env config."""
|
|
from hindsight_api import MemoryEngine
|
|
|
|
engine = MemoryEngine(
|
|
memory_llm_provider="mock",
|
|
memory_llm_model="explicit-default",
|
|
retain_llm_provider="mock",
|
|
retain_llm_model="explicit-retain",
|
|
reflect_llm_provider="mock",
|
|
reflect_llm_model="explicit-reflect",
|
|
skip_llm_verification=True,
|
|
lazy_reranker=True,
|
|
)
|
|
|
|
assert engine._llm_config.model == "explicit-default"
|
|
assert engine._retain_llm_config.model == "explicit-retain"
|
|
assert engine._reflect_llm_config.model == "explicit-reflect"
|
|
|
|
def test_memory_engine_fallback_when_no_per_operation_config(self):
|
|
"""Test that per-operation configs fall back to default when not set."""
|
|
from hindsight_api.config import clear_config_cache as clear_cache
|
|
|
|
# Temporarily clear per-operation env vars
|
|
retain_provider = os.environ.pop("HINDSIGHT_API_RETAIN_LLM_PROVIDER", None)
|
|
retain_model = os.environ.pop("HINDSIGHT_API_RETAIN_LLM_MODEL", None)
|
|
reflect_provider = os.environ.pop("HINDSIGHT_API_REFLECT_LLM_PROVIDER", None)
|
|
reflect_model = os.environ.pop("HINDSIGHT_API_REFLECT_LLM_MODEL", None)
|
|
|
|
try:
|
|
clear_cache()
|
|
from hindsight_api import MemoryEngine
|
|
|
|
engine = MemoryEngine(
|
|
skip_llm_verification=True,
|
|
lazy_reranker=True,
|
|
)
|
|
|
|
# All should fall back to default
|
|
assert engine._llm_config.model == "default-model"
|
|
assert engine._retain_llm_config.model == "default-model"
|
|
assert engine._reflect_llm_config.model == "default-model"
|
|
finally:
|
|
# Restore env vars
|
|
if retain_provider:
|
|
os.environ["HINDSIGHT_API_RETAIN_LLM_PROVIDER"] = retain_provider
|
|
if retain_model:
|
|
os.environ["HINDSIGHT_API_RETAIN_LLM_MODEL"] = retain_model
|
|
if reflect_provider:
|
|
os.environ["HINDSIGHT_API_REFLECT_LLM_PROVIDER"] = reflect_provider
|
|
if reflect_model:
|
|
os.environ["HINDSIGHT_API_REFLECT_LLM_MODEL"] = reflect_model
|
|
clear_cache()
|
|
|
|
|
|
class TestMockLLMProvider:
|
|
"""Test the mock LLM provider functionality."""
|
|
|
|
def test_mock_provider_records_calls(self):
|
|
"""Test that mock provider records calls."""
|
|
from hindsight_api.engine.llm_wrapper import LLMProvider
|
|
|
|
provider = LLMProvider(
|
|
provider="mock",
|
|
api_key="",
|
|
base_url="",
|
|
model="test-model",
|
|
)
|
|
|
|
import asyncio
|
|
|
|
async def make_call():
|
|
return await provider.call(
|
|
messages=[{"role": "user", "content": "test"}],
|
|
scope="test_scope",
|
|
)
|
|
|
|
result = asyncio.get_event_loop().run_until_complete(make_call())
|
|
|
|
# Verify call was recorded
|
|
calls = provider.get_mock_calls()
|
|
assert len(calls) == 1
|
|
assert calls[0]["model"] == "test-model"
|
|
assert calls[0]["scope"] == "test_scope"
|
|
assert calls[0]["messages"] == [{"role": "user", "content": "test"}]
|
|
|
|
def test_mock_provider_returns_custom_response(self):
|
|
"""Test that mock provider can return custom responses."""
|
|
from hindsight_api.engine.llm_wrapper import LLMProvider
|
|
|
|
provider = LLMProvider(
|
|
provider="mock",
|
|
api_key="",
|
|
base_url="",
|
|
model="test-model",
|
|
)
|
|
|
|
provider.set_mock_response({"custom": "response"})
|
|
|
|
import asyncio
|
|
|
|
async def make_call():
|
|
return await provider.call(
|
|
messages=[{"role": "user", "content": "test"}],
|
|
)
|
|
|
|
result = asyncio.get_event_loop().run_until_complete(make_call())
|
|
assert result == {"custom": "response"}
|
|
|
|
def test_mock_provider_returns_usage_when_requested(self):
|
|
"""Test that mock provider returns token usage."""
|
|
from hindsight_api.engine.llm_wrapper import LLMProvider
|
|
|
|
provider = LLMProvider(
|
|
provider="mock",
|
|
api_key="",
|
|
base_url="",
|
|
model="test-model",
|
|
)
|
|
|
|
import asyncio
|
|
|
|
async def make_call():
|
|
return await provider.call(
|
|
messages=[{"role": "user", "content": "test"}],
|
|
return_usage=True,
|
|
)
|
|
|
|
result, usage = asyncio.get_event_loop().run_until_complete(make_call())
|
|
assert usage.input_tokens == 10
|
|
assert usage.output_tokens == 5
|
|
assert usage.total_tokens == 15
|
|
|
|
|
|
class TestRetainUsesRetainLLMConfig:
|
|
"""Test that retain operations use the retain LLM config."""
|
|
|
|
def test_retain_llm_config_is_passed_to_orchestrator(self):
|
|
"""Verify retain operation is configured to use _retain_llm_config."""
|
|
from hindsight_api import MemoryEngine
|
|
|
|
engine = MemoryEngine(
|
|
memory_llm_provider="mock",
|
|
memory_llm_model="default-model",
|
|
retain_llm_provider="mock",
|
|
retain_llm_model="retain-specific-model",
|
|
reflect_llm_provider="mock",
|
|
reflect_llm_model="reflect-specific-model",
|
|
skip_llm_verification=True,
|
|
lazy_reranker=True,
|
|
)
|
|
|
|
# Verify the retain LLM config is set correctly
|
|
assert engine._retain_llm_config.model == "retain-specific-model"
|
|
assert engine._retain_llm_config.provider == "mock"
|
|
|
|
# Verify it's different from the reflect config
|
|
assert engine._retain_llm_config.model != engine._reflect_llm_config.model
|
|
|
|
|
|
class TestReflectUsesReflectLLMConfig:
|
|
"""Test that reflect operations use the reflect LLM config."""
|
|
|
|
def test_reflect_llm_config_is_set_correctly(self):
|
|
"""Verify reflect/think operation is configured to use _reflect_llm_config."""
|
|
from hindsight_api import MemoryEngine
|
|
|
|
engine = MemoryEngine(
|
|
memory_llm_provider="mock",
|
|
memory_llm_model="default-model",
|
|
retain_llm_provider="mock",
|
|
retain_llm_model="retain-specific-model",
|
|
reflect_llm_provider="mock",
|
|
reflect_llm_model="reflect-specific-model",
|
|
skip_llm_verification=True,
|
|
lazy_reranker=True,
|
|
)
|
|
|
|
# Verify the reflect LLM config is set correctly
|
|
assert engine._reflect_llm_config.model == "reflect-specific-model"
|
|
assert engine._reflect_llm_config.provider == "mock"
|
|
|
|
# Verify it's different from the retain config
|
|
assert engine._reflect_llm_config.model != engine._retain_llm_config.model
|
|
|
|
|
|
class TestRetryAndBackoffConfiguration:
|
|
"""Test retry and backoff configuration options."""
|
|
|
|
def test_global_retry_backoff_config_defaults(self):
|
|
"""Test that global retry/backoff settings have correct defaults."""
|
|
from hindsight_api.config import get_config
|
|
|
|
config = get_config()
|
|
|
|
# Verify global defaults
|
|
assert config.llm_max_retries == 10
|
|
assert config.llm_initial_backoff == 1.0
|
|
assert config.llm_max_backoff == 60.0
|
|
|
|
def test_per_operation_retry_backoff_config_from_env(self):
|
|
"""Test that per-operation retry/backoff settings are loaded from environment."""
|
|
from hindsight_api.config import clear_config_cache
|
|
|
|
# Set per-operation overrides
|
|
os.environ["HINDSIGHT_API_RETAIN_LLM_MAX_RETRIES"] = "3"
|
|
os.environ["HINDSIGHT_API_RETAIN_LLM_INITIAL_BACKOFF"] = "2.0"
|
|
os.environ["HINDSIGHT_API_RETAIN_LLM_MAX_BACKOFF"] = "120.0"
|
|
os.environ["HINDSIGHT_API_REFLECT_LLM_MAX_RETRIES"] = "5"
|
|
os.environ["HINDSIGHT_API_REFLECT_LLM_INITIAL_BACKOFF"] = "1.5"
|
|
os.environ["HINDSIGHT_API_REFLECT_LLM_MAX_BACKOFF"] = "90.0"
|
|
|
|
try:
|
|
clear_config_cache()
|
|
from hindsight_api.config import get_config
|
|
|
|
config = get_config()
|
|
|
|
# Verify retain overrides
|
|
assert config.retain_llm_max_retries == 3
|
|
assert config.retain_llm_initial_backoff == 2.0
|
|
assert config.retain_llm_max_backoff == 120.0
|
|
|
|
# Verify reflect overrides
|
|
assert config.reflect_llm_max_retries == 5
|
|
assert config.reflect_llm_initial_backoff == 1.5
|
|
assert config.reflect_llm_max_backoff == 90.0
|
|
|
|
# Verify global defaults remain unchanged
|
|
assert config.llm_max_retries == 10
|
|
assert config.llm_initial_backoff == 1.0
|
|
assert config.llm_max_backoff == 60.0
|
|
finally:
|
|
# Clean up
|
|
os.environ.pop("HINDSIGHT_API_RETAIN_LLM_MAX_RETRIES", None)
|
|
os.environ.pop("HINDSIGHT_API_RETAIN_LLM_INITIAL_BACKOFF", None)
|
|
os.environ.pop("HINDSIGHT_API_RETAIN_LLM_MAX_BACKOFF", None)
|
|
os.environ.pop("HINDSIGHT_API_REFLECT_LLM_MAX_RETRIES", None)
|
|
os.environ.pop("HINDSIGHT_API_REFLECT_LLM_INITIAL_BACKOFF", None)
|
|
os.environ.pop("HINDSIGHT_API_REFLECT_LLM_MAX_BACKOFF", None)
|
|
clear_config_cache()
|
|
|
|
def test_per_operation_retry_backoff_fallback_to_global(self):
|
|
"""Test that per-operation settings fall back to global when not set."""
|
|
from hindsight_api.config import clear_config_cache, get_config
|
|
|
|
# Set only global values
|
|
os.environ["HINDSIGHT_API_LLM_MAX_RETRIES"] = "7"
|
|
os.environ["HINDSIGHT_API_LLM_INITIAL_BACKOFF"] = "3.0"
|
|
os.environ["HINDSIGHT_API_LLM_MAX_BACKOFF"] = "180.0"
|
|
|
|
try:
|
|
clear_config_cache()
|
|
config = get_config()
|
|
|
|
# Per-operation should be None (will fall back to global at runtime)
|
|
assert config.retain_llm_max_retries is None
|
|
assert config.retain_llm_initial_backoff is None
|
|
assert config.retain_llm_max_backoff is None
|
|
|
|
# Global values should be set
|
|
assert config.llm_max_retries == 7
|
|
assert config.llm_initial_backoff == 3.0
|
|
assert config.llm_max_backoff == 180.0
|
|
finally:
|
|
os.environ.pop("HINDSIGHT_API_LLM_MAX_RETRIES", None)
|
|
os.environ.pop("HINDSIGHT_API_LLM_INITIAL_BACKOFF", None)
|
|
os.environ.pop("HINDSIGHT_API_LLM_MAX_BACKOFF", None)
|
|
clear_config_cache()
|