From 9e5a066d26a0d49cfa06f13efc5695434158001a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=B2=20Boschi?= Date: Wed, 25 Mar 2026 18:01:20 +0100 Subject: [PATCH] feat: add 'none' LLM provider for chunk-only storage mode (#691) Adds a proper 'none' provider option so users can run Hindsight as a chunk store with semantic search but without any LLM dependency, replacing the hacky workaround of setting provider to 'mock'. When HINDSIGHT_API_LLM_PROVIDER=none: - Retain automatically uses chunks mode (no fact extraction) - Recall works normally (semantic search, BM25, graph retrieval) - Reflect returns HTTP 400 with clear error message - Consolidation/observations are disabled - Mental model refresh returns HTTP 400 - No API key required --- hindsight-api-slim/hindsight_api/api/http.py | 5 + hindsight-api-slim/hindsight_api/config.py | 13 +- .../hindsight_api/engine/llm_wrapper.py | 12 + .../hindsight_api/engine/memory_engine.py | 32 +++ .../engine/providers/__init__.py | 12 +- .../engine/providers/none_llm.py | 78 ++++++ .../tests/test_none_llm_provider.py | 232 ++++++++++++++++++ .../docs/developer/configuration.md | 9 +- 8 files changed, 390 insertions(+), 3 deletions(-) create mode 100644 hindsight-api-slim/hindsight_api/engine/providers/none_llm.py create mode 100644 hindsight-api-slim/tests/test_none_llm_provider.py diff --git a/hindsight-api-slim/hindsight_api/api/http.py b/hindsight-api-slim/hindsight_api/api/http.py index 81a43f66..799cfee7 100644 --- a/hindsight-api-slim/hindsight_api/api/http.py +++ b/hindsight-api-slim/hindsight_api/api/http.py @@ -72,6 +72,7 @@ def FieldWithDefault(default_factory: Callable, **kwargs) -> Any: from hindsight_api.config import get_config from hindsight_api.engine.memory_engine import Budget, _current_schema, _get_tiktoken_encoding, fq_table +from hindsight_api.engine.providers.none_llm import LLMNotAvailableError from hindsight_api.engine.response_models import VALID_RECALL_FACT_TYPES, MemoryFact, TokenUsage from hindsight_api.engine.search.tags import TagGroup, TagsMatch from hindsight_api.extensions import HttpExtension, OperationValidationError, load_extension @@ -2651,6 +2652,8 @@ def _register_routes(app: FastAPI): raise HTTPException(status_code=e.status_code, detail=e.reason) except (AuthenticationError, HTTPException): raise + except LLMNotAvailableError as e: + raise HTTPException(status_code=400, detail=str(e)) except TimeoutError as e: logger.error("Timeout in /v1/default/banks/%s/reflect: %s", bank_id, e) raise HTTPException( @@ -3017,6 +3020,8 @@ def _register_routes(app: FastAPI): request_context=request_context, ) return AsyncOperationSubmitResponse(operation_id=result["operation_id"], status="queued") + except LLMNotAvailableError as e: + raise HTTPException(status_code=400, detail=str(e)) except ValueError as e: raise HTTPException(status_code=404, detail=str(e)) except (AuthenticationError, HTTPException): diff --git a/hindsight-api-slim/hindsight_api/config.py b/hindsight-api-slim/hindsight_api/config.py index 9c0a63c6..83e0ba49 100644 --- a/hindsight-api-slim/hindsight_api/config.py +++ b/hindsight-api-slim/hindsight_api/config.py @@ -365,6 +365,7 @@ PROVIDER_DEFAULT_MODELS = { "openai-codex": "gpt-5.2-codex", "claude-code": "claude-sonnet-4-5-20250929", "mock": "mock-model", + "none": "none", "litellm": "gpt-4o-mini", "bedrock": "us.amazon.nova-2-lite-v1:0", } @@ -958,9 +959,19 @@ class HindsightConfig: f"Invalid text_search_extension: {self.text_search_extension}. Must be one of: {', '.join(valid_text_search)}" ) + # When LLM provider is "none", force chunks-only mode and disable LLM-dependent features + if self.llm_provider == "none": + self.retain_extraction_mode = "chunks" + self.enable_observations = False + logger.info( + "LLM provider set to 'none': forcing retain_extraction_mode='chunks', " + "disabling observations/consolidation. Reflect will return HTTP 400." + ) + # RETAIN_MAX_COMPLETION_TOKENS must be greater than RETAIN_CHUNK_SIZE # to ensure the LLM has enough output capacity to extract facts from chunks - if self.retain_max_completion_tokens <= self.retain_chunk_size: + # (not applicable when provider is "none" since no LLM calls are made) + if self.llm_provider != "none" and self.retain_max_completion_tokens <= self.retain_chunk_size: raise ValueError( f"Invalid configuration: HINDSIGHT_API_RETAIN_MAX_COMPLETION_TOKENS " f"({self.retain_max_completion_tokens}) must be greater than " diff --git a/hindsight-api-slim/hindsight_api/engine/llm_wrapper.py b/hindsight-api-slim/hindsight_api/engine/llm_wrapper.py index 7a8e46c1..79162c34 100644 --- a/hindsight-api-slim/hindsight_api/engine/llm_wrapper.py +++ b/hindsight-api-slim/hindsight_api/engine/llm_wrapper.py @@ -125,6 +125,7 @@ _PROVIDERS_WITHOUT_API_KEY = frozenset( "openai-codex", "claude-code", "mock", + "none", "vertexai", "litellm", "bedrock", @@ -176,6 +177,7 @@ def create_llm_provider( GeminiLLM, LiteLLMLLM, MockLLM, + NoneLLM, OpenAICompatibleLLM, ) @@ -208,6 +210,15 @@ def create_llm_provider( reasoning_effort=reasoning_effort, ) + elif provider_lower == "none": + return NoneLLM( + provider=provider, + api_key=api_key, + base_url=base_url, + model=model, + reasoning_effort=reasoning_effort, + ) + elif provider_lower in ("gemini", "vertexai"): return GeminiLLM( provider=provider, @@ -319,6 +330,7 @@ class LLMProvider: "openai-codex", "claude-code", "mock", + "none", "minimax", "litellm", "bedrock", diff --git a/hindsight-api-slim/hindsight_api/engine/memory_engine.py b/hindsight-api-slim/hindsight_api/engine/memory_engine.py index de3d8c6c..da6d59b4 100644 --- a/hindsight-api-slim/hindsight_api/engine/memory_engine.py +++ b/hindsight-api-slim/hindsight_api/engine/memory_engine.py @@ -328,6 +328,10 @@ class MemoryEngine(MemoryEngineInterface): # Apply defaults from config db_url = db_url or config.database_url memory_llm_provider = memory_llm_provider or config.llm_provider + + # Force skip LLM verification when provider is "none" (no LLM to verify) + if memory_llm_provider == "none": + self._skip_llm_verification = True memory_llm_api_key = memory_llm_api_key or config.llm_api_key if not memory_llm_api_key and requires_api_key(memory_llm_provider): raise ValueError("LLM API key is required. Set HINDSIGHT_API_LLM_API_KEY environment variable.") @@ -820,6 +824,11 @@ class MemoryEngine(MemoryEngineInterface): if not bank_id: raise ValueError("bank_id is required for consolidation task") + # Skip consolidation when LLM provider is "none" + if self._llm_config.provider == "none": + logger.info(f"[CONSOLIDATION] Skipping consolidation for bank {bank_id}: LLM provider is 'none'") + return {"memories_processed": 0, "skipped": True} + from hindsight_api.models import RequestContext from .consolidation import run_consolidation_job @@ -2254,6 +2263,11 @@ class MemoryEngine(MemoryEngineInterface): # Resolve bank-specific config for this operation resolved_config = await self._config_resolver.resolve_full_config(bank_id, request_context) + # Force chunks mode when LLM provider is "none" (no LLM available for fact extraction) + if self._llm_config.provider == "none": + resolved_config.retain_extraction_mode = "chunks" + resolved_config.enable_observations = False + # Apply strategy overrides: explicit strategy > bank default strategy from hindsight_api.config_resolver import apply_strategy @@ -5192,6 +5206,15 @@ class MemoryEngine(MemoryEngineInterface): if self._reflect_llm_config is None: raise ValueError("Memory LLM API key not set. Set HINDSIGHT_API_LLM_API_KEY environment variable.") + # Block reflect when LLM provider is "none" + if self._llm_config.provider == "none": + from .providers.none_llm import LLMNotAvailableError + + raise LLMNotAvailableError( + "Reflect requires an LLM provider. Current provider is set to 'none'. " + "Set HINDSIGHT_API_LLM_PROVIDER to a real provider (e.g., openai, anthropic, gemini)." + ) + # Authenticate tenant and set schema in context (for fq_table()) await self._authenticate_tenant(request_context) @@ -7856,6 +7879,15 @@ class MemoryEngine(MemoryEngineInterface): Returns: Dict with operation_id """ + # Block mental model refresh when LLM provider is "none" + if self._llm_config.provider == "none": + from .providers.none_llm import LLMNotAvailableError + + raise LLMNotAvailableError( + "Mental model refresh requires an LLM provider. Current provider is set to 'none'. " + "Set HINDSIGHT_API_LLM_PROVIDER to a real provider (e.g., openai, anthropic, gemini)." + ) + await self._authenticate_tenant(request_context) # Pre-operation validation (credit check) diff --git a/hindsight-api-slim/hindsight_api/engine/providers/__init__.py b/hindsight-api-slim/hindsight_api/engine/providers/__init__.py index 89ffd3f6..e45eb3cb 100644 --- a/hindsight-api-slim/hindsight_api/engine/providers/__init__.py +++ b/hindsight-api-slim/hindsight_api/engine/providers/__init__.py @@ -10,6 +10,16 @@ from .codex_llm import CodexLLM from .gemini_llm import GeminiLLM from .litellm_llm import LiteLLMLLM from .mock_llm import MockLLM +from .none_llm import NoneLLM from .openai_compatible_llm import OpenAICompatibleLLM -__all__ = ["AnthropicLLM", "ClaudeCodeLLM", "CodexLLM", "GeminiLLM", "LiteLLMLLM", "MockLLM", "OpenAICompatibleLLM"] +__all__ = [ + "AnthropicLLM", + "ClaudeCodeLLM", + "CodexLLM", + "GeminiLLM", + "LiteLLMLLM", + "MockLLM", + "NoneLLM", + "OpenAICompatibleLLM", +] diff --git a/hindsight-api-slim/hindsight_api/engine/providers/none_llm.py b/hindsight-api-slim/hindsight_api/engine/providers/none_llm.py new file mode 100644 index 00000000..97fd3d54 --- /dev/null +++ b/hindsight-api-slim/hindsight_api/engine/providers/none_llm.py @@ -0,0 +1,78 @@ +""" +No-op LLM provider for chunk-only storage mode. + +When the LLM provider is set to "none", the system operates without any LLM dependency. +Retain uses chunks mode (no fact extraction), and reflect/consolidation are disabled. +This provider acts as a safety net — if any code path unexpectedly tries to call the LLM, +it raises a clear error instead of a confusing connection failure. +""" + +import logging +from typing import Any + +from ..llm_interface import LLMInterface +from ..response_models import LLMToolCallResult + +logger = logging.getLogger(__name__) + + +class LLMNotAvailableError(Exception): + """Raised when an operation requires an LLM but the provider is set to 'none'.""" + + pass + + +class NoneLLM(LLMInterface): + """ + No-op LLM provider that rejects all LLM calls. + + Used when HINDSIGHT_API_LLM_PROVIDER=none to run Hindsight as a chunk store + with semantic search but without LLM-based features (fact extraction, reflect, + consolidation). + """ + + async def verify_connection(self) -> None: + """No-op — no LLM connection to verify.""" + logger.debug("NoneLLM: no LLM connection to verify (provider=none)") + + async def call( + self, + messages: list[dict[str, str]], + response_format: Any | None = None, + max_completion_tokens: int | None = None, + temperature: float | None = None, + scope: str = "memory", + max_retries: int = 10, + initial_backoff: float = 1.0, + max_backoff: float = 60.0, + skip_validation: bool = False, + strict_schema: bool = False, + return_usage: bool = False, + ) -> Any: + """Raise LLMNotAvailableError — no LLM is configured.""" + raise LLMNotAvailableError( + "LLM provider is set to 'none'. This operation requires an LLM. " + "Set HINDSIGHT_API_LLM_PROVIDER to a real provider (e.g., openai, anthropic, gemini)." + ) + + async def call_with_tools( + self, + messages: list[dict[str, Any]], + tools: list[dict[str, Any]], + max_completion_tokens: int | None = None, + temperature: float | None = None, + scope: str = "tools", + max_retries: int = 5, + initial_backoff: float = 1.0, + max_backoff: float = 30.0, + tool_choice: str | dict[str, Any] = "auto", + ) -> LLMToolCallResult: + """Raise LLMNotAvailableError — no LLM is configured.""" + raise LLMNotAvailableError( + "LLM provider is set to 'none'. This operation requires an LLM. " + "Set HINDSIGHT_API_LLM_PROVIDER to a real provider (e.g., openai, anthropic, gemini)." + ) + + async def cleanup(self) -> None: + """No-op — nothing to clean up.""" + pass diff --git a/hindsight-api-slim/tests/test_none_llm_provider.py b/hindsight-api-slim/tests/test_none_llm_provider.py new file mode 100644 index 00000000..c3e059db --- /dev/null +++ b/hindsight-api-slim/tests/test_none_llm_provider.py @@ -0,0 +1,232 @@ +""" +Tests for the 'none' LLM provider mode. + +Verifies that when HINDSIGHT_API_LLM_PROVIDER=none: +- Retain defaults to chunks mode (no LLM calls) +- Reflect returns 400 +- Mental model refresh returns 400 +- Consolidation is skipped +- NoneLLM.call() raises LLMNotAvailableError +""" + +import os +from datetime import datetime, timezone + +import httpx +import pytest +import pytest_asyncio + +from hindsight_api import LLMConfig, LocalSTEmbeddings, MemoryEngine, RequestContext +from hindsight_api.api import create_app +from hindsight_api.engine.cross_encoder import LocalSTCrossEncoder +from hindsight_api.engine.memory_engine import Budget +from hindsight_api.engine.providers.none_llm import LLMNotAvailableError, NoneLLM +from hindsight_api.engine.query_analyzer import DateparserQueryAnalyzer +from hindsight_api.engine.task_backend import SyncTaskBackend + + +@pytest.fixture(scope="function") +def request_context(): + return RequestContext() + + +@pytest_asyncio.fixture(scope="function") +async def none_memory(pg0_db_url, embeddings, cross_encoder, query_analyzer): + """MemoryEngine with provider=none.""" + mem = MemoryEngine( + db_url=pg0_db_url, + memory_llm_provider="none", + memory_llm_api_key=None, + memory_llm_model="none", + embeddings=embeddings, + cross_encoder=cross_encoder, + query_analyzer=query_analyzer, + pool_min_size=1, + pool_max_size=5, + run_migrations=False, + task_backend=SyncTaskBackend(), + ) + await mem.initialize() + yield mem + try: + if mem._pool and not mem._pool._closing: + await mem.close() + except Exception: + pass + + +@pytest_asyncio.fixture +async def none_api_client(none_memory): + """HTTP test client backed by a none-provider MemoryEngine.""" + app = create_app(none_memory, initialize_memory=False) + transport = httpx.ASGITransport(app=app) + async with httpx.AsyncClient(transport=transport, base_url="http://test") as client: + yield client + + +# -- Unit tests for NoneLLM --------------------------------------------------- + + +@pytest.mark.asyncio +async def test_none_llm_call_raises(): + """NoneLLM.call() should raise LLMNotAvailableError.""" + llm = NoneLLM(provider="none", api_key="", base_url="", model="none") + with pytest.raises(LLMNotAvailableError): + await llm.call(messages=[{"role": "user", "content": "hello"}]) + + +@pytest.mark.asyncio +async def test_none_llm_call_with_tools_raises(): + """NoneLLM.call_with_tools() should raise LLMNotAvailableError.""" + llm = NoneLLM(provider="none", api_key="", base_url="", model="none") + with pytest.raises(LLMNotAvailableError): + await llm.call_with_tools( + messages=[{"role": "user", "content": "hello"}], + tools=[{"type": "function", "function": {"name": "test", "parameters": {}}}], + ) + + +@pytest.mark.asyncio +async def test_none_llm_verify_connection_succeeds(): + """NoneLLM.verify_connection() should be a no-op.""" + llm = NoneLLM(provider="none", api_key="", base_url="", model="none") + await llm.verify_connection() # Should not raise + + +# -- Config validation tests --------------------------------------------------- + + +def test_config_forces_chunks_mode(): + """When provider is 'none', config.validate() forces retain_extraction_mode='chunks'.""" + from hindsight_api.config import HindsightConfig + + config = HindsightConfig.from_env() + # Override to none for test + config.llm_provider = "none" + config.retain_extraction_mode = "facts" + config.enable_observations = True + config.validate() + + assert config.retain_extraction_mode == "chunks" + assert config.enable_observations is False + + +# -- Integration tests (require database) ------------------------------------- + + +@pytest.mark.asyncio +async def test_retain_works_with_none_provider(none_memory, request_context): + """Retain should work with provider=none, storing chunks without LLM calls.""" + bank_id = f"test_none_retain_{datetime.now(timezone.utc).timestamp()}" + + unit_ids = await none_memory.retain_async( + bank_id=bank_id, + content="Alice is a software engineer. She works at TechCorp and loves Python.", + context="team info", + request_context=request_context, + ) + + assert len(unit_ids) > 0, "Should store chunks even without an LLM" + + +@pytest.mark.asyncio +async def test_recall_works_with_none_provider(none_memory, request_context): + """Recall should work with provider=none (uses embeddings, not LLM).""" + bank_id = f"test_none_recall_{datetime.now(timezone.utc).timestamp()}" + + await none_memory.retain_async( + bank_id=bank_id, + content="Alice is a software engineer at TechCorp.", + context="team info", + request_context=request_context, + ) + + result = await none_memory.recall_async( + bank_id=bank_id, + query="Who is Alice?", + budget=Budget.LOW, + request_context=request_context, + ) + + assert len(result.results) > 0, "Should find results via semantic search" + + +@pytest.mark.asyncio +async def test_reflect_raises_with_none_provider(none_memory, request_context): + """Reflect should raise LLMNotAvailableError with provider=none.""" + bank_id = f"test_none_reflect_{datetime.now(timezone.utc).timestamp()}" + + with pytest.raises(LLMNotAvailableError): + await none_memory.reflect_async( + bank_id=bank_id, + query="What do you know?", + request_context=request_context, + ) + + +@pytest.mark.asyncio +async def test_consolidation_skipped_with_none_provider(none_memory, request_context): + """Consolidation handler should skip when provider=none.""" + result = await none_memory._handle_consolidation({"bank_id": "test_bank"}) + assert result["skipped"] is True + assert result["memories_processed"] == 0 + + +@pytest.mark.asyncio +async def test_mental_model_refresh_raises_with_none_provider(none_memory, request_context): + """Mental model refresh should raise LLMNotAvailableError with provider=none.""" + bank_id = f"test_none_mm_{datetime.now(timezone.utc).timestamp()}" + + with pytest.raises(LLMNotAvailableError): + await none_memory.submit_async_refresh_mental_model( + bank_id=bank_id, + mental_model_id="fake-id", + request_context=request_context, + ) + + +# -- HTTP API tests ----------------------------------------------------------- + + +@pytest.mark.asyncio +async def test_http_reflect_returns_400(none_api_client): + """Reflect endpoint should return 400 when LLM provider is none.""" + bank_id = f"test_none_http_{datetime.now(timezone.utc).timestamp()}" + + response = await none_api_client.post( + f"/v1/default/banks/{bank_id}/reflect", + json={"query": "What do you know?"}, + ) + assert response.status_code == 400 + assert "none" in response.json()["detail"].lower() + + +@pytest.mark.asyncio +async def test_http_retain_works(none_api_client): + """Retain endpoint should work with provider=none (chunks mode).""" + bank_id = f"test_none_http_retain_{datetime.now(timezone.utc).timestamp()}" + + response = await none_api_client.post( + f"/v1/default/banks/{bank_id}/memories", + json={"items": [{"content": "Hello world", "context": "test"}]}, + ) + assert response.status_code == 200 + + +@pytest.mark.asyncio +async def test_http_recall_works(none_api_client): + """Recall endpoint should work with provider=none.""" + bank_id = f"test_none_http_recall_{datetime.now(timezone.utc).timestamp()}" + + # Retain first + await none_api_client.post( + f"/v1/default/banks/{bank_id}/memories", + json={"items": [{"content": "Alice is an engineer.", "context": "test"}]}, + ) + + # Recall + response = await none_api_client.post( + f"/v1/default/banks/{bank_id}/memories/recall", + json={"query": "Alice"}, + ) + assert response.status_code == 200 diff --git a/hindsight-docs/docs/developer/configuration.md b/hindsight-docs/docs/developer/configuration.md index 6a14273a..215b612f 100644 --- a/hindsight-docs/docs/developer/configuration.md +++ b/hindsight-docs/docs/developer/configuration.md @@ -160,7 +160,7 @@ To switch between backends: | Variable | Description | Default | |----------|-------------|---------| -| `HINDSIGHT_API_LLM_PROVIDER` | Provider: `openai`, `openai-codex`, `claude-code`, `anthropic`, `gemini`, `groq`, `minimax`, `ollama`, `lmstudio`, `vertexai`, `bedrock`, `litellm` | `openai` | +| `HINDSIGHT_API_LLM_PROVIDER` | Provider: `openai`, `openai-codex`, `claude-code`, `anthropic`, `gemini`, `groq`, `minimax`, `ollama`, `lmstudio`, `vertexai`, `bedrock`, `litellm`, `none` | `openai` | | `HINDSIGHT_API_LLM_API_KEY` | API key for LLM provider | - | | `HINDSIGHT_API_LLM_MODEL` | Model name | `gpt-5-mini` | | `HINDSIGHT_API_LLM_BASE_URL` | Custom LLM endpoint | Provider default | @@ -250,6 +250,13 @@ export HINDSIGHT_API_LLM_MODEL=azure/gpt-4o export HINDSIGHT_API_LLM_PROVIDER=litellm export HINDSIGHT_API_LLM_API_KEY=your-together-api-key export HINDSIGHT_API_LLM_MODEL=together_ai/meta-llama/Llama-3-70b-chat-hf + +# No LLM (chunk storage + semantic search only, no API key needed) +export HINDSIGHT_API_LLM_PROVIDER=none +# Retain automatically uses chunks mode (no fact extraction) +# Recall works normally (semantic search, BM25, graph retrieval) +# Reflect returns HTTP 400 (requires an LLM) +# Consolidation/observations are disabled ``` :::tip OpenAI Codex, Claude Code & Vertex AI Setup