diff --git a/hindsight-api/hindsight_api/config.py b/hindsight-api/hindsight_api/config.py index c999f9dd..6b5b9ef6 100644 --- a/hindsight-api/hindsight_api/config.py +++ b/hindsight-api/hindsight_api/config.py @@ -447,6 +447,22 @@ class HindsightConfig: # Reflect agent settings reflect_max_iterations: int + def validate(self) -> None: + """Validate configuration values and raise errors for invalid combinations.""" + # RETAIN_MAX_COMPLETION_TOKENS must be greater than RETAIN_CHUNK_SIZE + # to ensure the LLM has enough output capacity to extract facts from chunks + if self.retain_max_completion_tokens <= self.retain_chunk_size: + raise ValueError( + f"Invalid configuration: HINDSIGHT_API_RETAIN_MAX_COMPLETION_TOKENS " + f"({self.retain_max_completion_tokens}) must be greater than " + f"HINDSIGHT_API_RETAIN_CHUNK_SIZE ({self.retain_chunk_size}). " + f"\n\nYou have two options to fix this:" + f"\n 1. Increase HINDSIGHT_API_RETAIN_MAX_COMPLETION_TOKENS to a value > {self.retain_chunk_size}" + f"\n 2. Use a model that supports at least {self.retain_max_completion_tokens} output tokens" + f"\n (current model: {self.retain_llm_model or self.llm_model}, " + f"provider: {self.retain_llm_provider or self.llm_provider})" + ) + @classmethod def from_env(cls) -> "HindsightConfig": """Create configuration from environment variables.""" @@ -454,7 +470,7 @@ class HindsightConfig: llm_provider = os.getenv(ENV_LLM_PROVIDER, DEFAULT_LLM_PROVIDER) llm_model = os.getenv(ENV_LLM_MODEL) or _get_default_model_for_provider(llm_provider) - return cls( + config = cls( # Database database_url=os.getenv(ENV_DATABASE_URL, DEFAULT_DATABASE_URL), database_schema=os.getenv(ENV_DATABASE_SCHEMA, DEFAULT_DATABASE_SCHEMA), @@ -631,6 +647,8 @@ class HindsightConfig: # Reflect agent settings reflect_max_iterations=int(os.getenv(ENV_REFLECT_MAX_ITERATIONS, str(DEFAULT_REFLECT_MAX_ITERATIONS))), ) + config.validate() + return config def get_llm_base_url(self) -> str: """Get the LLM base URL, with provider-specific defaults.""" diff --git a/hindsight-api/hindsight_api/engine/providers/mock_llm.py b/hindsight-api/hindsight_api/engine/providers/mock_llm.py index 7d02b346..1a31ebeb 100644 --- a/hindsight-api/hindsight_api/engine/providers/mock_llm.py +++ b/hindsight-api/hindsight_api/engine/providers/mock_llm.py @@ -65,6 +65,7 @@ class MockLLM(LLMInterface): # Storage for test verification self._mock_calls: list[dict] = [] self._mock_response: Any = None + self._mock_exception: Exception | None = None async def verify_connection(self) -> None: """ @@ -124,6 +125,10 @@ class MockLLM(LLMInterface): self._mock_calls.append(call_record) logger.debug(f"Mock LLM call recorded: scope={scope}, model={self.model}") + # Raise mock exception if configured + if self._mock_exception is not None: + raise self._mock_exception + # Return mock response if self._mock_response is not None: result = self._mock_response @@ -183,6 +188,10 @@ class MockLLM(LLMInterface): } self._mock_calls.append(call_record) + # Raise mock exception if configured + if self._mock_exception is not None: + raise self._mock_exception + if self._mock_response is not None: if isinstance(self._mock_response, LLMToolCallResult): return self._mock_response @@ -215,6 +224,16 @@ class MockLLM(LLMInterface): """ self._mock_response = response + def set_mock_exception(self, exception: Exception) -> None: + """ + Set an exception to raise from mock calls. + + Args: + exception: The exception to raise on the next call. + After raising, the exception is cleared. + """ + self._mock_exception = exception + def get_mock_calls(self) -> list[dict]: """ Get the list of recorded mock calls. @@ -230,5 +249,6 @@ class MockLLM(LLMInterface): return self._mock_calls def clear_mock_calls(self) -> None: - """Clear the recorded mock calls.""" + """Clear the recorded mock calls and any set exception.""" self._mock_calls = [] + self._mock_exception = None diff --git a/hindsight-api/hindsight_api/engine/retain/fact_extraction.py b/hindsight-api/hindsight_api/engine/retain/fact_extraction.py index 5046c23f..b2248e29 100644 --- a/hindsight-api/hindsight_api/engine/retain/fact_extraction.py +++ b/hindsight-api/hindsight_api/engine/retain/fact_extraction.py @@ -1011,6 +1011,29 @@ Text: except BadRequestError as e: last_error = e + error_str = str(e).lower() + + # Check if error is related to max_tokens/completion_tokens not being supported + if any( + keyword in error_str + for keyword in [ + "max_tokens", + "max_completion_tokens", + "maximum context", + "token limit", + "context length", + ] + ): + # Provide helpful error message with configuration suggestions + raise ValueError( + f"Model does not support the required output token limit.\n\n" + f"The model '{llm_config.model}' (provider: {llm_config.provider}) failed with: {e}\n\n" + f"You have two options to fix this:\n" + f" 1. Use a different model that supports at least {config.retain_max_completion_tokens} output tokens\n" + f" 2. Decrease HINDSIGHT_API_RETAIN_MAX_COMPLETION_TOKENS to a value your model supports\n" + f" (current value: {config.retain_max_completion_tokens}, must be > RETAIN_CHUNK_SIZE={config.retain_chunk_size})" + ) from e + if "json_validate_failed" in str(e): logger.warning( f" [1.3.{chunk_index + 1}] Attempt {attempt + 1}/{max_retries} failed with JSON validation error: {e}" diff --git a/hindsight-docs/docs/developer/models.md b/hindsight-docs/docs/developer/models.md index cbb94bc5..5afd4eb4 100644 --- a/hindsight-docs/docs/developer/models.md +++ b/hindsight-docs/docs/developer/models.md @@ -91,6 +91,20 @@ export HINDSIGHT_API_RETAIN_LLM_PROVIDER=anthropic Other LLM models not listed above may work with Hindsight, but they must support **at least 65,000 output tokens** to ensure reliable fact extraction. If you need support for a specific model that doesn't meet this requirement, please [open an issue](https://github.com/hindsight-ai/hindsight/issues) to request an exception. +:::tip Models with Limited Output Tokens +If your model only supports 32k or fewer output tokens (e.g., some older models), you can reduce the retain completion token limit: + +```bash +# For models that support 32k output tokens +export HINDSIGHT_API_RETAIN_MAX_COMPLETION_TOKENS=32000 + +# For models that support 16k output tokens +export HINDSIGHT_API_RETAIN_MAX_COMPLETION_TOKENS=16000 +``` + +**Important:** `HINDSIGHT_API_RETAIN_MAX_COMPLETION_TOKENS` must be greater than `HINDSIGHT_API_RETAIN_CHUNK_SIZE` (default: 3000). The system will validate this on startup and provide an error message if the configuration is invalid. +::: + ### Configuration ```bash @@ -175,20 +189,40 @@ You can use any model supported by OpenAI Codex CLI - Usage is billed to your ChatGPT subscription (not separate API costs) - For personal development use only (see ChatGPT Terms of Service) -**Troubleshooting:** - -If authentication fails: -```bash -# Re-login to refresh tokens -codex auth login -``` - --- ### Claude Code Setup (Claude Pro/Max) Use your Claude Pro or Max subscription for Hindsight without separate Anthropic API costs. + +:::warning Terms of Service Notice + +This integration uses the Claude Agent SDK with your personal Claude Pro/Max subscription +credentials. You must be logged into Claude Code on your own machine before using this provider. + +**Please be aware:** + +- Anthropic's [Agent SDK documentation](https://docs.claude.com/en/api/agent-sdk/overview) + states that third-party developers should not offer claude.ai login or rate limits for + their products. Hindsight does **not** perform any login on your behalf — it uses + credentials you've already authenticated via `claude auth login`. +- In January 2026, Anthropic [enforced restrictions](https://paddo.dev/blog/anthropic-walled-garden-crackdown/) + against third-party tools using Claude subscription OAuth tokens. Those restrictions + targeted tools that **spoofed the Claude Code client identity** — Hindsight uses the + official Claude Agent SDK instead. +- This provider is intended for **local, personal development use only**. Do not use it + in production deployments or shared environments. +- Anthropic's terms may change. If you want guaranteed compliance, use the `anthropic` + provider with an API key instead. +- Usage counts against your Claude Pro/Max subscription limits. + +For production or team use, we recommend using `HINDSIGHT_API_LLM_PROVIDER=anthropic` with +an API key from the [Anthropic Console](https://console.anthropic.com/). + +::: + + **Prerequisites:** - Active Claude Pro or Max subscription - Claude Code CLI installed @@ -233,6 +267,7 @@ You can use any model supported by Claude Code CLI. - Usage billed to your Claude subscription (not separate API costs) - For personal development use only (see Claude Terms of Service) + --- ## Embedding Model diff --git a/hindsight-docs/versioned_docs/version-0.4/developer/installation.md b/hindsight-docs/versioned_docs/version-0.4/developer/installation.md index eea36344..983f8972 100644 --- a/hindsight-docs/versioned_docs/version-0.4/developer/installation.md +++ b/hindsight-docs/versioned_docs/version-0.4/developer/installation.md @@ -50,6 +50,81 @@ docker run --rm -it --pull always -p 8888:8888 -p 9999:9999 \ - **API Server**: http://localhost:8888 - **Control Plane** (Web UI): http://localhost:9999 +### Docker Image Variants + +Hindsight provides two image variants with different size/capability tradeoffs: + +| Variant | Size (AMD64) | Size (ARM64) | Use Case | +|---------|--------------|--------------|----------| +| **Full** (`latest`) | ~9 GB | ~3.7 GB | Includes local ML models (embeddings, reranking) | +| **Slim** (`slim`) | ~500 MB | ~500 MB | Requires external embedding/reranking providers | + +**Full image** (default): +```bash +docker run --rm -it -p 8888:8888 \ + -e HINDSIGHT_API_LLM_API_KEY=$OPENAI_API_KEY \ + ghcr.io/vectorize-io/hindsight:latest +``` +- ✅ Works out of the box with local ML models +- ✅ No additional services needed +- ❌ Larger image size (AMD64 includes CUDA libraries for GPU support) + +**Slim image**: +```bash +docker run --rm -it -p 8888:8888 \ + -e HINDSIGHT_API_LLM_API_KEY=$OPENAI_API_KEY \ + -e HINDSIGHT_API_EMBEDDINGS_PROVIDER=openai \ + -e HINDSIGHT_API_RERANKER_PROVIDER=cohere \ + -e HINDSIGHT_API_COHERE_API_KEY=$COHERE_API_KEY \ + ghcr.io/vectorize-io/hindsight:slim +``` +- ✅ Dramatically smaller image (~95% reduction on AMD64) +- ✅ Faster pull/deploy times +- ✅ Lower memory footprint +- ❌ Requires external embedding/reranking services (OpenAI, Cohere, TEI) + +**When to use slim:** +- Cloud deployments where image size matters +- Using managed embedding services (OpenAI, Cohere) +- Running on Text Embeddings Inference (TEI) infrastructure +- Kubernetes environments with fast pull requirements + +:::warning Slim Image Requires External Providers +If you run the slim image **without** setting external embedding providers, you'll see this error: + +``` +ImportError: sentence-transformers is required for LocalSTEmbeddings. +Install it with: pip install sentence-transformers +``` + +**Fix:** Always set embedding and reranking providers when using slim images: +```bash +-e HINDSIGHT_API_EMBEDDINGS_PROVIDER=openai +-e HINDSIGHT_API_EMBEDDINGS_OPENAI_API_KEY=sk-xxx +-e HINDSIGHT_API_RERANKER_PROVIDER=cohere +-e HINDSIGHT_API_COHERE_API_KEY=xxx +``` +::: + +See [Configuration](./configuration#embeddings-and-reranking) for all embedding provider options. + +### Available Tags + +```bash +# Standalone (API + Control Plane) +ghcr.io/vectorize-io/hindsight:latest # Full, latest release +ghcr.io/vectorize-io/hindsight:slim # Slim, latest release +ghcr.io/vectorize-io/hindsight:0.4.9 # Full, specific version +ghcr.io/vectorize-io/hindsight:0.4.9-slim # Slim, specific version + +# API only +ghcr.io/vectorize-io/hindsight-api:latest +ghcr.io/vectorize-io/hindsight-api:slim + +# Control Plane only +ghcr.io/vectorize-io/hindsight-control-plane:latest +``` + --- ## Helm / Kubernetes diff --git a/hindsight-docs/versioned_docs/version-0.4/developer/models.md b/hindsight-docs/versioned_docs/version-0.4/developer/models.md index cbb94bc5..5afd4eb4 100644 --- a/hindsight-docs/versioned_docs/version-0.4/developer/models.md +++ b/hindsight-docs/versioned_docs/version-0.4/developer/models.md @@ -91,6 +91,20 @@ export HINDSIGHT_API_RETAIN_LLM_PROVIDER=anthropic Other LLM models not listed above may work with Hindsight, but they must support **at least 65,000 output tokens** to ensure reliable fact extraction. If you need support for a specific model that doesn't meet this requirement, please [open an issue](https://github.com/hindsight-ai/hindsight/issues) to request an exception. +:::tip Models with Limited Output Tokens +If your model only supports 32k or fewer output tokens (e.g., some older models), you can reduce the retain completion token limit: + +```bash +# For models that support 32k output tokens +export HINDSIGHT_API_RETAIN_MAX_COMPLETION_TOKENS=32000 + +# For models that support 16k output tokens +export HINDSIGHT_API_RETAIN_MAX_COMPLETION_TOKENS=16000 +``` + +**Important:** `HINDSIGHT_API_RETAIN_MAX_COMPLETION_TOKENS` must be greater than `HINDSIGHT_API_RETAIN_CHUNK_SIZE` (default: 3000). The system will validate this on startup and provide an error message if the configuration is invalid. +::: + ### Configuration ```bash @@ -175,20 +189,40 @@ You can use any model supported by OpenAI Codex CLI - Usage is billed to your ChatGPT subscription (not separate API costs) - For personal development use only (see ChatGPT Terms of Service) -**Troubleshooting:** - -If authentication fails: -```bash -# Re-login to refresh tokens -codex auth login -``` - --- ### Claude Code Setup (Claude Pro/Max) Use your Claude Pro or Max subscription for Hindsight without separate Anthropic API costs. + +:::warning Terms of Service Notice + +This integration uses the Claude Agent SDK with your personal Claude Pro/Max subscription +credentials. You must be logged into Claude Code on your own machine before using this provider. + +**Please be aware:** + +- Anthropic's [Agent SDK documentation](https://docs.claude.com/en/api/agent-sdk/overview) + states that third-party developers should not offer claude.ai login or rate limits for + their products. Hindsight does **not** perform any login on your behalf — it uses + credentials you've already authenticated via `claude auth login`. +- In January 2026, Anthropic [enforced restrictions](https://paddo.dev/blog/anthropic-walled-garden-crackdown/) + against third-party tools using Claude subscription OAuth tokens. Those restrictions + targeted tools that **spoofed the Claude Code client identity** — Hindsight uses the + official Claude Agent SDK instead. +- This provider is intended for **local, personal development use only**. Do not use it + in production deployments or shared environments. +- Anthropic's terms may change. If you want guaranteed compliance, use the `anthropic` + provider with an API key instead. +- Usage counts against your Claude Pro/Max subscription limits. + +For production or team use, we recommend using `HINDSIGHT_API_LLM_PROVIDER=anthropic` with +an API key from the [Anthropic Console](https://console.anthropic.com/). + +::: + + **Prerequisites:** - Active Claude Pro or Max subscription - Claude Code CLI installed @@ -233,6 +267,7 @@ You can use any model supported by Claude Code CLI. - Usage billed to your Claude subscription (not separate API costs) - For personal development use only (see Claude Terms of Service) + --- ## Embedding Model