diff --git a/hindsight-api/hindsight_api/config.py b/hindsight-api/hindsight_api/config.py index 5d636cb6..e75c9e13 100644 --- a/hindsight-api/hindsight_api/config.py +++ b/hindsight-api/hindsight_api/config.py @@ -26,6 +26,9 @@ ENV_LLM_API_KEY = "HINDSIGHT_API_LLM_API_KEY" ENV_LLM_MODEL = "HINDSIGHT_API_LLM_MODEL" ENV_LLM_BASE_URL = "HINDSIGHT_API_LLM_BASE_URL" ENV_LLM_MAX_CONCURRENT = "HINDSIGHT_API_LLM_MAX_CONCURRENT" +ENV_LLM_MAX_RETRIES = "HINDSIGHT_API_LLM_MAX_RETRIES" +ENV_LLM_INITIAL_BACKOFF = "HINDSIGHT_API_LLM_INITIAL_BACKOFF" +ENV_LLM_MAX_BACKOFF = "HINDSIGHT_API_LLM_MAX_BACKOFF" ENV_LLM_TIMEOUT = "HINDSIGHT_API_LLM_TIMEOUT" ENV_LLM_GROQ_SERVICE_TIER = "HINDSIGHT_API_LLM_GROQ_SERVICE_TIER" @@ -34,16 +37,31 @@ ENV_RETAIN_LLM_PROVIDER = "HINDSIGHT_API_RETAIN_LLM_PROVIDER" ENV_RETAIN_LLM_API_KEY = "HINDSIGHT_API_RETAIN_LLM_API_KEY" ENV_RETAIN_LLM_MODEL = "HINDSIGHT_API_RETAIN_LLM_MODEL" ENV_RETAIN_LLM_BASE_URL = "HINDSIGHT_API_RETAIN_LLM_BASE_URL" +ENV_RETAIN_LLM_MAX_CONCURRENT = "HINDSIGHT_API_RETAIN_LLM_MAX_CONCURRENT" +ENV_RETAIN_LLM_MAX_RETRIES = "HINDSIGHT_API_RETAIN_LLM_MAX_RETRIES" +ENV_RETAIN_LLM_INITIAL_BACKOFF = "HINDSIGHT_API_RETAIN_LLM_INITIAL_BACKOFF" +ENV_RETAIN_LLM_MAX_BACKOFF = "HINDSIGHT_API_RETAIN_LLM_MAX_BACKOFF" +ENV_RETAIN_LLM_TIMEOUT = "HINDSIGHT_API_RETAIN_LLM_TIMEOUT" ENV_REFLECT_LLM_PROVIDER = "HINDSIGHT_API_REFLECT_LLM_PROVIDER" ENV_REFLECT_LLM_API_KEY = "HINDSIGHT_API_REFLECT_LLM_API_KEY" ENV_REFLECT_LLM_MODEL = "HINDSIGHT_API_REFLECT_LLM_MODEL" ENV_REFLECT_LLM_BASE_URL = "HINDSIGHT_API_REFLECT_LLM_BASE_URL" +ENV_REFLECT_LLM_MAX_CONCURRENT = "HINDSIGHT_API_REFLECT_LLM_MAX_CONCURRENT" +ENV_REFLECT_LLM_MAX_RETRIES = "HINDSIGHT_API_REFLECT_LLM_MAX_RETRIES" +ENV_REFLECT_LLM_INITIAL_BACKOFF = "HINDSIGHT_API_REFLECT_LLM_INITIAL_BACKOFF" +ENV_REFLECT_LLM_MAX_BACKOFF = "HINDSIGHT_API_REFLECT_LLM_MAX_BACKOFF" +ENV_REFLECT_LLM_TIMEOUT = "HINDSIGHT_API_REFLECT_LLM_TIMEOUT" ENV_CONSOLIDATION_LLM_PROVIDER = "HINDSIGHT_API_CONSOLIDATION_LLM_PROVIDER" ENV_CONSOLIDATION_LLM_API_KEY = "HINDSIGHT_API_CONSOLIDATION_LLM_API_KEY" ENV_CONSOLIDATION_LLM_MODEL = "HINDSIGHT_API_CONSOLIDATION_LLM_MODEL" ENV_CONSOLIDATION_LLM_BASE_URL = "HINDSIGHT_API_CONSOLIDATION_LLM_BASE_URL" +ENV_CONSOLIDATION_LLM_MAX_CONCURRENT = "HINDSIGHT_API_CONSOLIDATION_LLM_MAX_CONCURRENT" +ENV_CONSOLIDATION_LLM_MAX_RETRIES = "HINDSIGHT_API_CONSOLIDATION_LLM_MAX_RETRIES" +ENV_CONSOLIDATION_LLM_INITIAL_BACKOFF = "HINDSIGHT_API_CONSOLIDATION_LLM_INITIAL_BACKOFF" +ENV_CONSOLIDATION_LLM_MAX_BACKOFF = "HINDSIGHT_API_CONSOLIDATION_LLM_MAX_BACKOFF" +ENV_CONSOLIDATION_LLM_TIMEOUT = "HINDSIGHT_API_CONSOLIDATION_LLM_TIMEOUT" ENV_EMBEDDINGS_PROVIDER = "HINDSIGHT_API_EMBEDDINGS_PROVIDER" ENV_EMBEDDINGS_LOCAL_MODEL = "HINDSIGHT_API_EMBEDDINGS_LOCAL_MODEL" @@ -133,6 +151,9 @@ DEFAULT_DATABASE_SCHEMA = "public" DEFAULT_LLM_PROVIDER = "openai" DEFAULT_LLM_MODEL = "gpt-5-mini" DEFAULT_LLM_MAX_CONCURRENT = 32 +DEFAULT_LLM_MAX_RETRIES = 10 # Max retry attempts for LLM API calls +DEFAULT_LLM_INITIAL_BACKOFF = 1.0 # Initial backoff in seconds for retry exponential backoff +DEFAULT_LLM_MAX_BACKOFF = 60.0 # Max backoff cap in seconds for retry exponential backoff DEFAULT_LLM_TIMEOUT = 120.0 # seconds DEFAULT_EMBEDDINGS_PROVIDER = "local" @@ -286,6 +307,9 @@ class HindsightConfig: llm_model: str llm_base_url: str | None llm_max_concurrent: int + llm_max_retries: int + llm_initial_backoff: float + llm_max_backoff: float llm_timeout: float # Per-operation LLM configuration (None = use default LLM config) @@ -293,16 +317,31 @@ class HindsightConfig: retain_llm_api_key: str | None retain_llm_model: str | None retain_llm_base_url: str | None + retain_llm_max_concurrent: int | None + retain_llm_max_retries: int | None + retain_llm_initial_backoff: float | None + retain_llm_max_backoff: float | None + retain_llm_timeout: float | None reflect_llm_provider: str | None reflect_llm_api_key: str | None reflect_llm_model: str | None reflect_llm_base_url: str | None + reflect_llm_max_concurrent: int | None + reflect_llm_max_retries: int | None + reflect_llm_initial_backoff: float | None + reflect_llm_max_backoff: float | None + reflect_llm_timeout: float | None consolidation_llm_provider: str | None consolidation_llm_api_key: str | None consolidation_llm_model: str | None consolidation_llm_base_url: str | None + consolidation_llm_max_concurrent: int | None + consolidation_llm_max_retries: int | None + consolidation_llm_initial_backoff: float | None + consolidation_llm_max_backoff: float | None + consolidation_llm_timeout: float | None # Embeddings embeddings_provider: str @@ -387,20 +426,66 @@ class HindsightConfig: llm_model=os.getenv(ENV_LLM_MODEL, DEFAULT_LLM_MODEL), llm_base_url=os.getenv(ENV_LLM_BASE_URL) or None, llm_max_concurrent=int(os.getenv(ENV_LLM_MAX_CONCURRENT, str(DEFAULT_LLM_MAX_CONCURRENT))), + llm_max_retries=int(os.getenv(ENV_LLM_MAX_RETRIES, str(DEFAULT_LLM_MAX_RETRIES))), + llm_initial_backoff=float(os.getenv(ENV_LLM_INITIAL_BACKOFF, str(DEFAULT_LLM_INITIAL_BACKOFF))), + llm_max_backoff=float(os.getenv(ENV_LLM_MAX_BACKOFF, str(DEFAULT_LLM_MAX_BACKOFF))), llm_timeout=float(os.getenv(ENV_LLM_TIMEOUT, str(DEFAULT_LLM_TIMEOUT))), # Per-operation LLM config (None = use default) retain_llm_provider=os.getenv(ENV_RETAIN_LLM_PROVIDER) or None, retain_llm_api_key=os.getenv(ENV_RETAIN_LLM_API_KEY) or None, retain_llm_model=os.getenv(ENV_RETAIN_LLM_MODEL) or None, retain_llm_base_url=os.getenv(ENV_RETAIN_LLM_BASE_URL) or None, + retain_llm_max_concurrent=int(os.getenv(ENV_RETAIN_LLM_MAX_CONCURRENT)) + if os.getenv(ENV_RETAIN_LLM_MAX_CONCURRENT) + else None, + retain_llm_max_retries=int(os.getenv(ENV_RETAIN_LLM_MAX_RETRIES)) + if os.getenv(ENV_RETAIN_LLM_MAX_RETRIES) + else None, + retain_llm_initial_backoff=float(os.getenv(ENV_RETAIN_LLM_INITIAL_BACKOFF)) + if os.getenv(ENV_RETAIN_LLM_INITIAL_BACKOFF) + else None, + retain_llm_max_backoff=float(os.getenv(ENV_RETAIN_LLM_MAX_BACKOFF)) + if os.getenv(ENV_RETAIN_LLM_MAX_BACKOFF) + else None, + retain_llm_timeout=float(os.getenv(ENV_RETAIN_LLM_TIMEOUT)) if os.getenv(ENV_RETAIN_LLM_TIMEOUT) else None, reflect_llm_provider=os.getenv(ENV_REFLECT_LLM_PROVIDER) or None, reflect_llm_api_key=os.getenv(ENV_REFLECT_LLM_API_KEY) or None, reflect_llm_model=os.getenv(ENV_REFLECT_LLM_MODEL) or None, reflect_llm_base_url=os.getenv(ENV_REFLECT_LLM_BASE_URL) or None, + reflect_llm_max_concurrent=int(os.getenv(ENV_REFLECT_LLM_MAX_CONCURRENT)) + if os.getenv(ENV_REFLECT_LLM_MAX_CONCURRENT) + else None, + reflect_llm_max_retries=int(os.getenv(ENV_REFLECT_LLM_MAX_RETRIES)) + if os.getenv(ENV_REFLECT_LLM_MAX_RETRIES) + else None, + reflect_llm_initial_backoff=float(os.getenv(ENV_REFLECT_LLM_INITIAL_BACKOFF)) + if os.getenv(ENV_REFLECT_LLM_INITIAL_BACKOFF) + else None, + reflect_llm_max_backoff=float(os.getenv(ENV_REFLECT_LLM_MAX_BACKOFF)) + if os.getenv(ENV_REFLECT_LLM_MAX_BACKOFF) + else None, + reflect_llm_timeout=float(os.getenv(ENV_REFLECT_LLM_TIMEOUT)) + if os.getenv(ENV_REFLECT_LLM_TIMEOUT) + else None, consolidation_llm_provider=os.getenv(ENV_CONSOLIDATION_LLM_PROVIDER) or None, consolidation_llm_api_key=os.getenv(ENV_CONSOLIDATION_LLM_API_KEY) or None, consolidation_llm_model=os.getenv(ENV_CONSOLIDATION_LLM_MODEL) or None, consolidation_llm_base_url=os.getenv(ENV_CONSOLIDATION_LLM_BASE_URL) or None, + consolidation_llm_max_concurrent=int(os.getenv(ENV_CONSOLIDATION_LLM_MAX_CONCURRENT)) + if os.getenv(ENV_CONSOLIDATION_LLM_MAX_CONCURRENT) + else None, + consolidation_llm_max_retries=int(os.getenv(ENV_CONSOLIDATION_LLM_MAX_RETRIES)) + if os.getenv(ENV_CONSOLIDATION_LLM_MAX_RETRIES) + else None, + consolidation_llm_initial_backoff=float(os.getenv(ENV_CONSOLIDATION_LLM_INITIAL_BACKOFF)) + if os.getenv(ENV_CONSOLIDATION_LLM_INITIAL_BACKOFF) + else None, + consolidation_llm_max_backoff=float(os.getenv(ENV_CONSOLIDATION_LLM_MAX_BACKOFF)) + if os.getenv(ENV_CONSOLIDATION_LLM_MAX_BACKOFF) + else None, + consolidation_llm_timeout=float(os.getenv(ENV_CONSOLIDATION_LLM_TIMEOUT)) + if os.getenv(ENV_CONSOLIDATION_LLM_TIMEOUT) + else None, # Embeddings embeddings_provider=os.getenv(ENV_EMBEDDINGS_PROVIDER, DEFAULT_EMBEDDINGS_PROVIDER), embeddings_local_model=os.getenv(ENV_EMBEDDINGS_LOCAL_MODEL, DEFAULT_EMBEDDINGS_LOCAL_MODEL), diff --git a/hindsight-api/hindsight_api/engine/retain/fact_extraction.py b/hindsight-api/hindsight_api/engine/retain/fact_extraction.py index 95b0b33e..32bdb5da 100644 --- a/hindsight-api/hindsight_api/engine/retain/fact_extraction.py +++ b/hindsight-api/hindsight_api/engine/retain/fact_extraction.py @@ -782,12 +782,28 @@ Text: usage = TokenUsage() # Track cumulative usage across retries for attempt in range(max_retries): try: + # Use retain-specific overrides if set, otherwise fall back to global LLM config + max_retries = ( + config.retain_llm_max_retries if config.retain_llm_max_retries is not None else config.llm_max_retries + ) + initial_backoff = ( + config.retain_llm_initial_backoff + if config.retain_llm_initial_backoff is not None + else config.llm_initial_backoff + ) + max_backoff = ( + config.retain_llm_max_backoff if config.retain_llm_max_backoff is not None else config.llm_max_backoff + ) + extraction_response_json, call_usage = await llm_config.call( messages=[{"role": "system", "content": prompt}, {"role": "user", "content": user_message}], response_format=response_schema, scope="memory_extract_facts", temperature=0.1, max_completion_tokens=config.retain_max_completion_tokens, + max_retries=max_retries, + initial_backoff=initial_backoff, + max_backoff=max_backoff, skip_validation=True, # Get raw JSON, we'll validate leniently return_usage=True, ) diff --git a/hindsight-api/hindsight_api/main.py b/hindsight-api/hindsight_api/main.py index 3262064d..86f2375a 100644 --- a/hindsight-api/hindsight_api/main.py +++ b/hindsight-api/hindsight_api/main.py @@ -176,19 +176,37 @@ def main(): llm_model=config.llm_model, llm_base_url=config.llm_base_url, llm_max_concurrent=config.llm_max_concurrent, + llm_max_retries=config.llm_max_retries, + llm_initial_backoff=config.llm_initial_backoff, + llm_max_backoff=config.llm_max_backoff, llm_timeout=config.llm_timeout, retain_llm_provider=config.retain_llm_provider, retain_llm_api_key=config.retain_llm_api_key, retain_llm_model=config.retain_llm_model, retain_llm_base_url=config.retain_llm_base_url, + retain_llm_max_concurrent=config.retain_llm_max_concurrent, + retain_llm_max_retries=config.retain_llm_max_retries, + retain_llm_initial_backoff=config.retain_llm_initial_backoff, + retain_llm_max_backoff=config.retain_llm_max_backoff, + retain_llm_timeout=config.retain_llm_timeout, reflect_llm_provider=config.reflect_llm_provider, reflect_llm_api_key=config.reflect_llm_api_key, reflect_llm_model=config.reflect_llm_model, reflect_llm_base_url=config.reflect_llm_base_url, + reflect_llm_max_concurrent=config.reflect_llm_max_concurrent, + reflect_llm_max_retries=config.reflect_llm_max_retries, + reflect_llm_initial_backoff=config.reflect_llm_initial_backoff, + reflect_llm_max_backoff=config.reflect_llm_max_backoff, + reflect_llm_timeout=config.reflect_llm_timeout, consolidation_llm_provider=config.consolidation_llm_provider, consolidation_llm_api_key=config.consolidation_llm_api_key, consolidation_llm_model=config.consolidation_llm_model, consolidation_llm_base_url=config.consolidation_llm_base_url, + consolidation_llm_max_concurrent=config.consolidation_llm_max_concurrent, + consolidation_llm_max_retries=config.consolidation_llm_max_retries, + consolidation_llm_initial_backoff=config.consolidation_llm_initial_backoff, + consolidation_llm_max_backoff=config.consolidation_llm_max_backoff, + consolidation_llm_timeout=config.consolidation_llm_timeout, embeddings_provider=config.embeddings_provider, embeddings_local_model=config.embeddings_local_model, embeddings_local_force_cpu=config.embeddings_local_force_cpu, diff --git a/hindsight-api/tests/test_per_operation_llm_config.py b/hindsight-api/tests/test_per_operation_llm_config.py index c1a37517..ba25ca01 100644 --- a/hindsight-api/tests/test_per_operation_llm_config.py +++ b/hindsight-api/tests/test_per_operation_llm_config.py @@ -275,3 +275,88 @@ class TestReflectUsesReflectLLMConfig: # Verify it's different from the retain config assert engine._reflect_llm_config.model != engine._retain_llm_config.model + + +class TestRetryAndBackoffConfiguration: + """Test retry and backoff configuration options.""" + + def test_global_retry_backoff_config_defaults(self): + """Test that global retry/backoff settings have correct defaults.""" + from hindsight_api.config import get_config + + config = get_config() + + # Verify global defaults + assert config.llm_max_retries == 10 + assert config.llm_initial_backoff == 1.0 + assert config.llm_max_backoff == 60.0 + + def test_per_operation_retry_backoff_config_from_env(self): + """Test that per-operation retry/backoff settings are loaded from environment.""" + from hindsight_api.config import clear_config_cache + + # Set per-operation overrides + os.environ["HINDSIGHT_API_RETAIN_LLM_MAX_RETRIES"] = "3" + os.environ["HINDSIGHT_API_RETAIN_LLM_INITIAL_BACKOFF"] = "2.0" + os.environ["HINDSIGHT_API_RETAIN_LLM_MAX_BACKOFF"] = "120.0" + os.environ["HINDSIGHT_API_REFLECT_LLM_MAX_RETRIES"] = "5" + os.environ["HINDSIGHT_API_REFLECT_LLM_INITIAL_BACKOFF"] = "1.5" + os.environ["HINDSIGHT_API_REFLECT_LLM_MAX_BACKOFF"] = "90.0" + + try: + clear_config_cache() + from hindsight_api.config import get_config + + config = get_config() + + # Verify retain overrides + assert config.retain_llm_max_retries == 3 + assert config.retain_llm_initial_backoff == 2.0 + assert config.retain_llm_max_backoff == 120.0 + + # Verify reflect overrides + assert config.reflect_llm_max_retries == 5 + assert config.reflect_llm_initial_backoff == 1.5 + assert config.reflect_llm_max_backoff == 90.0 + + # Verify global defaults remain unchanged + assert config.llm_max_retries == 10 + assert config.llm_initial_backoff == 1.0 + assert config.llm_max_backoff == 60.0 + finally: + # Clean up + os.environ.pop("HINDSIGHT_API_RETAIN_LLM_MAX_RETRIES", None) + os.environ.pop("HINDSIGHT_API_RETAIN_LLM_INITIAL_BACKOFF", None) + os.environ.pop("HINDSIGHT_API_RETAIN_LLM_MAX_BACKOFF", None) + os.environ.pop("HINDSIGHT_API_REFLECT_LLM_MAX_RETRIES", None) + os.environ.pop("HINDSIGHT_API_REFLECT_LLM_INITIAL_BACKOFF", None) + os.environ.pop("HINDSIGHT_API_REFLECT_LLM_MAX_BACKOFF", None) + clear_config_cache() + + def test_per_operation_retry_backoff_fallback_to_global(self): + """Test that per-operation settings fall back to global when not set.""" + from hindsight_api.config import clear_config_cache, get_config + + # Set only global values + os.environ["HINDSIGHT_API_LLM_MAX_RETRIES"] = "7" + os.environ["HINDSIGHT_API_LLM_INITIAL_BACKOFF"] = "3.0" + os.environ["HINDSIGHT_API_LLM_MAX_BACKOFF"] = "180.0" + + try: + clear_config_cache() + config = get_config() + + # Per-operation should be None (will fall back to global at runtime) + assert config.retain_llm_max_retries is None + assert config.retain_llm_initial_backoff is None + assert config.retain_llm_max_backoff is None + + # Global values should be set + assert config.llm_max_retries == 7 + assert config.llm_initial_backoff == 3.0 + assert config.llm_max_backoff == 180.0 + finally: + os.environ.pop("HINDSIGHT_API_LLM_MAX_RETRIES", None) + os.environ.pop("HINDSIGHT_API_LLM_INITIAL_BACKOFF", None) + os.environ.pop("HINDSIGHT_API_LLM_MAX_BACKOFF", None) + clear_config_cache() diff --git a/hindsight-docs/docs/developer/configuration.md b/hindsight-docs/docs/developer/configuration.md index f646413e..92c4fb02 100644 --- a/hindsight-docs/docs/developer/configuration.md +++ b/hindsight-docs/docs/developer/configuration.md @@ -66,6 +66,9 @@ hindsight-admin run-db-migration --schema tenant_acme | `HINDSIGHT_API_LLM_MODEL` | Model name | `gpt-5-mini` | | `HINDSIGHT_API_LLM_BASE_URL` | Custom LLM endpoint | Provider default | | `HINDSIGHT_API_LLM_MAX_CONCURRENT` | Max concurrent LLM requests | `32` | +| `HINDSIGHT_API_LLM_MAX_RETRIES` | Max retry attempts for LLM API calls | `10` | +| `HINDSIGHT_API_LLM_INITIAL_BACKOFF` | Initial retry backoff in seconds (exponential backoff) | `1.0` | +| `HINDSIGHT_API_LLM_MAX_BACKOFF` | Max retry backoff cap in seconds | `60.0` | | `HINDSIGHT_API_LLM_TIMEOUT` | LLM request timeout in seconds | `120` | | `HINDSIGHT_API_LLM_GROQ_SERVICE_TIER` | Groq service tier: `on_demand`, `flex`, `auto` | `auto` | @@ -121,14 +124,29 @@ Different memory operations have different requirements. **Retain** (fact extrac | `HINDSIGHT_API_RETAIN_LLM_API_KEY` | API key for retain LLM | Falls back to `HINDSIGHT_API_LLM_API_KEY` | | `HINDSIGHT_API_RETAIN_LLM_MODEL` | Model for retain operations | Falls back to `HINDSIGHT_API_LLM_MODEL` | | `HINDSIGHT_API_RETAIN_LLM_BASE_URL` | Base URL for retain LLM | Falls back to `HINDSIGHT_API_LLM_BASE_URL` | +| `HINDSIGHT_API_RETAIN_LLM_MAX_CONCURRENT` | Max concurrent requests for retain | Falls back to `HINDSIGHT_API_LLM_MAX_CONCURRENT` | +| `HINDSIGHT_API_RETAIN_LLM_MAX_RETRIES` | Max retries for retain | Falls back to `HINDSIGHT_API_LLM_MAX_RETRIES` | +| `HINDSIGHT_API_RETAIN_LLM_INITIAL_BACKOFF` | Initial backoff for retain retries (seconds) | Falls back to `HINDSIGHT_API_LLM_INITIAL_BACKOFF` | +| `HINDSIGHT_API_RETAIN_LLM_MAX_BACKOFF` | Max backoff cap for retain retries (seconds) | Falls back to `HINDSIGHT_API_LLM_MAX_BACKOFF` | +| `HINDSIGHT_API_RETAIN_LLM_TIMEOUT` | Timeout for retain requests (seconds) | Falls back to `HINDSIGHT_API_LLM_TIMEOUT` | | `HINDSIGHT_API_REFLECT_LLM_PROVIDER` | LLM provider for reflect operations | Falls back to `HINDSIGHT_API_LLM_PROVIDER` | | `HINDSIGHT_API_REFLECT_LLM_API_KEY` | API key for reflect LLM | Falls back to `HINDSIGHT_API_LLM_API_KEY` | | `HINDSIGHT_API_REFLECT_LLM_MODEL` | Model for reflect operations | Falls back to `HINDSIGHT_API_LLM_MODEL` | | `HINDSIGHT_API_REFLECT_LLM_BASE_URL` | Base URL for reflect LLM | Falls back to `HINDSIGHT_API_LLM_BASE_URL` | +| `HINDSIGHT_API_REFLECT_LLM_MAX_CONCURRENT` | Max concurrent requests for reflect | Falls back to `HINDSIGHT_API_LLM_MAX_CONCURRENT` | +| `HINDSIGHT_API_REFLECT_LLM_MAX_RETRIES` | Max retries for reflect | Falls back to `HINDSIGHT_API_LLM_MAX_RETRIES` | +| `HINDSIGHT_API_REFLECT_LLM_INITIAL_BACKOFF` | Initial backoff for reflect retries (seconds) | Falls back to `HINDSIGHT_API_LLM_INITIAL_BACKOFF` | +| `HINDSIGHT_API_REFLECT_LLM_MAX_BACKOFF` | Max backoff cap for reflect retries (seconds) | Falls back to `HINDSIGHT_API_LLM_MAX_BACKOFF` | +| `HINDSIGHT_API_REFLECT_LLM_TIMEOUT` | Timeout for reflect requests (seconds) | Falls back to `HINDSIGHT_API_LLM_TIMEOUT` | | `HINDSIGHT_API_CONSOLIDATION_LLM_PROVIDER` | LLM provider for observation consolidation | Falls back to `HINDSIGHT_API_LLM_PROVIDER` | | `HINDSIGHT_API_CONSOLIDATION_LLM_API_KEY` | API key for consolidation LLM | Falls back to `HINDSIGHT_API_LLM_API_KEY` | | `HINDSIGHT_API_CONSOLIDATION_LLM_MODEL` | Model for consolidation operations | Falls back to `HINDSIGHT_API_LLM_MODEL` | | `HINDSIGHT_API_CONSOLIDATION_LLM_BASE_URL` | Base URL for consolidation LLM | Falls back to `HINDSIGHT_API_LLM_BASE_URL` | +| `HINDSIGHT_API_CONSOLIDATION_LLM_MAX_CONCURRENT` | Max concurrent requests for consolidation | Falls back to `HINDSIGHT_API_LLM_MAX_CONCURRENT` | +| `HINDSIGHT_API_CONSOLIDATION_LLM_MAX_RETRIES` | Max retries for consolidation | Falls back to `HINDSIGHT_API_LLM_MAX_RETRIES` | +| `HINDSIGHT_API_CONSOLIDATION_LLM_INITIAL_BACKOFF` | Initial backoff for consolidation retries (seconds) | Falls back to `HINDSIGHT_API_LLM_INITIAL_BACKOFF` | +| `HINDSIGHT_API_CONSOLIDATION_LLM_MAX_BACKOFF` | Max backoff cap for consolidation retries (seconds) | Falls back to `HINDSIGHT_API_LLM_MAX_BACKOFF` | +| `HINDSIGHT_API_CONSOLIDATION_LLM_TIMEOUT` | Timeout for consolidation requests (seconds) | Falls back to `HINDSIGHT_API_LLM_TIMEOUT` | :::tip When to Use Per-Operation Config - **Retain**: Use models with strong structured output (e.g., GPT-4o, Claude) for accurate fact extraction @@ -153,6 +171,25 @@ export HINDSIGHT_API_REFLECT_LLM_API_KEY=gsk_xxxxxxxxxxxx export HINDSIGHT_API_REFLECT_LLM_MODEL=llama-3.3-70b-versatile ``` +**Example: Tuning Retry Behavior for Rate-Limited APIs** + +```bash +# For Anthropic with tight rate limits (10k output tokens/minute) +export HINDSIGHT_API_LLM_PROVIDER=anthropic +export HINDSIGHT_API_LLM_API_KEY=sk-ant-xxxxxxxxxxxx +export HINDSIGHT_API_LLM_MODEL=claude-sonnet-4-20250514 + +# Reduce concurrent requests for retain to avoid rate limits +export HINDSIGHT_API_RETAIN_LLM_MAX_CONCURRENT=3 + +# Fail faster with fewer retries +export HINDSIGHT_API_RETAIN_LLM_MAX_RETRIES=3 + +# Or increase backoff times to wait out rate limit windows +export HINDSIGHT_API_RETAIN_LLM_INITIAL_BACKOFF=2.0 # Start at 2s instead of 1s +export HINDSIGHT_API_RETAIN_LLM_MAX_BACKOFF=120.0 # Cap at 2min instead of 1min +``` + ### Embeddings | Variable | Description | Default |