diff --git a/hindsight-api-slim/hindsight_api/config.py b/hindsight-api-slim/hindsight_api/config.py index c6365901..1f91e417 100644 --- a/hindsight-api-slim/hindsight_api/config.py +++ b/hindsight-api-slim/hindsight_api/config.py @@ -131,10 +131,12 @@ ENV_LLM_MAX_BACKOFF = "HINDSIGHT_API_LLM_MAX_BACKOFF" ENV_LLM_TIMEOUT = "HINDSIGHT_API_LLM_TIMEOUT" ENV_LLM_GROQ_SERVICE_TIER = "HINDSIGHT_API_LLM_GROQ_SERVICE_TIER" ENV_LLM_OPENAI_SERVICE_TIER = "HINDSIGHT_API_LLM_OPENAI_SERVICE_TIER" +ENV_LLM_EXTRA_BODY = "HINDSIGHT_API_LLM_EXTRA_BODY" # Defaults for service tiers DEFAULT_LLM_GROQ_SERVICE_TIER = "auto" # "on_demand", "flex", or "auto" DEFAULT_LLM_OPENAI_SERVICE_TIER = None # None (default) or "flex" (50% cheaper) +DEFAULT_LLM_EXTRA_BODY = None # None = no extra body params; JSON dict merged into OpenAI extra_body # Per-operation LLM configuration (optional, falls back to global LLM config) ENV_RETAIN_LLM_PROVIDER = "HINDSIGHT_API_RETAIN_LLM_PROVIDER" @@ -640,6 +642,9 @@ class HindsightConfig: llm_timeout: float llm_groq_service_tier: str # Groq: "on_demand", "flex", or "auto" llm_openai_service_tier: str | None # OpenAI: None (default) or "flex" (50% cheaper) + llm_extra_body: ( + dict | None + ) # Extra body params merged into OpenAI-compatible API calls (e.g. {"chat_template_kwargs": {"enable_thinking": true}}) # Vertex AI configuration llm_vertexai_project_id: str | None @@ -1039,6 +1044,7 @@ class HindsightConfig: llm_timeout=float(os.getenv(ENV_LLM_TIMEOUT, str(DEFAULT_LLM_TIMEOUT))), llm_groq_service_tier=os.getenv(ENV_LLM_GROQ_SERVICE_TIER, DEFAULT_LLM_GROQ_SERVICE_TIER), llm_openai_service_tier=os.getenv(ENV_LLM_OPENAI_SERVICE_TIER, DEFAULT_LLM_OPENAI_SERVICE_TIER), + llm_extra_body=json.loads(os.getenv(ENV_LLM_EXTRA_BODY, "null")), # Vertex AI llm_vertexai_project_id=os.getenv(ENV_LLM_VERTEXAI_PROJECT_ID) or DEFAULT_LLM_VERTEXAI_PROJECT_ID, llm_vertexai_region=os.getenv(ENV_LLM_VERTEXAI_REGION, DEFAULT_LLM_VERTEXAI_REGION), diff --git a/hindsight-api-slim/hindsight_api/engine/llm_wrapper.py b/hindsight-api-slim/hindsight_api/engine/llm_wrapper.py index 3781b962..437637ba 100644 --- a/hindsight-api-slim/hindsight_api/engine/llm_wrapper.py +++ b/hindsight-api-slim/hindsight_api/engine/llm_wrapper.py @@ -146,6 +146,7 @@ def create_llm_provider( reasoning_effort: str, groq_service_tier: str | None = None, openai_service_tier: str | None = None, + extra_body: dict[str, Any] | None = None, vertexai_project_id: str | None = None, vertexai_region: str | None = None, vertexai_credentials: Any = None, @@ -162,6 +163,7 @@ def create_llm_provider( reasoning_effort: Reasoning effort level for supported providers. groq_service_tier: Groq service tier (for Groq provider) - "on_demand", "flex", or "auto". openai_service_tier: OpenAI service tier (for OpenAI provider) - None (default) or "flex" (50% cheaper). + extra_body: Extra body params merged into OpenAI-compatible API calls. vertexai_project_id: Vertex AI project ID (for VertexAI provider). vertexai_region: Vertex AI region (for VertexAI provider). vertexai_credentials: Vertex AI credentials object (for VertexAI provider). @@ -270,6 +272,7 @@ def create_llm_provider( reasoning_effort=reasoning_effort, groq_service_tier=groq_service_tier, openai_service_tier=openai_service_tier, + extra_body=extra_body, ) else: @@ -293,6 +296,7 @@ class LLMProvider: groq_service_tier: str | None = None, openai_service_tier: str | None = None, gemini_safety_settings: list | None = None, + extra_body: dict[str, Any] | None = None, ): """ Initialize LLM provider. @@ -306,6 +310,7 @@ class LLMProvider: groq_service_tier: Groq service tier ("on_demand", "flex", "auto") - from config. openai_service_tier: OpenAI service tier (None or "flex") - from config. gemini_safety_settings: Safety settings for Gemini/VertexAI providers. + extra_body: Extra body params merged into OpenAI-compatible API calls. """ self.provider = provider.lower() self.api_key = api_key @@ -317,6 +322,8 @@ class LLMProvider: self.openai_service_tier = openai_service_tier # Gemini safety settings (instance default; can be overridden per-request via context var) self.gemini_safety_settings = gemini_safety_settings + # Extra body params for OpenAI-compatible providers (e.g. chat_template_kwargs) + self.extra_body = extra_body # Validate provider valid_providers = [ @@ -413,6 +420,7 @@ class LLMProvider: reasoning_effort=self.reasoning_effort, groq_service_tier=self.groq_service_tier, openai_service_tier=self.openai_service_tier, + extra_body=self.extra_body, vertexai_project_id=vertexai_project_id, vertexai_region=vertexai_region, vertexai_credentials=vertexai_credentials, @@ -725,7 +733,15 @@ class LLMProvider: base_url = os.getenv("HINDSIGHT_API_LLM_BASE_URL", "") model = os.getenv("HINDSIGHT_API_LLM_MODEL", "openai/gpt-oss-120b") - return cls(provider=provider, api_key=api_key, base_url=base_url, model=model, reasoning_effort="low") + extra_body = json.loads(os.getenv("HINDSIGHT_API_LLM_EXTRA_BODY", "null")) + return cls( + provider=provider, + api_key=api_key, + base_url=base_url, + model=model, + reasoning_effort="low", + extra_body=extra_body, + ) @classmethod def for_answer_generation(cls) -> "LLMProvider": @@ -745,7 +761,15 @@ class LLMProvider: base_url = os.getenv("HINDSIGHT_API_ANSWER_LLM_BASE_URL", os.getenv("HINDSIGHT_API_LLM_BASE_URL", "")) model = os.getenv("HINDSIGHT_API_ANSWER_LLM_MODEL", os.getenv("HINDSIGHT_API_LLM_MODEL", "openai/gpt-oss-120b")) - return cls(provider=provider, api_key=api_key, base_url=base_url, model=model, reasoning_effort="high") + extra_body = json.loads(os.getenv("HINDSIGHT_API_LLM_EXTRA_BODY", "null")) + return cls( + provider=provider, + api_key=api_key, + base_url=base_url, + model=model, + reasoning_effort="high", + extra_body=extra_body, + ) @classmethod def for_judge(cls) -> "LLMProvider": @@ -765,7 +789,15 @@ class LLMProvider: base_url = os.getenv("HINDSIGHT_API_JUDGE_LLM_BASE_URL", os.getenv("HINDSIGHT_API_LLM_BASE_URL", "")) model = os.getenv("HINDSIGHT_API_JUDGE_LLM_MODEL", os.getenv("HINDSIGHT_API_LLM_MODEL", "openai/gpt-oss-120b")) - return cls(provider=provider, api_key=api_key, base_url=base_url, model=model, reasoning_effort="high") + extra_body = json.loads(os.getenv("HINDSIGHT_API_LLM_EXTRA_BODY", "null")) + return cls( + provider=provider, + api_key=api_key, + base_url=base_url, + model=model, + reasoning_effort="high", + extra_body=extra_body, + ) class ConfiguredLLMProvider: diff --git a/hindsight-api-slim/hindsight_api/engine/memory_engine.py b/hindsight-api-slim/hindsight_api/engine/memory_engine.py index 44eb69ae..09128975 100644 --- a/hindsight-api-slim/hindsight_api/engine/memory_engine.py +++ b/hindsight-api-slim/hindsight_api/engine/memory_engine.py @@ -396,6 +396,7 @@ class MemoryEngine(MemoryEngineInterface): api_key=memory_llm_api_key, base_url=memory_llm_base_url, model=memory_llm_model, + extra_body=config.llm_extra_body, ) # Store client and model for convenience (deprecated: use _llm_config.call() instead) @@ -422,6 +423,7 @@ class MemoryEngine(MemoryEngineInterface): api_key=retain_api_key, base_url=retain_base_url, model=retain_model, + extra_body=config.llm_extra_body, ) # Reflect LLM config - for think/observe operations (can use lighter models) @@ -443,6 +445,7 @@ class MemoryEngine(MemoryEngineInterface): api_key=reflect_api_key, base_url=reflect_base_url, model=reflect_model, + extra_body=config.llm_extra_body, ) # Consolidation LLM config - for mental model consolidation (can use efficient models) @@ -464,6 +467,7 @@ class MemoryEngine(MemoryEngineInterface): api_key=consolidation_api_key, base_url=consolidation_base_url, model=consolidation_model, + extra_body=config.llm_extra_body, ) # Initialize cross-encoder reranker (cached for performance) diff --git a/hindsight-api-slim/hindsight_api/engine/providers/openai_compatible_llm.py b/hindsight-api-slim/hindsight_api/engine/providers/openai_compatible_llm.py index 13601b75..02ee656f 100644 --- a/hindsight-api-slim/hindsight_api/engine/providers/openai_compatible_llm.py +++ b/hindsight-api-slim/hindsight_api/engine/providers/openai_compatible_llm.py @@ -80,6 +80,7 @@ class OpenAICompatibleLLM(LLMInterface): reasoning_effort: str = "low", timeout: float | None = None, groq_service_tier: str | None = None, + extra_body: dict[str, Any] | None = None, **kwargs: Any, ): """ @@ -93,6 +94,7 @@ class OpenAICompatibleLLM(LLMInterface): reasoning_effort: Reasoning effort level for supported models ("low", "medium", "high"). timeout: Request timeout in seconds (uses env var or 300s default). groq_service_tier: Groq service tier ("on_demand", "flex", "auto"). + extra_body: Extra body params merged into every API call. **kwargs: Additional provider-specific parameters. """ super().__init__(provider, api_key, base_url, model, reasoning_effort, **kwargs) @@ -124,6 +126,8 @@ class OpenAICompatibleLLM(LLMInterface): # Service tier configuration (from config, not env vars) self.groq_service_tier = groq_service_tier self.openai_service_tier = kwargs.get("openai_service_tier") + # User-configured extra body params (merged into every API call) + self._config_extra_body = extra_body or {} # Get timeout config self.timeout = timeout or float(os.getenv(ENV_LLM_TIMEOUT, str(DEFAULT_LLM_TIMEOUT))) @@ -273,17 +277,17 @@ class OpenAICompatibleLLM(LLMInterface): call_params["reasoning_effort"] = self.reasoning_effort # Provider-specific parameters + extra_body: dict[str, Any] = {**self._config_extra_body} if self.provider == "groq": call_params["seed"] = DEFAULT_LLM_SEED - extra_body: dict[str, Any] = {} # Add service_tier if configured if self.groq_service_tier: extra_body["service_tier"] = self.groq_service_tier # Add reasoning parameters for reasoning models if is_reasoning_model: extra_body["include_reasoning"] = False - if extra_body: - call_params["extra_body"] = extra_body + if extra_body: + call_params["extra_body"] = extra_body # Prepare response format ONCE before retry loop if response_format is not None: @@ -581,8 +585,11 @@ class OpenAICompatibleLLM(LLMInterface): call_params["temperature"] = temperature # Provider-specific parameters + extra_body: dict[str, Any] = {**self._config_extra_body} if self.provider == "groq": call_params["seed"] = DEFAULT_LLM_SEED + if extra_body: + call_params["extra_body"] = extra_body last_exception = None