holetron-lab/fleet-memory
1
0
Fork 0
Code Releases Packages Activity Actions

feat(api): add HINDSIGHT_API_LLM_EXTRA_BODY config for custom model params (#781)

Browse source 
Enable passing arbitrary extra_body parameters to OpenAI-compatible API
calls via a JSON-encoded env var. This supports custom model servers
(e.g. vLLM) that need parameters like chat_template_kwargs to control
thinking mode.

Co-authored-by: EMIRHAN GAZI <emirhan.gazi+tcell@turkcell.com.tr>
This commit is contained in:
emirhan-gazi 2026-03-31 18:03:33 +03:00 committed by GitHub
parent ea0c616240
commit ecaa1ad1e0
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
4 changed files with 55 additions and 6 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
hindsight-api-slim/hindsight_api/config.py
View file

@ -131,10 +131,12 @@ ENV_LLM_MAX_BACKOFF = "HINDSIGHT_API_LLM_MAX_BACKOFF"
ENV_LLM_TIMEOUT = "HINDSIGHT_API_LLM_TIMEOUT"
ENV_LLM_GROQ_SERVICE_TIER = "HINDSIGHT_API_LLM_GROQ_SERVICE_TIER"
ENV_LLM_OPENAI_SERVICE_TIER = "HINDSIGHT_API_LLM_OPENAI_SERVICE_TIER"
ENV_LLM_EXTRA_BODY = "HINDSIGHT_API_LLM_EXTRA_BODY"
# Defaults for service tiers
DEFAULT_LLM_GROQ_SERVICE_TIER = "auto" # "on_demand", "flex", or "auto"
DEFAULT_LLM_OPENAI_SERVICE_TIER = None # None (default) or "flex" (50% cheaper)
DEFAULT_LLM_EXTRA_BODY = None # None = no extra body params; JSON dict merged into OpenAI extra_body
# Per-operation LLM configuration (optional, falls back to global LLM config)
ENV_RETAIN_LLM_PROVIDER = "HINDSIGHT_API_RETAIN_LLM_PROVIDER"
@ -640,6 +642,9 @@ class HindsightConfig:
llm_timeout: float
llm_groq_service_tier: str # Groq: "on_demand", "flex", or "auto"
llm_openai_service_tier: str | None # OpenAI: None (default) or "flex" (50% cheaper)
llm_extra_body: (
dict | None
) # Extra body params merged into OpenAI-compatible API calls (e.g. {"chat_template_kwargs": {"enable_thinking": true}})
# Vertex AI configuration
llm_vertexai_project_id: str | None
@ -1039,6 +1044,7 @@ class HindsightConfig:
llm_timeout=float(os.getenv(ENV_LLM_TIMEOUT, str(DEFAULT_LLM_TIMEOUT))),
llm_groq_service_tier=os.getenv(ENV_LLM_GROQ_SERVICE_TIER, DEFAULT_LLM_GROQ_SERVICE_TIER),
llm_openai_service_tier=os.getenv(ENV_LLM_OPENAI_SERVICE_TIER, DEFAULT_LLM_OPENAI_SERVICE_TIER),
llm_extra_body=json.loads(os.getenv(ENV_LLM_EXTRA_BODY, "null")),
# Vertex AI
llm_vertexai_project_id=os.getenv(ENV_LLM_VERTEXAI_PROJECT_ID) or DEFAULT_LLM_VERTEXAI_PROJECT_ID,
llm_vertexai_region=os.getenv(ENV_LLM_VERTEXAI_REGION, DEFAULT_LLM_VERTEXAI_REGION),

38
hindsight-api-slim/hindsight_api/engine/llm_wrapper.py
View file

@ -146,6 +146,7 @@ def create_llm_provider(
reasoning_effort: str,
groq_service_tier: str | None = None,
openai_service_tier: str | None = None,
extra_body: dict[str, Any] | None = None,
vertexai_project_id: str | None = None,
vertexai_region: str | None = None,
vertexai_credentials: Any = None,
@ -162,6 +163,7 @@ def create_llm_provider(
reasoning_effort: Reasoning effort level for supported providers.
groq_service_tier: Groq service tier (for Groq provider) - "on_demand", "flex", or "auto".
openai_service_tier: OpenAI service tier (for OpenAI provider) - None (default) or "flex" (50% cheaper).
extra_body: Extra body params merged into OpenAI-compatible API calls.
vertexai_project_id: Vertex AI project ID (for VertexAI provider).
vertexai_region: Vertex AI region (for VertexAI provider).
vertexai_credentials: Vertex AI credentials object (for VertexAI provider).
@ -270,6 +272,7 @@ def create_llm_provider(
reasoning_effort=reasoning_effort,
groq_service_tier=groq_service_tier,
openai_service_tier=openai_service_tier,
extra_body=extra_body,
)
else:
@ -293,6 +296,7 @@ class LLMProvider:
groq_service_tier: str | None = None,
openai_service_tier: str | None = None,
gemini_safety_settings: list | None = None,
extra_body: dict[str, Any] | None = None,
):
"""
Initialize LLM provider.
@ -306,6 +310,7 @@ class LLMProvider:
groq_service_tier: Groq service tier ("on_demand", "flex", "auto") - from config.
openai_service_tier: OpenAI service tier (None or "flex") - from config.
gemini_safety_settings: Safety settings for Gemini/VertexAI providers.
extra_body: Extra body params merged into OpenAI-compatible API calls.
"""
self.provider = provider.lower()
self.api_key = api_key
@ -317,6 +322,8 @@ class LLMProvider:
self.openai_service_tier = openai_service_tier
# Gemini safety settings (instance default; can be overridden per-request via context var)
self.gemini_safety_settings = gemini_safety_settings
# Extra body params for OpenAI-compatible providers (e.g. chat_template_kwargs)
self.extra_body = extra_body
# Validate provider
valid_providers = [
@ -413,6 +420,7 @@ class LLMProvider:
reasoning_effort=self.reasoning_effort,
groq_service_tier=self.groq_service_tier,
openai_service_tier=self.openai_service_tier,
extra_body=self.extra_body,
vertexai_project_id=vertexai_project_id,
vertexai_region=vertexai_region,
vertexai_credentials=vertexai_credentials,
@ -725,7 +733,15 @@ class LLMProvider:
base_url = os.getenv("HINDSIGHT_API_LLM_BASE_URL", "")
model = os.getenv("HINDSIGHT_API_LLM_MODEL", "openai/gpt-oss-120b")
return cls(provider=provider, api_key=api_key, base_url=base_url, model=model, reasoning_effort="low")
extra_body = json.loads(os.getenv("HINDSIGHT_API_LLM_EXTRA_BODY", "null"))
return cls(
provider=provider,
api_key=api_key,
base_url=base_url,
model=model,
reasoning_effort="low",
extra_body=extra_body,
)
@classmethod
def for_answer_generation(cls) -> "LLMProvider":
@ -745,7 +761,15 @@ class LLMProvider:
base_url = os.getenv("HINDSIGHT_API_ANSWER_LLM_BASE_URL", os.getenv("HINDSIGHT_API_LLM_BASE_URL", ""))
model = os.getenv("HINDSIGHT_API_ANSWER_LLM_MODEL", os.getenv("HINDSIGHT_API_LLM_MODEL", "openai/gpt-oss-120b"))
return cls(provider=provider, api_key=api_key, base_url=base_url, model=model, reasoning_effort="high")
extra_body = json.loads(os.getenv("HINDSIGHT_API_LLM_EXTRA_BODY", "null"))
return cls(
provider=provider,
api_key=api_key,
base_url=base_url,
model=model,
reasoning_effort="high",
extra_body=extra_body,
)
@classmethod
def for_judge(cls) -> "LLMProvider":
@ -765,7 +789,15 @@ class LLMProvider:
base_url = os.getenv("HINDSIGHT_API_JUDGE_LLM_BASE_URL", os.getenv("HINDSIGHT_API_LLM_BASE_URL", ""))
model = os.getenv("HINDSIGHT_API_JUDGE_LLM_MODEL", os.getenv("HINDSIGHT_API_LLM_MODEL", "openai/gpt-oss-120b"))
return cls(provider=provider, api_key=api_key, base_url=base_url, model=model, reasoning_effort="high")
extra_body = json.loads(os.getenv("HINDSIGHT_API_LLM_EXTRA_BODY", "null"))
return cls(
provider=provider,
api_key=api_key,
base_url=base_url,
model=model,
reasoning_effort="high",
extra_body=extra_body,
)
class ConfiguredLLMProvider:

4
hindsight-api-slim/hindsight_api/engine/memory_engine.py
View file

@ -396,6 +396,7 @@ class MemoryEngine(MemoryEngineInterface):
api_key=memory_llm_api_key,
base_url=memory_llm_base_url,
model=memory_llm_model,
extra_body=config.llm_extra_body,
)
# Store client and model for convenience (deprecated: use _llm_config.call() instead)
@ -422,6 +423,7 @@ class MemoryEngine(MemoryEngineInterface):
api_key=retain_api_key,
base_url=retain_base_url,
model=retain_model,
extra_body=config.llm_extra_body,
)
# Reflect LLM config - for think/observe operations (can use lighter models)
@ -443,6 +445,7 @@ class MemoryEngine(MemoryEngineInterface):
api_key=reflect_api_key,
base_url=reflect_base_url,
model=reflect_model,
extra_body=config.llm_extra_body,
)
# Consolidation LLM config - for mental model consolidation (can use efficient models)
@ -464,6 +467,7 @@ class MemoryEngine(MemoryEngineInterface):
api_key=consolidation_api_key,
base_url=consolidation_base_url,
model=consolidation_model,
extra_body=config.llm_extra_body,
)
# Initialize cross-encoder reranker (cached for performance)

13
hindsight-api-slim/hindsight_api/engine/providers/openai_compatible_llm.py
View file

@ -80,6 +80,7 @@ class OpenAICompatibleLLM(LLMInterface):
reasoning_effort: str = "low",
timeout: float | None = None,
groq_service_tier: str | None = None,
extra_body: dict[str, Any] | None = None,
**kwargs: Any,
):
"""
@ -93,6 +94,7 @@ class OpenAICompatibleLLM(LLMInterface):
reasoning_effort: Reasoning effort level for supported models ("low", "medium", "high").
timeout: Request timeout in seconds (uses env var or 300s default).
groq_service_tier: Groq service tier ("on_demand", "flex", "auto").
extra_body: Extra body params merged into every API call.
**kwargs: Additional provider-specific parameters.
"""
super().__init__(provider, api_key, base_url, model, reasoning_effort, **kwargs)
@ -124,6 +126,8 @@ class OpenAICompatibleLLM(LLMInterface):
# Service tier configuration (from config, not env vars)
self.groq_service_tier = groq_service_tier
self.openai_service_tier = kwargs.get("openai_service_tier")
# User-configured extra body params (merged into every API call)
self._config_extra_body = extra_body or {}
# Get timeout config
self.timeout = timeout or float(os.getenv(ENV_LLM_TIMEOUT, str(DEFAULT_LLM_TIMEOUT)))
@ -273,17 +277,17 @@ class OpenAICompatibleLLM(LLMInterface):
call_params["reasoning_effort"] = self.reasoning_effort
# Provider-specific parameters
extra_body: dict[str, Any] = {**self._config_extra_body}
if self.provider == "groq":
call_params["seed"] = DEFAULT_LLM_SEED
extra_body: dict[str, Any] = {}
# Add service_tier if configured
if self.groq_service_tier:
extra_body["service_tier"] = self.groq_service_tier
# Add reasoning parameters for reasoning models
if is_reasoning_model:
extra_body["include_reasoning"] = False
if extra_body:
call_params["extra_body"] = extra_body
if extra_body:
call_params["extra_body"] = extra_body
# Prepare response format ONCE before retry loop
if response_format is not None:
@ -581,8 +585,11 @@ class OpenAICompatibleLLM(LLMInterface):
call_params["temperature"] = temperature
# Provider-specific parameters
extra_body: dict[str, Any] = {**self._config_extra_body}
if self.provider == "groq":
call_params["seed"] = DEFAULT_LLM_SEED
if extra_body:
call_params["extra_body"] = extra_body
last_exception = None