From d899d1890d7eaa016ceaf0a9176e656922cbed66 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=B2=20Boschi?= Date: Mon, 5 Jan 2026 15:10:35 +0100 Subject: [PATCH] fix: groq llm with free tier doesn't work (#102) * fix: groq with free tier doens't work * fix: groq with free tier doens't work --- hindsight-api/hindsight_api/config.py | 1 + hindsight-api/hindsight_api/engine/llm_wrapper.py | 15 ++++++++++++--- hindsight-docs/docs/developer/configuration.md | 3 +++ 3 files changed, 16 insertions(+), 3 deletions(-) diff --git a/hindsight-api/hindsight_api/config.py b/hindsight-api/hindsight_api/config.py index 57dcde81..ea55277c 100644 --- a/hindsight-api/hindsight_api/config.py +++ b/hindsight-api/hindsight_api/config.py @@ -18,6 +18,7 @@ ENV_LLM_MODEL = "HINDSIGHT_API_LLM_MODEL" ENV_LLM_BASE_URL = "HINDSIGHT_API_LLM_BASE_URL" ENV_LLM_MAX_CONCURRENT = "HINDSIGHT_API_LLM_MAX_CONCURRENT" ENV_LLM_TIMEOUT = "HINDSIGHT_API_LLM_TIMEOUT" +ENV_LLM_GROQ_SERVICE_TIER = "HINDSIGHT_API_LLM_GROQ_SERVICE_TIER" ENV_EMBEDDINGS_PROVIDER = "HINDSIGHT_API_EMBEDDINGS_PROVIDER" ENV_EMBEDDINGS_LOCAL_MODEL = "HINDSIGHT_API_EMBEDDINGS_LOCAL_MODEL" diff --git a/hindsight-api/hindsight_api/engine/llm_wrapper.py b/hindsight-api/hindsight_api/engine/llm_wrapper.py index 783d386c..fd8f436a 100644 --- a/hindsight-api/hindsight_api/engine/llm_wrapper.py +++ b/hindsight-api/hindsight_api/engine/llm_wrapper.py @@ -19,6 +19,7 @@ from openai import APIConnectionError, APIStatusError, AsyncOpenAI, LengthFinish from ..config import ( DEFAULT_LLM_MAX_CONCURRENT, DEFAULT_LLM_TIMEOUT, + ENV_LLM_GROQ_SERVICE_TIER, ENV_LLM_MAX_CONCURRENT, ENV_LLM_TIMEOUT, ) @@ -63,6 +64,7 @@ class LLMProvider: base_url: str, model: str, reasoning_effort: str = "low", + groq_service_tier: str | None = None, ): """ Initialize LLM provider. @@ -73,12 +75,15 @@ class LLMProvider: base_url: Base URL for the API. model: Model name. reasoning_effort: Reasoning effort level for supported providers. + groq_service_tier: Groq service tier ("on_demand", "flex", "auto"). Default: None (uses Groq's default). """ self.provider = provider.lower() self.api_key = api_key self.base_url = base_url self.model = model self.reasoning_effort = reasoning_effort + # Default to 'auto' for best performance, users can override to 'on_demand' for free tier + self.groq_service_tier = groq_service_tier or os.getenv(ENV_LLM_GROQ_SERVICE_TIER, "auto") # Validate provider valid_providers = ["openai", "groq", "ollama", "gemini", "anthropic", "lmstudio"] @@ -263,11 +268,15 @@ class LLMProvider: # Provider-specific parameters if self.provider == "groq": call_params["seed"] = DEFAULT_LLM_SEED - extra_body = {"service_tier": "auto"} - # Only add reasoning parameters for reasoning models + extra_body: dict[str, Any] = {} + # Add service_tier if configured (requires paid plan for flex/auto) + if self.groq_service_tier: + extra_body["service_tier"] = self.groq_service_tier + # Add reasoning parameters for reasoning models if is_reasoning_model: extra_body["include_reasoning"] = False - call_params["extra_body"] = extra_body + if extra_body: + call_params["extra_body"] = extra_body last_exception = None diff --git a/hindsight-docs/docs/developer/configuration.md b/hindsight-docs/docs/developer/configuration.md index 9031e71d..04ca35ff 100644 --- a/hindsight-docs/docs/developer/configuration.md +++ b/hindsight-docs/docs/developer/configuration.md @@ -33,6 +33,7 @@ If not provided, the server uses embedded `pg0` — convenient for development b | `HINDSIGHT_API_LLM_BASE_URL` | Custom LLM endpoint | Provider default | | `HINDSIGHT_API_LLM_MAX_CONCURRENT` | Max concurrent LLM requests | `32` | | `HINDSIGHT_API_LLM_TIMEOUT` | LLM request timeout in seconds | `120` | +| `HINDSIGHT_API_LLM_GROQ_SERVICE_TIER` | Groq service tier: `on_demand`, `flex`, `auto` | `auto` | **Provider Examples** @@ -41,6 +42,8 @@ If not provided, the server uses embedded `pg0` — convenient for development b export HINDSIGHT_API_LLM_PROVIDER=groq export HINDSIGHT_API_LLM_API_KEY=gsk_xxxxxxxxxxxx export HINDSIGHT_API_LLM_MODEL=openai/gpt-oss-20b +# For free tier users: override to on_demand if you get service_tier errors +# export HINDSIGHT_API_LLM_GROQ_SERVICE_TIER=on_demand # OpenAI export HINDSIGHT_API_LLM_PROVIDER=openai