holetron-lab/fleet-memory
1
0
Fork 0
Code Releases Packages Activity Actions

fix: groq llm with free tier doesn't work (#102)

Browse source 
* fix: groq with free tier doens't work

* fix: groq with free tier doens't work
This commit is contained in:
Nicolò Boschi 2026-01-05 15:10:35 +01:00 committed by GitHub
parent 70de23ed85
commit d899d1890d
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
3 changed files with 16 additions and 3 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
hindsight-api/hindsight_api/config.py
View file

@ -18,6 +18,7 @@ ENV_LLM_MODEL = "HINDSIGHT_API_LLM_MODEL"
ENV_LLM_BASE_URL = "HINDSIGHT_API_LLM_BASE_URL"
ENV_LLM_MAX_CONCURRENT = "HINDSIGHT_API_LLM_MAX_CONCURRENT"
ENV_LLM_TIMEOUT = "HINDSIGHT_API_LLM_TIMEOUT"
ENV_LLM_GROQ_SERVICE_TIER = "HINDSIGHT_API_LLM_GROQ_SERVICE_TIER"
ENV_EMBEDDINGS_PROVIDER = "HINDSIGHT_API_EMBEDDINGS_PROVIDER"
ENV_EMBEDDINGS_LOCAL_MODEL = "HINDSIGHT_API_EMBEDDINGS_LOCAL_MODEL"

15
hindsight-api/hindsight_api/engine/llm_wrapper.py
View file

@ -19,6 +19,7 @@ from openai import APIConnectionError, APIStatusError, AsyncOpenAI, LengthFinish
from ..config import (
DEFAULT_LLM_MAX_CONCURRENT,
DEFAULT_LLM_TIMEOUT,
ENV_LLM_GROQ_SERVICE_TIER,
ENV_LLM_MAX_CONCURRENT,
ENV_LLM_TIMEOUT,
)
@ -63,6 +64,7 @@ class LLMProvider:
base_url: str,
model: str,
reasoning_effort: str = "low",
groq_service_tier: str | None = None,
):
"""
Initialize LLM provider.
@ -73,12 +75,15 @@ class LLMProvider:
base_url: Base URL for the API.
model: Model name.
reasoning_effort: Reasoning effort level for supported providers.
groq_service_tier: Groq service tier ("on_demand", "flex", "auto"). Default: None (uses Groq's default).
"""
self.provider = provider.lower()
self.api_key = api_key
self.base_url = base_url
self.model = model
self.reasoning_effort = reasoning_effort
# Default to 'auto' for best performance, users can override to 'on_demand' for free tier
self.groq_service_tier = groq_service_tier or os.getenv(ENV_LLM_GROQ_SERVICE_TIER, "auto")
# Validate provider
valid_providers = ["openai", "groq", "ollama", "gemini", "anthropic", "lmstudio"]
@ -263,11 +268,15 @@ class LLMProvider:
# Provider-specific parameters
if self.provider == "groq":
call_params["seed"] = DEFAULT_LLM_SEED
extra_body = {"service_tier": "auto"}
# Only add reasoning parameters for reasoning models
extra_body: dict[str, Any] = {}
# Add service_tier if configured (requires paid plan for flex/auto)
if self.groq_service_tier:
extra_body["service_tier"] = self.groq_service_tier
# Add reasoning parameters for reasoning models
if is_reasoning_model:
extra_body["include_reasoning"] = False
call_params["extra_body"] = extra_body
if extra_body:
call_params["extra_body"] = extra_body
last_exception = None

3
hindsight-docs/docs/developer/configuration.md
View file

@ -33,6 +33,7 @@ If not provided, the server uses embedded `pg0` — convenient for development b
| `HINDSIGHT_API_LLM_BASE_URL` | Custom LLM endpoint | Provider default |
| `HINDSIGHT_API_LLM_MAX_CONCURRENT` | Max concurrent LLM requests | `32` |
| `HINDSIGHT_API_LLM_TIMEOUT` | LLM request timeout in seconds | `120` |
| `HINDSIGHT_API_LLM_GROQ_SERVICE_TIER` | Groq service tier: `on_demand`, `flex`, `auto` | `auto` |
**Provider Examples**
@ -41,6 +42,8 @@ If not provided, the server uses embedded `pg0` — convenient for development b
export HINDSIGHT_API_LLM_PROVIDER=groq
export HINDSIGHT_API_LLM_API_KEY=gsk_xxxxxxxxxxxx
export HINDSIGHT_API_LLM_MODEL=openai/gpt-oss-20b
# For free tier users: override to on_demand if you get service_tier errors
# export HINDSIGHT_API_LLM_GROQ_SERVICE_TIER=on_demand
# OpenAI
export HINDSIGHT_API_LLM_PROVIDER=openai