""" Google Gemini/VertexAI LLM provider. This provider supports both: 1. Gemini API (api.generativeai.google.com) with API key authentication 2. Vertex AI with service account or Application Default Credentials (ADC) """ import asyncio import json import logging import os import time from contextvars import ContextVar from typing import Any from google import genai from google.genai import errors as genai_errors from google.genai import types as genai_types from hindsight_api.engine.llm_interface import LLMInterface, OutputTooLongError from hindsight_api.engine.llm_wrapper import parse_llm_json from hindsight_api.engine.response_models import LLMToolCall, LLMToolCallResult, TokenUsage from hindsight_api.metrics import get_metrics_collector logger = logging.getLogger(__name__) # Per-request Gemini safety settings override. # Set exclusively by ConfiguredLLMProvider.call() / call_with_tools() via token-based # set/reset, so it is properly scoped to each individual LLM call and never leaks. _safety_settings_ctx: ContextVar[list | None] = ContextVar("gemini_safety_settings", default=None) # Vertex AI imports (optional) try: import google.auth from google.oauth2 import service_account VERTEXAI_AVAILABLE = True except ImportError: VERTEXAI_AVAILABLE = False class GeminiLLM(LLMInterface): """ LLM provider for Google Gemini and Vertex AI. Supports: - Gemini API: provider="gemini", requires api_key - Vertex AI: provider="vertexai", requires project_id and region, uses ADC or service account """ def __init__( self, provider: str, api_key: str, base_url: str, model: str, reasoning_effort: str = "low", **kwargs: Any, ): """Initialize Gemini/VertexAI LLM provider.""" super().__init__(provider, api_key, base_url, model, reasoning_effort, **kwargs) self._client = None self._is_vertexai = self.provider == "vertexai" # Safety settings: None means use Gemini's defaults self._safety_settings: list | None = kwargs.get("gemini_safety_settings") if self._is_vertexai: self._init_vertexai(**kwargs) else: self._init_gemini() def _init_gemini(self) -> None: """Initialize Gemini API client.""" if not self.api_key: raise ValueError("Gemini provider requires api_key") self._client = genai.Client(api_key=self.api_key) logger.info(f"Gemini API: model={self.model}") def _init_vertexai(self, **kwargs: Any) -> None: """Initialize Vertex AI client with project, region, and credentials.""" # Extract Vertex AI config from kwargs project_id = kwargs.get("vertexai_project_id") region = kwargs.get("vertexai_region", "us-central1") service_account_key = kwargs.get("vertexai_service_account_key") credentials = kwargs.get("vertexai_credentials") # Pre-loaded credentials object if not project_id: raise ValueError( "HINDSIGHT_API_LLM_VERTEXAI_PROJECT_ID is required for Vertex AI provider. " "Set it to your GCP project ID." ) auth_method = "ADC" # Use pre-loaded credentials if provided (passed from LLMProvider) if credentials is not None: auth_method = "service_account" # Otherwise, load explicit service account credentials if path provided elif service_account_key: if not VERTEXAI_AVAILABLE: raise ValueError( "Vertex AI service account auth requires 'google-auth' package. " "Install with: pip install google-auth" ) credentials = service_account.Credentials.from_service_account_file( service_account_key, scopes=["https://www.googleapis.com/auth/cloud-platform"], ) auth_method = "service_account" logger.info(f"Vertex AI: Using service account key: {service_account_key}") # Strip google/ prefix from model name — native SDK uses bare names # e.g. "google/gemini-2.0-flash-lite-001" -> "gemini-2.0-flash-lite-001" if self.model.startswith("google/"): self.model = self.model[len("google/") :] # Create Vertex AI client client_kwargs: dict[str, Any] = { "vertexai": True, "project": project_id, "location": region, } if credentials is not None: client_kwargs["credentials"] = credentials self._client = genai.Client(**client_kwargs) logger.info(f"Vertex AI: project={project_id}, region={region}, model={self.model}, auth={auth_method}") async def verify_connection(self) -> None: """ Verify that the Gemini/VertexAI provider is configured correctly. Raises: RuntimeError: If the connection test fails. """ try: logger.info(f"Verifying {self.provider.upper()}: model={self.model}...") await self.call( messages=[{"role": "user", "content": "Say 'ok'"}], max_completion_tokens=100, max_retries=2, initial_backoff=0.5, max_backoff=2.0, scope="verification", ) logger.info(f"{self.provider.upper()} connection verified successfully") except Exception as e: raise RuntimeError(f"Failed to verify {self.provider.upper()} connection: {e}") from e async def call( self, messages: list[dict[str, str]], response_format: Any | None = None, max_completion_tokens: int | None = None, temperature: float | None = None, scope: str = "memory", max_retries: int = 10, initial_backoff: float = 1.0, max_backoff: float = 60.0, skip_validation: bool = False, strict_schema: bool = False, return_usage: bool = False, ) -> Any: """ Make a Gemini/VertexAI API call with retry logic. Args: messages: List of message dicts with 'role' and 'content'. response_format: Optional Pydantic model for structured output. max_completion_tokens: Maximum tokens in response (not supported by Gemini). temperature: Sampling temperature (0.0-2.0). scope: Scope identifier for tracking. max_retries: Maximum retry attempts. initial_backoff: Initial backoff time in seconds. max_backoff: Maximum backoff time in seconds. skip_validation: Return raw JSON without Pydantic validation. strict_schema: Use strict JSON schema enforcement (not supported by Gemini). return_usage: If True, return tuple (result, TokenUsage). Returns: If return_usage=False: Parsed response if response_format provided, else text. If return_usage=True: Tuple of (result, TokenUsage). """ start_time = time.time() # Convert OpenAI-style messages to Gemini format system_instruction = None gemini_contents = [] for msg in messages: role = msg.get("role", "user") content = msg.get("content", "") if role == "system": if system_instruction: system_instruction += "\n\n" + content else: system_instruction = content elif role == "assistant": gemini_contents.append(genai_types.Content(role="model", parts=[genai_types.Part(text=content)])) else: gemini_contents.append(genai_types.Content(role="user", parts=[genai_types.Part(text=content)])) # Add JSON schema instruction if response_format is provided if response_format is not None and hasattr(response_format, "model_json_schema"): schema = response_format.model_json_schema() schema_msg = f"\n\nYou must respond with valid JSON matching this schema:\n{json.dumps(schema, indent=2)}" if system_instruction: system_instruction += schema_msg else: system_instruction = schema_msg # Build generation config config_kwargs: dict[str, Any] = {} if system_instruction: config_kwargs["system_instruction"] = system_instruction if response_format is not None: config_kwargs["response_mime_type"] = "application/json" config_kwargs["response_schema"] = response_format if temperature is not None: config_kwargs["temperature"] = temperature # Apply safety settings: context var (per-request bank override) takes precedence over instance default effective_safety_settings = _safety_settings_ctx.get() if effective_safety_settings is None: effective_safety_settings = self._safety_settings if effective_safety_settings is not None: config_kwargs["safety_settings"] = [ genai_types.SafetySetting(category=s["category"], threshold=s["threshold"]) for s in effective_safety_settings ] generation_config = genai_types.GenerateContentConfig(**config_kwargs) if config_kwargs else None last_exception = None for attempt in range(max_retries + 1): try: response = await asyncio.wait_for( self._client.aio.models.generate_content( model=self.model, contents=gemini_contents, config=generation_config, ), timeout=90.0, # Safety net for network hangs; valid slow responses are <90s ) content = response.text # Handle empty response if content is None: block_reason = None if hasattr(response, "candidates") and response.candidates: candidate = response.candidates[0] if hasattr(candidate, "finish_reason"): block_reason = candidate.finish_reason if attempt < max_retries: logger.warning(f"Gemini returned empty response (reason: {block_reason}), retrying...") backoff = min(initial_backoff * (2**attempt), max_backoff) await asyncio.sleep(backoff) continue else: raise RuntimeError(f"Gemini returned empty response after {max_retries + 1} attempts") # Parse structured output if requested if response_format is not None: json_data = parse_llm_json(content) if skip_validation: result = json_data else: result = response_format.model_validate(json_data) else: result = content # Extract token usage input_tokens = 0 output_tokens = 0 if hasattr(response, "usage_metadata") and response.usage_metadata: usage = response.usage_metadata input_tokens = usage.prompt_token_count or 0 output_tokens = usage.candidates_token_count or 0 # Record metrics duration = time.time() - start_time metrics = get_metrics_collector() metrics.record_llm_call( provider=self.provider, model=self.model, scope=scope, duration=duration, input_tokens=input_tokens, output_tokens=output_tokens, success=True, ) # Record trace span from hindsight_api.tracing import get_span_recorder finish_reason = None if hasattr(response, "candidates") and response.candidates: if hasattr(response.candidates[0], "finish_reason"): finish_reason = str(response.candidates[0].finish_reason) span_recorder = get_span_recorder() from hindsight_api.tracing import _serialize_for_span span_recorder.record_llm_call( provider=self.provider, model=self.model, scope=scope, messages=messages, response_content=_serialize_for_span(result), input_tokens=input_tokens, output_tokens=output_tokens, duration=duration, finish_reason=finish_reason, error=None, ) # Log slow calls if duration > 10.0 and input_tokens > 0: logger.info( f"slow llm call: scope={scope}, model={self.provider}/{self.model}, " f"input_tokens={input_tokens}, output_tokens={output_tokens}, " f"time={duration:.3f}s" ) if return_usage: token_usage = TokenUsage( input_tokens=input_tokens, output_tokens=output_tokens, total_tokens=input_tokens + output_tokens, ) return result, token_usage return result except json.JSONDecodeError as e: last_exception = e if attempt < max_retries: logger.warning("Gemini returned invalid JSON, retrying...") backoff = min(initial_backoff * (2**attempt), max_backoff) await asyncio.sleep(backoff) continue else: logger.error(f"Gemini returned invalid JSON after {max_retries + 1} attempts") raise except genai_errors.APIError as e: # Fast fail on auth errors - these won't recover with retries if e.code in (401, 403): logger.error(f"Gemini auth error (HTTP {e.code}), not retrying: {str(e)}") raise # Retry on retryable errors (rate limits, server errors, client errors) if e.code in (400, 429, 500, 502, 503, 504) or (e.code and e.code >= 500): last_exception = e if attempt < max_retries: backoff = min(initial_backoff * (2**attempt), max_backoff) jitter = backoff * 0.2 * (2 * (time.time() % 1) - 1) await asyncio.sleep(backoff + jitter) else: logger.error(f"Gemini API error after {max_retries + 1} attempts: {str(e)}") raise else: logger.error(f"Gemini API error: {type(e).__name__}: {str(e)}") raise except Exception as e: logger.error(f"Unexpected error during Gemini call: {type(e).__name__}: {str(e)}") raise if last_exception: raise last_exception raise RuntimeError("Gemini call failed after all retries") async def call_with_tools( self, messages: list[dict[str, Any]], tools: list[dict[str, Any]], max_completion_tokens: int | None = None, temperature: float | None = None, scope: str = "tools", max_retries: int = 5, initial_backoff: float = 1.0, max_backoff: float = 30.0, tool_choice: str | dict[str, Any] = "auto", ) -> LLMToolCallResult: """ Make a Gemini/VertexAI API call with tool/function calling support. Args: messages: List of message dicts. Can include tool results with role='tool'. tools: List of tool definitions in OpenAI format. max_completion_tokens: Maximum tokens (not supported by Gemini). temperature: Sampling temperature. scope: Scope identifier for tracking. max_retries: Maximum retry attempts. initial_backoff: Initial backoff time in seconds. max_backoff: Maximum backoff time in seconds. tool_choice: How to choose tools (Gemini uses "auto" only). Returns: LLMToolCallResult with content and/or tool_calls. """ start_time = time.time() # Convert tools to Gemini format gemini_tools = [] for tool in tools: func = tool.get("function", {}) gemini_tools.append( genai_types.Tool( function_declarations=[ genai_types.FunctionDeclaration( name=func.get("name", ""), description=func.get("description", ""), parameters=func.get("parameters"), ) ] ) ) # Convert messages system_instruction = None gemini_contents = [] msg_list = list(messages) i = 0 while i < len(msg_list): msg = msg_list[i] role = msg.get("role", "user") content = msg.get("content", "") if role == "system": system_instruction = (system_instruction + "\n\n" + content) if system_instruction else content i += 1 elif role == "tool": # Gemini requires ALL tool responses for a given model turn to be grouped # into a single Content with multiple FunctionResponse parts. # Consecutive role="tool" messages correspond to one model turn's tool calls. parts = [] while i < len(msg_list) and msg_list[i].get("role") == "tool": tool_msg = msg_list[i] tool_content = tool_msg.get("content", "") parts.append( genai_types.Part( function_response=genai_types.FunctionResponse( name=tool_msg.get("name", ""), response={"result": tool_content}, ) ) ) i += 1 gemini_contents.append(genai_types.Content(role="user", parts=parts)) elif role == "assistant": tool_calls_in_msg = msg.get("tool_calls", []) if tool_calls_in_msg: # Convert OpenAI-style tool_calls to Gemini function_call parts # This is required for proper multi-turn conversation history parts = [] if content: parts.append(genai_types.Part(text=content)) for tc in tool_calls_in_msg: fn = tc.get("function", {}) fn_name = fn.get("name", "") fn_args_str = fn.get("arguments", "{}") fn_args = parse_llm_json(fn_args_str) parts.append( genai_types.Part(function_call=genai_types.FunctionCall(name=fn_name, args=fn_args)) ) gemini_contents.append(genai_types.Content(role="model", parts=parts)) else: gemini_contents.append(genai_types.Content(role="model", parts=[genai_types.Part(text=content)])) i += 1 else: gemini_contents.append(genai_types.Content(role="user", parts=[genai_types.Part(text=content)])) i += 1 config_kwargs: dict[str, Any] = {"tools": gemini_tools} if system_instruction: config_kwargs["system_instruction"] = system_instruction if temperature is not None: config_kwargs["temperature"] = temperature # Map OpenAI-style tool_choice to Gemini FunctionCallingConfig if tool_choice == "required": config_kwargs["tool_config"] = genai_types.ToolConfig( function_calling_config=genai_types.FunctionCallingConfig( mode="ANY", ) ) elif isinstance(tool_choice, dict) and tool_choice.get("type") == "function": fn_name = tool_choice.get("function", {}).get("name") if fn_name: config_kwargs["tool_config"] = genai_types.ToolConfig( function_calling_config=genai_types.FunctionCallingConfig( mode="ANY", allowed_function_names=[fn_name], ) ) elif tool_choice == "none": config_kwargs["tool_config"] = genai_types.ToolConfig( function_calling_config=genai_types.FunctionCallingConfig(mode="NONE") ) # "auto" is the default (no tool_config needed) # Apply safety settings: context var (per-request bank override) takes precedence over instance default effective_safety_settings = _safety_settings_ctx.get() if effective_safety_settings is None: effective_safety_settings = self._safety_settings if effective_safety_settings is not None: config_kwargs["safety_settings"] = [ genai_types.SafetySetting(category=s["category"], threshold=s["threshold"]) for s in effective_safety_settings ] config = genai_types.GenerateContentConfig(**config_kwargs) last_exception = None for attempt in range(max_retries + 1): try: response = await asyncio.wait_for( self._client.aio.models.generate_content( model=self.model, contents=gemini_contents, config=config, ), timeout=90.0, # Safety net for network hangs; valid slow responses are <90s ) # Extract content and tool calls content = None tool_calls: list[LLMToolCall] = [] if response.candidates and response.candidates[0].content: parts = response.candidates[0].content.parts if parts: for part in parts: if hasattr(part, "text") and part.text: content = part.text if hasattr(part, "function_call") and part.function_call: fc = part.function_call tool_calls.append( LLMToolCall( id=f"gemini_{len(tool_calls)}", name=fc.name, arguments=dict(fc.args) if fc.args else {}, ) ) finish_reason = "tool_calls" if tool_calls else "stop" # Extract token usage input_tokens = 0 output_tokens = 0 if response.usage_metadata: input_tokens = response.usage_metadata.prompt_token_count or 0 output_tokens = response.usage_metadata.candidates_token_count or 0 # Record metrics duration = time.time() - start_time metrics = get_metrics_collector() metrics.record_llm_call( provider=self.provider, model=self.model, scope=scope, duration=duration, input_tokens=input_tokens, output_tokens=output_tokens, success=True, ) # Record OpenTelemetry span from hindsight_api.tracing import get_span_recorder span_recorder = get_span_recorder() # Convert LLMToolCall objects to dicts for span recording tool_calls_dict = ( [{"id": tc.id, "name": tc.name, "arguments": tc.arguments} for tc in tool_calls] if tool_calls else None ) span_recorder.record_llm_call( provider=self.provider, model=self.model, scope=scope, messages=messages, response_content=content, input_tokens=input_tokens, output_tokens=output_tokens, duration=duration, finish_reason=finish_reason, error=None, tool_calls=tool_calls_dict, ) return LLMToolCallResult( content=content, tool_calls=tool_calls, finish_reason=finish_reason, input_tokens=input_tokens, output_tokens=output_tokens, ) except genai_errors.APIError as e: # Fast fail on auth errors if e.code in (401, 403): logger.error(f"Gemini auth error (HTTP {e.code}), not retrying: {str(e)}") raise # Retry on retryable errors last_exception = e if attempt < max_retries: backoff = min(initial_backoff * (2**attempt), max_backoff) await asyncio.sleep(backoff) continue raise except Exception as e: logger.error(f"Unexpected error during Gemini tool call: {type(e).__name__}: {str(e)}") raise if last_exception: raise last_exception raise RuntimeError("Gemini tool call failed") async def cleanup(self) -> None: """Clean up resources (close connections, etc.).""" # Gemini client doesn't require explicit cleanup pass