fix: add defensive error handling to PyTorch device detection (#221)
* fix: include correct __version__ in python packages * fix(embed): force CPU mode for local models in daemon to prevent XPC crashes Adds HINDSIGHT_API_EMBEDDINGS_LOCAL_FORCE_CPU and HINDSIGHT_API_RERANKER_LOCAL_FORCE_CPU environment variables to force CPU-only operation for local sentence-transformer models. This prevents XPC_ERROR_CONNECTION_INVALID crashes on macOS when running in daemon mode. The issue occurs because PyTorch's MPS (Metal Performance Shaders) backend has unstable XPC connections in background processes, leading to C++ assertion failures that Python exception handlers cannot catch. Changes: - config.py: Add ENV_*_FORCE_CPU constants and config dataclass fields - embeddings.py: Add force_cpu parameter to LocalSTEmbeddings constructor - cross_encoder.py: Add force_cpu parameter to LocalSTCrossEncoder constructor - main.py: Set force CPU env vars in daemon mode, add fields to config constructor The daemon mode automatically enables force CPU for both embeddings and reranker, while normal mode allows hardware acceleration (GPU/MPS) as before. 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude Sonnet 4.5 <noreply@anthropic.com> * fix: add defensive error handling to PyTorch device detection Wraps all PyTorch device detection code (torch.cuda.is_available() and torch.backends.mps.is_available()) in try-except blocks that gracefully fall back to CPU if any errors occur. This complements PR #218's force_cpu configuration by ensuring the code works reliably in all environments without configuration: - CI environments with CPU-only PyTorch builds - Systems without proper GPU/MPS support - Partial or misconfigured PyTorch installations The defensive approach prevents startup failures while still taking advantage of GPU/MPS acceleration when available and force_cpu is not explicitly set. Changes: - embeddings.py: Added try-except in initialize() and _reinitialize_model_sync() - cross_encoder.py: Added try-except in initialize() and _reinitialize_model_sync() * refactor: use get_config() for embeddings and reranker force_cpu Changes create_embeddings_from_env() and create_cross_encoder_from_env() to read configuration via get_config() instead of directly accessing os.environ. This ensures consistency across the codebase and properly respects the force_cpu configuration set by daemon mode. Changes: - embeddings.py: Use config.embeddings_local_model and config.embeddings_local_force_cpu - cross_encoder.py: Use config.reranker_local_model and config.reranker_local_force_cpu - Both: Use get_config() for provider, tei_url, and other config fields - Note: Some fields not in config (like max_concurrent for local reranker) still read from os.environ This fixes the issue where force_cpu was read inconsistently from environment variables instead of using the centralized config system. * test: clear config cache in test_create_from_env Fixes test failure caused by cached config not picking up environment variable changes in test. The test now calls clear_config_cache() before and after patching os.environ to ensure the factory function reads the test's env vars. * refactor: add reranker_local_max_concurrent to config system Adds reranker_local_max_concurrent to HindsightConfig dataclass and removes the workaround in create_cross_encoder_from_env() that was reading it directly from os.environ. Changes: - config.py: Add reranker_local_max_concurrent field to dataclass and from_env() - main.py: Add reranker_local_max_concurrent to manual config constructor - cross_encoder.py: Use config.reranker_local_max_concurrent instead of os.environ This completes the refactoring to use the centralized config system for all reranker configuration. --------- Co-authored-by: Claude Sonnet 4.5 <noreply@anthropic.com>
This commit is contained in:
Nicolò Boschi
GitHub
parent
2b72e1fd68
commit
67c47881cb
5 changed files with 128 additions and 44 deletions
|
|
@ -47,6 +47,7 @@ ENV_CONSOLIDATION_LLM_BASE_URL = "HINDSIGHT_API_CONSOLIDATION_LLM_BASE_URL"
|
|||
|
||||
ENV_EMBEDDINGS_PROVIDER = "HINDSIGHT_API_EMBEDDINGS_PROVIDER"
|
||||
ENV_EMBEDDINGS_LOCAL_MODEL = "HINDSIGHT_API_EMBEDDINGS_LOCAL_MODEL"
|
||||
ENV_EMBEDDINGS_LOCAL_FORCE_CPU = "HINDSIGHT_API_EMBEDDINGS_LOCAL_FORCE_CPU"
|
||||
ENV_EMBEDDINGS_TEI_URL = "HINDSIGHT_API_EMBEDDINGS_TEI_URL"
|
||||
ENV_EMBEDDINGS_OPENAI_API_KEY = "HINDSIGHT_API_EMBEDDINGS_OPENAI_API_KEY"
|
||||
ENV_EMBEDDINGS_OPENAI_MODEL = "HINDSIGHT_API_EMBEDDINGS_OPENAI_MODEL"
|
||||
|
|
@ -66,6 +67,7 @@ ENV_RERANKER_LITELLM_MODEL = "HINDSIGHT_API_RERANKER_LITELLM_MODEL"
|
|||
|
||||
ENV_RERANKER_PROVIDER = "HINDSIGHT_API_RERANKER_PROVIDER"
|
||||
ENV_RERANKER_LOCAL_MODEL = "HINDSIGHT_API_RERANKER_LOCAL_MODEL"
|
||||
ENV_RERANKER_LOCAL_FORCE_CPU = "HINDSIGHT_API_RERANKER_LOCAL_FORCE_CPU"
|
||||
ENV_RERANKER_LOCAL_MAX_CONCURRENT = "HINDSIGHT_API_RERANKER_LOCAL_MAX_CONCURRENT"
|
||||
ENV_RERANKER_TEI_URL = "HINDSIGHT_API_RERANKER_TEI_URL"
|
||||
ENV_RERANKER_TEI_BATCH_SIZE = "HINDSIGHT_API_RERANKER_TEI_BATCH_SIZE"
|
||||
|
|
@ -134,11 +136,13 @@ DEFAULT_LLM_TIMEOUT = 120.0 # seconds
|
|||
|
||||
DEFAULT_EMBEDDINGS_PROVIDER = "local"
|
||||
DEFAULT_EMBEDDINGS_LOCAL_MODEL = "BAAI/bge-small-en-v1.5"
|
||||
DEFAULT_EMBEDDINGS_LOCAL_FORCE_CPU = False # Force CPU mode for local embeddings (avoids MPS/XPC issues on macOS)
|
||||
DEFAULT_EMBEDDINGS_OPENAI_MODEL = "text-embedding-3-small"
|
||||
DEFAULT_EMBEDDING_DIMENSION = 384
|
||||
|
||||
DEFAULT_RERANKER_PROVIDER = "local"
|
||||
DEFAULT_RERANKER_LOCAL_MODEL = "cross-encoder/ms-marco-MiniLM-L-6-v2"
|
||||
DEFAULT_RERANKER_LOCAL_FORCE_CPU = False # Force CPU mode for local reranker (avoids MPS/XPC issues on macOS)
|
||||
DEFAULT_RERANKER_LOCAL_MAX_CONCURRENT = 4 # Limit concurrent CPU-bound reranking to prevent thrashing
|
||||
DEFAULT_RERANKER_TEI_BATCH_SIZE = 128
|
||||
DEFAULT_RERANKER_TEI_MAX_CONCURRENT = 8
|
||||
|
|
@ -301,6 +305,7 @@ class HindsightConfig:
|
|||
# Embeddings
|
||||
embeddings_provider: str
|
||||
embeddings_local_model: str
|
||||
embeddings_local_force_cpu: bool
|
||||
embeddings_tei_url: str | None
|
||||
embeddings_openai_base_url: str | None
|
||||
embeddings_cohere_base_url: str | None
|
||||
|
|
@ -308,6 +313,8 @@ class HindsightConfig:
|
|||
# Reranker
|
||||
reranker_provider: str
|
||||
reranker_local_model: str
|
||||
reranker_local_force_cpu: bool
|
||||
reranker_local_max_concurrent: int
|
||||
reranker_tei_url: str | None
|
||||
reranker_tei_batch_size: int
|
||||
reranker_tei_max_concurrent: int
|
||||
|
|
@ -394,12 +401,23 @@ class HindsightConfig:
|
|||
# Embeddings
|
||||
embeddings_provider=os.getenv(ENV_EMBEDDINGS_PROVIDER, DEFAULT_EMBEDDINGS_PROVIDER),
|
||||
embeddings_local_model=os.getenv(ENV_EMBEDDINGS_LOCAL_MODEL, DEFAULT_EMBEDDINGS_LOCAL_MODEL),
|
||||
embeddings_local_force_cpu=os.getenv(
|
||||
ENV_EMBEDDINGS_LOCAL_FORCE_CPU, str(DEFAULT_EMBEDDINGS_LOCAL_FORCE_CPU)
|
||||
).lower()
|
||||
in ("true", "1"),
|
||||
embeddings_tei_url=os.getenv(ENV_EMBEDDINGS_TEI_URL),
|
||||
embeddings_openai_base_url=os.getenv(ENV_EMBEDDINGS_OPENAI_BASE_URL) or None,
|
||||
embeddings_cohere_base_url=os.getenv(ENV_EMBEDDINGS_COHERE_BASE_URL) or None,
|
||||
# Reranker
|
||||
reranker_provider=os.getenv(ENV_RERANKER_PROVIDER, DEFAULT_RERANKER_PROVIDER),
|
||||
reranker_local_model=os.getenv(ENV_RERANKER_LOCAL_MODEL, DEFAULT_RERANKER_LOCAL_MODEL),
|
||||
reranker_local_force_cpu=os.getenv(
|
||||
ENV_RERANKER_LOCAL_FORCE_CPU, str(DEFAULT_RERANKER_LOCAL_FORCE_CPU)
|
||||
).lower()
|
||||
in ("true", "1"),
|
||||
reranker_local_max_concurrent=int(
|
||||
os.getenv(ENV_RERANKER_LOCAL_MAX_CONCURRENT, str(DEFAULT_RERANKER_LOCAL_MAX_CONCURRENT))
|
||||
),
|
||||
reranker_tei_url=os.getenv(ENV_RERANKER_TEI_URL),
|
||||
reranker_tei_batch_size=int(os.getenv(ENV_RERANKER_TEI_BATCH_SIZE, str(DEFAULT_RERANKER_TEI_BATCH_SIZE))),
|
||||
reranker_tei_max_concurrent=int(
|
||||
|
|
|
|||
|
|
@ -20,6 +20,7 @@ from ..config import (
|
|||
DEFAULT_RERANKER_FLASHRANK_CACHE_DIR,
|
||||
DEFAULT_RERANKER_FLASHRANK_MODEL,
|
||||
DEFAULT_RERANKER_LITELLM_MODEL,
|
||||
DEFAULT_RERANKER_LOCAL_FORCE_CPU,
|
||||
DEFAULT_RERANKER_LOCAL_MAX_CONCURRENT,
|
||||
DEFAULT_RERANKER_LOCAL_MODEL,
|
||||
DEFAULT_RERANKER_PROVIDER,
|
||||
|
|
@ -33,6 +34,7 @@ from ..config import (
|
|||
ENV_RERANKER_FLASHRANK_CACHE_DIR,
|
||||
ENV_RERANKER_FLASHRANK_MODEL,
|
||||
ENV_RERANKER_LITELLM_MODEL,
|
||||
ENV_RERANKER_LOCAL_FORCE_CPU,
|
||||
ENV_RERANKER_LOCAL_MAX_CONCURRENT,
|
||||
ENV_RERANKER_LOCAL_MODEL,
|
||||
ENV_RERANKER_PROVIDER,
|
||||
|
|
@ -99,7 +101,7 @@ class LocalSTCrossEncoder(CrossEncoderModel):
|
|||
_executor: ThreadPoolExecutor | None = None
|
||||
_max_concurrent: int = 4 # Limit concurrent CPU-bound reranking calls
|
||||
|
||||
def __init__(self, model_name: str | None = None, max_concurrent: int = 4):
|
||||
def __init__(self, model_name: str | None = None, max_concurrent: int = 4, force_cpu: bool = False):
|
||||
"""
|
||||
Initialize local SentenceTransformers cross-encoder.
|
||||
|
||||
|
|
@ -108,8 +110,11 @@ class LocalSTCrossEncoder(CrossEncoderModel):
|
|||
Default: cross-encoder/ms-marco-MiniLM-L-6-v2
|
||||
max_concurrent: Maximum concurrent reranking calls (default: 2).
|
||||
Higher values may cause CPU thrashing under load.
|
||||
force_cpu: Force CPU mode (avoids MPS/XPC issues on macOS in daemon mode).
|
||||
Default: False
|
||||
"""
|
||||
self.model_name = model_name or DEFAULT_RERANKER_LOCAL_MODEL
|
||||
self.force_cpu = force_cpu
|
||||
self._model = None
|
||||
LocalSTCrossEncoder._max_concurrent = max_concurrent
|
||||
|
||||
|
|
@ -139,13 +144,23 @@ class LocalSTCrossEncoder(CrossEncoderModel):
|
|||
# after loading, which conflicts with accelerate's device_map handling.
|
||||
import torch
|
||||
|
||||
# Check for GPU (CUDA) or Apple Silicon (MPS)
|
||||
has_gpu = torch.cuda.is_available() or (hasattr(torch.backends, "mps") and torch.backends.mps.is_available())
|
||||
|
||||
if has_gpu:
|
||||
device = None # Let sentence-transformers auto-detect GPU/MPS
|
||||
else:
|
||||
# Force CPU mode if configured (used in daemon mode to avoid MPS/XPC issues on macOS)
|
||||
if self.force_cpu:
|
||||
device = "cpu"
|
||||
logger.info("Reranker: forcing CPU mode (HINDSIGHT_API_RERANKER_LOCAL_FORCE_CPU=1)")
|
||||
else:
|
||||
# Check for GPU (CUDA) or Apple Silicon (MPS)
|
||||
# Wrap in try-except to gracefully handle any device detection issues
|
||||
# (e.g., in CI environments or when PyTorch is built without GPU support)
|
||||
device = "cpu" # Default to CPU
|
||||
try:
|
||||
has_gpu = torch.cuda.is_available() or (
|
||||
hasattr(torch.backends, "mps") and torch.backends.mps.is_available()
|
||||
)
|
||||
if has_gpu:
|
||||
device = None # Let sentence-transformers auto-detect GPU/MPS
|
||||
except Exception as e:
|
||||
logger.warning(f"Failed to detect GPU/MPS, falling back to CPU: {e}")
|
||||
|
||||
self._model = CrossEncoder(
|
||||
self.model_name,
|
||||
|
|
@ -211,12 +226,19 @@ class LocalSTCrossEncoder(CrossEncoderModel):
|
|||
)
|
||||
|
||||
# Determine device based on hardware availability
|
||||
has_gpu = torch.cuda.is_available() or (hasattr(torch.backends, "mps") and torch.backends.mps.is_available())
|
||||
|
||||
if has_gpu:
|
||||
device = None # Let sentence-transformers auto-detect GPU/MPS
|
||||
else:
|
||||
if self.force_cpu:
|
||||
device = "cpu"
|
||||
else:
|
||||
# Wrap in try-except to gracefully handle any device detection issues
|
||||
device = "cpu" # Default to CPU
|
||||
try:
|
||||
has_gpu = torch.cuda.is_available() or (
|
||||
hasattr(torch.backends, "mps") and torch.backends.mps.is_available()
|
||||
)
|
||||
if has_gpu:
|
||||
device = None # Let sentence-transformers auto-detect GPU/MPS
|
||||
except Exception as e:
|
||||
logger.warning(f"Failed to detect GPU/MPS during reinit, falling back to CPU: {e}")
|
||||
|
||||
self._model = CrossEncoder(
|
||||
self.model_name,
|
||||
|
|
@ -873,29 +895,33 @@ class LiteLLMCrossEncoder(CrossEncoderModel):
|
|||
|
||||
def create_cross_encoder_from_env() -> CrossEncoderModel:
|
||||
"""
|
||||
Create a CrossEncoderModel instance based on environment variables.
|
||||
Create a CrossEncoderModel instance based on configuration.
|
||||
|
||||
See hindsight_api.config for environment variable names and defaults.
|
||||
Reads configuration via get_config() to ensure consistency across the codebase.
|
||||
|
||||
Returns:
|
||||
Configured CrossEncoderModel instance
|
||||
"""
|
||||
provider = os.environ.get(ENV_RERANKER_PROVIDER, DEFAULT_RERANKER_PROVIDER).lower()
|
||||
from ..config import get_config
|
||||
|
||||
config = get_config()
|
||||
provider = config.reranker_provider.lower()
|
||||
|
||||
if provider == "tei":
|
||||
url = os.environ.get(ENV_RERANKER_TEI_URL)
|
||||
url = config.reranker_tei_url
|
||||
if not url:
|
||||
raise ValueError(f"{ENV_RERANKER_TEI_URL} is required when {ENV_RERANKER_PROVIDER} is 'tei'")
|
||||
batch_size = int(os.environ.get(ENV_RERANKER_TEI_BATCH_SIZE, str(DEFAULT_RERANKER_TEI_BATCH_SIZE)))
|
||||
max_concurrent = int(os.environ.get(ENV_RERANKER_TEI_MAX_CONCURRENT, str(DEFAULT_RERANKER_TEI_MAX_CONCURRENT)))
|
||||
return RemoteTEICrossEncoder(base_url=url, batch_size=batch_size, max_concurrent=max_concurrent)
|
||||
elif provider == "local":
|
||||
model = os.environ.get(ENV_RERANKER_LOCAL_MODEL)
|
||||
model_name = model or DEFAULT_RERANKER_LOCAL_MODEL
|
||||
max_concurrent = int(
|
||||
os.environ.get(ENV_RERANKER_LOCAL_MAX_CONCURRENT, str(DEFAULT_RERANKER_LOCAL_MAX_CONCURRENT))
|
||||
return RemoteTEICrossEncoder(
|
||||
base_url=url,
|
||||
batch_size=config.reranker_tei_batch_size,
|
||||
max_concurrent=config.reranker_tei_max_concurrent,
|
||||
)
|
||||
elif provider == "local":
|
||||
return LocalSTCrossEncoder(
|
||||
model_name=config.reranker_local_model,
|
||||
max_concurrent=config.reranker_local_max_concurrent,
|
||||
force_cpu=config.reranker_local_force_cpu,
|
||||
)
|
||||
return LocalSTCrossEncoder(model_name=model_name, max_concurrent=max_concurrent)
|
||||
elif provider == "cohere":
|
||||
api_key = os.environ.get(ENV_COHERE_API_KEY)
|
||||
if not api_key:
|
||||
|
|
|
|||
|
|
@ -18,6 +18,7 @@ import httpx
|
|||
from ..config import (
|
||||
DEFAULT_EMBEDDINGS_COHERE_MODEL,
|
||||
DEFAULT_EMBEDDINGS_LITELLM_MODEL,
|
||||
DEFAULT_EMBEDDINGS_LOCAL_FORCE_CPU,
|
||||
DEFAULT_EMBEDDINGS_LOCAL_MODEL,
|
||||
DEFAULT_EMBEDDINGS_OPENAI_MODEL,
|
||||
DEFAULT_EMBEDDINGS_PROVIDER,
|
||||
|
|
@ -26,6 +27,7 @@ from ..config import (
|
|||
ENV_EMBEDDINGS_COHERE_BASE_URL,
|
||||
ENV_EMBEDDINGS_COHERE_MODEL,
|
||||
ENV_EMBEDDINGS_LITELLM_MODEL,
|
||||
ENV_EMBEDDINGS_LOCAL_FORCE_CPU,
|
||||
ENV_EMBEDDINGS_LOCAL_MODEL,
|
||||
ENV_EMBEDDINGS_OPENAI_API_KEY,
|
||||
ENV_EMBEDDINGS_OPENAI_BASE_URL,
|
||||
|
|
@ -92,15 +94,18 @@ class LocalSTEmbeddings(Embeddings):
|
|||
The embedding dimension is auto-detected from the model.
|
||||
"""
|
||||
|
||||
def __init__(self, model_name: str | None = None):
|
||||
def __init__(self, model_name: str | None = None, force_cpu: bool = False):
|
||||
"""
|
||||
Initialize local SentenceTransformers embeddings.
|
||||
|
||||
Args:
|
||||
model_name: Name of the SentenceTransformer model to use.
|
||||
Default: BAAI/bge-small-en-v1.5
|
||||
force_cpu: Force CPU mode (avoids MPS/XPC issues on macOS in daemon mode).
|
||||
Default: False
|
||||
"""
|
||||
self.model_name = model_name or DEFAULT_EMBEDDINGS_LOCAL_MODEL
|
||||
self.force_cpu = force_cpu
|
||||
self._model = None
|
||||
self._dimension: int | None = None
|
||||
|
||||
|
|
@ -134,13 +139,23 @@ class LocalSTEmbeddings(Embeddings):
|
|||
# which can cause issues when accelerate is installed but no GPU is available.
|
||||
import torch
|
||||
|
||||
# Check for GPU (CUDA) or Apple Silicon (MPS)
|
||||
has_gpu = torch.cuda.is_available() or (hasattr(torch.backends, "mps") and torch.backends.mps.is_available())
|
||||
|
||||
if has_gpu:
|
||||
device = None # Let sentence-transformers auto-detect GPU/MPS
|
||||
else:
|
||||
# Force CPU mode if configured (used in daemon mode to avoid MPS/XPC issues on macOS)
|
||||
if self.force_cpu:
|
||||
device = "cpu"
|
||||
logger.info("Embeddings: forcing CPU mode")
|
||||
else:
|
||||
# Check for GPU (CUDA) or Apple Silicon (MPS)
|
||||
# Wrap in try-except to gracefully handle any device detection issues
|
||||
# (e.g., in CI environments or when PyTorch is built without GPU support)
|
||||
device = "cpu" # Default to CPU
|
||||
try:
|
||||
has_gpu = torch.cuda.is_available() or (
|
||||
hasattr(torch.backends, "mps") and torch.backends.mps.is_available()
|
||||
)
|
||||
if has_gpu:
|
||||
device = None # Let sentence-transformers auto-detect GPU/MPS
|
||||
except Exception as e:
|
||||
logger.warning(f"Failed to detect GPU/MPS, falling back to CPU: {e}")
|
||||
|
||||
self._model = SentenceTransformer(
|
||||
self.model_name,
|
||||
|
|
@ -199,12 +214,19 @@ class LocalSTEmbeddings(Embeddings):
|
|||
)
|
||||
|
||||
# Determine device based on hardware availability
|
||||
has_gpu = torch.cuda.is_available() or (hasattr(torch.backends, "mps") and torch.backends.mps.is_available())
|
||||
|
||||
if has_gpu:
|
||||
device = None # Let sentence-transformers auto-detect GPU/MPS
|
||||
else:
|
||||
if self.force_cpu:
|
||||
device = "cpu"
|
||||
else:
|
||||
# Wrap in try-except to gracefully handle any device detection issues
|
||||
device = "cpu" # Default to CPU
|
||||
try:
|
||||
has_gpu = torch.cuda.is_available() or (
|
||||
hasattr(torch.backends, "mps") and torch.backends.mps.is_available()
|
||||
)
|
||||
if has_gpu:
|
||||
device = None # Let sentence-transformers auto-detect GPU/MPS
|
||||
except Exception as e:
|
||||
logger.warning(f"Failed to detect GPU/MPS during reinit, falling back to CPU: {e}")
|
||||
|
||||
self._model = SentenceTransformer(
|
||||
self.model_name,
|
||||
|
|
@ -770,24 +792,28 @@ class LiteLLMEmbeddings(Embeddings):
|
|||
|
||||
def create_embeddings_from_env() -> Embeddings:
|
||||
"""
|
||||
Create an Embeddings instance based on environment variables.
|
||||
Create an Embeddings instance based on configuration.
|
||||
|
||||
See hindsight_api.config for environment variable names and defaults.
|
||||
Reads configuration via get_config() to ensure consistency across the codebase.
|
||||
|
||||
Returns:
|
||||
Configured Embeddings instance
|
||||
"""
|
||||
provider = os.environ.get(ENV_EMBEDDINGS_PROVIDER, DEFAULT_EMBEDDINGS_PROVIDER).lower()
|
||||
from ..config import get_config
|
||||
|
||||
config = get_config()
|
||||
provider = config.embeddings_provider.lower()
|
||||
|
||||
if provider == "tei":
|
||||
url = os.environ.get(ENV_EMBEDDINGS_TEI_URL)
|
||||
url = config.embeddings_tei_url
|
||||
if not url:
|
||||
raise ValueError(f"{ENV_EMBEDDINGS_TEI_URL} is required when {ENV_EMBEDDINGS_PROVIDER} is 'tei'")
|
||||
return RemoteTEIEmbeddings(base_url=url)
|
||||
elif provider == "local":
|
||||
model = os.environ.get(ENV_EMBEDDINGS_LOCAL_MODEL)
|
||||
model_name = model or DEFAULT_EMBEDDINGS_LOCAL_MODEL
|
||||
return LocalSTEmbeddings(model_name=model_name)
|
||||
return LocalSTEmbeddings(
|
||||
model_name=config.embeddings_local_model,
|
||||
force_cpu=config.embeddings_local_force_cpu,
|
||||
)
|
||||
elif provider == "openai":
|
||||
# Use dedicated embeddings API key, or fall back to LLM API key
|
||||
api_key = os.environ.get(ENV_EMBEDDINGS_OPENAI_API_KEY) or os.environ.get(ENV_LLM_API_KEY)
|
||||
|
|
|
|||
|
|
@ -140,6 +140,13 @@ def main():
|
|||
args.port = DEFAULT_DAEMON_PORT
|
||||
args.host = "127.0.0.1" # Only bind to localhost for security
|
||||
|
||||
# Force CPU mode for daemon to avoid macOS MPS/XPC issues
|
||||
# MPS (Metal Performance Shaders) has unstable XPC connections in background processes
|
||||
# that can cause assertion failures and process crashes at the C++ level
|
||||
# (which Python exception handlers cannot catch)
|
||||
os.environ["HINDSIGHT_API_EMBEDDINGS_LOCAL_FORCE_CPU"] = "1"
|
||||
os.environ["HINDSIGHT_API_RERANKER_LOCAL_FORCE_CPU"] = "1"
|
||||
|
||||
# Check if another daemon is already running
|
||||
daemon_lock = DaemonLock()
|
||||
if not daemon_lock.acquire():
|
||||
|
|
@ -191,11 +198,14 @@ def main():
|
|||
consolidation_llm_base_url=config.consolidation_llm_base_url,
|
||||
embeddings_provider=config.embeddings_provider,
|
||||
embeddings_local_model=config.embeddings_local_model,
|
||||
embeddings_local_force_cpu=config.embeddings_local_force_cpu,
|
||||
embeddings_tei_url=config.embeddings_tei_url,
|
||||
embeddings_openai_base_url=config.embeddings_openai_base_url,
|
||||
embeddings_cohere_base_url=config.embeddings_cohere_base_url,
|
||||
reranker_provider=config.reranker_provider,
|
||||
reranker_local_model=config.reranker_local_model,
|
||||
reranker_local_force_cpu=config.reranker_local_force_cpu,
|
||||
reranker_local_max_concurrent=config.reranker_local_max_concurrent,
|
||||
reranker_tei_url=config.reranker_tei_url,
|
||||
reranker_tei_batch_size=config.reranker_tei_batch_size,
|
||||
reranker_tei_max_concurrent=config.reranker_tei_max_concurrent,
|
||||
|
|
|
|||
|
|
@ -527,6 +527,7 @@ class TestRemoteTEICrossEncoderConfig:
|
|||
"""Test creating encoder from environment variables."""
|
||||
import os
|
||||
|
||||
from hindsight_api.config import clear_config_cache
|
||||
from hindsight_api.engine.cross_encoder import create_cross_encoder_from_env
|
||||
|
||||
with patch.dict(
|
||||
|
|
@ -538,6 +539,7 @@ class TestRemoteTEICrossEncoderConfig:
|
|||
"HINDSIGHT_API_RERANKER_TEI_MAX_CONCURRENT": "16",
|
||||
},
|
||||
):
|
||||
clear_config_cache() # Clear cache to pick up patched env vars
|
||||
encoder = create_cross_encoder_from_env()
|
||||
|
||||
assert isinstance(encoder, RemoteTEICrossEncoder)
|
||||
|
|
@ -545,6 +547,8 @@ class TestRemoteTEICrossEncoderConfig:
|
|||
assert encoder.batch_size == 256
|
||||
assert encoder.max_concurrent == 16
|
||||
|
||||
clear_config_cache() # Clear cache after test
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# TEI Reranker Performance Benchmark Tests
|
||||
|
|
|
|||
Loading…
Reference in a new issue