diff --git a/hindsight-api/hindsight_api/config.py b/hindsight-api/hindsight_api/config.py index 5461aef9..f291ec01 100644 --- a/hindsight-api/hindsight_api/config.py +++ b/hindsight-api/hindsight_api/config.py @@ -47,6 +47,7 @@ ENV_CONSOLIDATION_LLM_BASE_URL = "HINDSIGHT_API_CONSOLIDATION_LLM_BASE_URL" ENV_EMBEDDINGS_PROVIDER = "HINDSIGHT_API_EMBEDDINGS_PROVIDER" ENV_EMBEDDINGS_LOCAL_MODEL = "HINDSIGHT_API_EMBEDDINGS_LOCAL_MODEL" +ENV_EMBEDDINGS_LOCAL_FORCE_CPU = "HINDSIGHT_API_EMBEDDINGS_LOCAL_FORCE_CPU" ENV_EMBEDDINGS_TEI_URL = "HINDSIGHT_API_EMBEDDINGS_TEI_URL" ENV_EMBEDDINGS_OPENAI_API_KEY = "HINDSIGHT_API_EMBEDDINGS_OPENAI_API_KEY" ENV_EMBEDDINGS_OPENAI_MODEL = "HINDSIGHT_API_EMBEDDINGS_OPENAI_MODEL" @@ -66,6 +67,7 @@ ENV_RERANKER_LITELLM_MODEL = "HINDSIGHT_API_RERANKER_LITELLM_MODEL" ENV_RERANKER_PROVIDER = "HINDSIGHT_API_RERANKER_PROVIDER" ENV_RERANKER_LOCAL_MODEL = "HINDSIGHT_API_RERANKER_LOCAL_MODEL" +ENV_RERANKER_LOCAL_FORCE_CPU = "HINDSIGHT_API_RERANKER_LOCAL_FORCE_CPU" ENV_RERANKER_LOCAL_MAX_CONCURRENT = "HINDSIGHT_API_RERANKER_LOCAL_MAX_CONCURRENT" ENV_RERANKER_TEI_URL = "HINDSIGHT_API_RERANKER_TEI_URL" ENV_RERANKER_TEI_BATCH_SIZE = "HINDSIGHT_API_RERANKER_TEI_BATCH_SIZE" @@ -134,11 +136,13 @@ DEFAULT_LLM_TIMEOUT = 120.0 # seconds DEFAULT_EMBEDDINGS_PROVIDER = "local" DEFAULT_EMBEDDINGS_LOCAL_MODEL = "BAAI/bge-small-en-v1.5" +DEFAULT_EMBEDDINGS_LOCAL_FORCE_CPU = False # Force CPU mode for local embeddings (avoids MPS/XPC issues on macOS) DEFAULT_EMBEDDINGS_OPENAI_MODEL = "text-embedding-3-small" DEFAULT_EMBEDDING_DIMENSION = 384 DEFAULT_RERANKER_PROVIDER = "local" DEFAULT_RERANKER_LOCAL_MODEL = "cross-encoder/ms-marco-MiniLM-L-6-v2" +DEFAULT_RERANKER_LOCAL_FORCE_CPU = False # Force CPU mode for local reranker (avoids MPS/XPC issues on macOS) DEFAULT_RERANKER_LOCAL_MAX_CONCURRENT = 4 # Limit concurrent CPU-bound reranking to prevent thrashing DEFAULT_RERANKER_TEI_BATCH_SIZE = 128 DEFAULT_RERANKER_TEI_MAX_CONCURRENT = 8 @@ -301,6 +305,7 @@ class HindsightConfig: # Embeddings embeddings_provider: str embeddings_local_model: str + embeddings_local_force_cpu: bool embeddings_tei_url: str | None embeddings_openai_base_url: str | None embeddings_cohere_base_url: str | None @@ -308,6 +313,8 @@ class HindsightConfig: # Reranker reranker_provider: str reranker_local_model: str + reranker_local_force_cpu: bool + reranker_local_max_concurrent: int reranker_tei_url: str | None reranker_tei_batch_size: int reranker_tei_max_concurrent: int @@ -394,12 +401,23 @@ class HindsightConfig: # Embeddings embeddings_provider=os.getenv(ENV_EMBEDDINGS_PROVIDER, DEFAULT_EMBEDDINGS_PROVIDER), embeddings_local_model=os.getenv(ENV_EMBEDDINGS_LOCAL_MODEL, DEFAULT_EMBEDDINGS_LOCAL_MODEL), + embeddings_local_force_cpu=os.getenv( + ENV_EMBEDDINGS_LOCAL_FORCE_CPU, str(DEFAULT_EMBEDDINGS_LOCAL_FORCE_CPU) + ).lower() + in ("true", "1"), embeddings_tei_url=os.getenv(ENV_EMBEDDINGS_TEI_URL), embeddings_openai_base_url=os.getenv(ENV_EMBEDDINGS_OPENAI_BASE_URL) or None, embeddings_cohere_base_url=os.getenv(ENV_EMBEDDINGS_COHERE_BASE_URL) or None, # Reranker reranker_provider=os.getenv(ENV_RERANKER_PROVIDER, DEFAULT_RERANKER_PROVIDER), reranker_local_model=os.getenv(ENV_RERANKER_LOCAL_MODEL, DEFAULT_RERANKER_LOCAL_MODEL), + reranker_local_force_cpu=os.getenv( + ENV_RERANKER_LOCAL_FORCE_CPU, str(DEFAULT_RERANKER_LOCAL_FORCE_CPU) + ).lower() + in ("true", "1"), + reranker_local_max_concurrent=int( + os.getenv(ENV_RERANKER_LOCAL_MAX_CONCURRENT, str(DEFAULT_RERANKER_LOCAL_MAX_CONCURRENT)) + ), reranker_tei_url=os.getenv(ENV_RERANKER_TEI_URL), reranker_tei_batch_size=int(os.getenv(ENV_RERANKER_TEI_BATCH_SIZE, str(DEFAULT_RERANKER_TEI_BATCH_SIZE))), reranker_tei_max_concurrent=int( diff --git a/hindsight-api/hindsight_api/engine/cross_encoder.py b/hindsight-api/hindsight_api/engine/cross_encoder.py index 7d70c147..4d91bdb1 100644 --- a/hindsight-api/hindsight_api/engine/cross_encoder.py +++ b/hindsight-api/hindsight_api/engine/cross_encoder.py @@ -20,6 +20,7 @@ from ..config import ( DEFAULT_RERANKER_FLASHRANK_CACHE_DIR, DEFAULT_RERANKER_FLASHRANK_MODEL, DEFAULT_RERANKER_LITELLM_MODEL, + DEFAULT_RERANKER_LOCAL_FORCE_CPU, DEFAULT_RERANKER_LOCAL_MAX_CONCURRENT, DEFAULT_RERANKER_LOCAL_MODEL, DEFAULT_RERANKER_PROVIDER, @@ -33,6 +34,7 @@ from ..config import ( ENV_RERANKER_FLASHRANK_CACHE_DIR, ENV_RERANKER_FLASHRANK_MODEL, ENV_RERANKER_LITELLM_MODEL, + ENV_RERANKER_LOCAL_FORCE_CPU, ENV_RERANKER_LOCAL_MAX_CONCURRENT, ENV_RERANKER_LOCAL_MODEL, ENV_RERANKER_PROVIDER, @@ -99,7 +101,7 @@ class LocalSTCrossEncoder(CrossEncoderModel): _executor: ThreadPoolExecutor | None = None _max_concurrent: int = 4 # Limit concurrent CPU-bound reranking calls - def __init__(self, model_name: str | None = None, max_concurrent: int = 4): + def __init__(self, model_name: str | None = None, max_concurrent: int = 4, force_cpu: bool = False): """ Initialize local SentenceTransformers cross-encoder. @@ -108,8 +110,11 @@ class LocalSTCrossEncoder(CrossEncoderModel): Default: cross-encoder/ms-marco-MiniLM-L-6-v2 max_concurrent: Maximum concurrent reranking calls (default: 2). Higher values may cause CPU thrashing under load. + force_cpu: Force CPU mode (avoids MPS/XPC issues on macOS in daemon mode). + Default: False """ self.model_name = model_name or DEFAULT_RERANKER_LOCAL_MODEL + self.force_cpu = force_cpu self._model = None LocalSTCrossEncoder._max_concurrent = max_concurrent @@ -139,13 +144,23 @@ class LocalSTCrossEncoder(CrossEncoderModel): # after loading, which conflicts with accelerate's device_map handling. import torch - # Check for GPU (CUDA) or Apple Silicon (MPS) - has_gpu = torch.cuda.is_available() or (hasattr(torch.backends, "mps") and torch.backends.mps.is_available()) - - if has_gpu: - device = None # Let sentence-transformers auto-detect GPU/MPS - else: + # Force CPU mode if configured (used in daemon mode to avoid MPS/XPC issues on macOS) + if self.force_cpu: device = "cpu" + logger.info("Reranker: forcing CPU mode (HINDSIGHT_API_RERANKER_LOCAL_FORCE_CPU=1)") + else: + # Check for GPU (CUDA) or Apple Silicon (MPS) + # Wrap in try-except to gracefully handle any device detection issues + # (e.g., in CI environments or when PyTorch is built without GPU support) + device = "cpu" # Default to CPU + try: + has_gpu = torch.cuda.is_available() or ( + hasattr(torch.backends, "mps") and torch.backends.mps.is_available() + ) + if has_gpu: + device = None # Let sentence-transformers auto-detect GPU/MPS + except Exception as e: + logger.warning(f"Failed to detect GPU/MPS, falling back to CPU: {e}") self._model = CrossEncoder( self.model_name, @@ -211,12 +226,19 @@ class LocalSTCrossEncoder(CrossEncoderModel): ) # Determine device based on hardware availability - has_gpu = torch.cuda.is_available() or (hasattr(torch.backends, "mps") and torch.backends.mps.is_available()) - - if has_gpu: - device = None # Let sentence-transformers auto-detect GPU/MPS - else: + if self.force_cpu: device = "cpu" + else: + # Wrap in try-except to gracefully handle any device detection issues + device = "cpu" # Default to CPU + try: + has_gpu = torch.cuda.is_available() or ( + hasattr(torch.backends, "mps") and torch.backends.mps.is_available() + ) + if has_gpu: + device = None # Let sentence-transformers auto-detect GPU/MPS + except Exception as e: + logger.warning(f"Failed to detect GPU/MPS during reinit, falling back to CPU: {e}") self._model = CrossEncoder( self.model_name, @@ -873,29 +895,33 @@ class LiteLLMCrossEncoder(CrossEncoderModel): def create_cross_encoder_from_env() -> CrossEncoderModel: """ - Create a CrossEncoderModel instance based on environment variables. + Create a CrossEncoderModel instance based on configuration. - See hindsight_api.config for environment variable names and defaults. + Reads configuration via get_config() to ensure consistency across the codebase. Returns: Configured CrossEncoderModel instance """ - provider = os.environ.get(ENV_RERANKER_PROVIDER, DEFAULT_RERANKER_PROVIDER).lower() + from ..config import get_config + + config = get_config() + provider = config.reranker_provider.lower() if provider == "tei": - url = os.environ.get(ENV_RERANKER_TEI_URL) + url = config.reranker_tei_url if not url: raise ValueError(f"{ENV_RERANKER_TEI_URL} is required when {ENV_RERANKER_PROVIDER} is 'tei'") - batch_size = int(os.environ.get(ENV_RERANKER_TEI_BATCH_SIZE, str(DEFAULT_RERANKER_TEI_BATCH_SIZE))) - max_concurrent = int(os.environ.get(ENV_RERANKER_TEI_MAX_CONCURRENT, str(DEFAULT_RERANKER_TEI_MAX_CONCURRENT))) - return RemoteTEICrossEncoder(base_url=url, batch_size=batch_size, max_concurrent=max_concurrent) - elif provider == "local": - model = os.environ.get(ENV_RERANKER_LOCAL_MODEL) - model_name = model or DEFAULT_RERANKER_LOCAL_MODEL - max_concurrent = int( - os.environ.get(ENV_RERANKER_LOCAL_MAX_CONCURRENT, str(DEFAULT_RERANKER_LOCAL_MAX_CONCURRENT)) + return RemoteTEICrossEncoder( + base_url=url, + batch_size=config.reranker_tei_batch_size, + max_concurrent=config.reranker_tei_max_concurrent, + ) + elif provider == "local": + return LocalSTCrossEncoder( + model_name=config.reranker_local_model, + max_concurrent=config.reranker_local_max_concurrent, + force_cpu=config.reranker_local_force_cpu, ) - return LocalSTCrossEncoder(model_name=model_name, max_concurrent=max_concurrent) elif provider == "cohere": api_key = os.environ.get(ENV_COHERE_API_KEY) if not api_key: diff --git a/hindsight-api/hindsight_api/engine/embeddings.py b/hindsight-api/hindsight_api/engine/embeddings.py index dddee9d5..af60cc73 100644 --- a/hindsight-api/hindsight_api/engine/embeddings.py +++ b/hindsight-api/hindsight_api/engine/embeddings.py @@ -18,6 +18,7 @@ import httpx from ..config import ( DEFAULT_EMBEDDINGS_COHERE_MODEL, DEFAULT_EMBEDDINGS_LITELLM_MODEL, + DEFAULT_EMBEDDINGS_LOCAL_FORCE_CPU, DEFAULT_EMBEDDINGS_LOCAL_MODEL, DEFAULT_EMBEDDINGS_OPENAI_MODEL, DEFAULT_EMBEDDINGS_PROVIDER, @@ -26,6 +27,7 @@ from ..config import ( ENV_EMBEDDINGS_COHERE_BASE_URL, ENV_EMBEDDINGS_COHERE_MODEL, ENV_EMBEDDINGS_LITELLM_MODEL, + ENV_EMBEDDINGS_LOCAL_FORCE_CPU, ENV_EMBEDDINGS_LOCAL_MODEL, ENV_EMBEDDINGS_OPENAI_API_KEY, ENV_EMBEDDINGS_OPENAI_BASE_URL, @@ -92,15 +94,18 @@ class LocalSTEmbeddings(Embeddings): The embedding dimension is auto-detected from the model. """ - def __init__(self, model_name: str | None = None): + def __init__(self, model_name: str | None = None, force_cpu: bool = False): """ Initialize local SentenceTransformers embeddings. Args: model_name: Name of the SentenceTransformer model to use. Default: BAAI/bge-small-en-v1.5 + force_cpu: Force CPU mode (avoids MPS/XPC issues on macOS in daemon mode). + Default: False """ self.model_name = model_name or DEFAULT_EMBEDDINGS_LOCAL_MODEL + self.force_cpu = force_cpu self._model = None self._dimension: int | None = None @@ -134,13 +139,23 @@ class LocalSTEmbeddings(Embeddings): # which can cause issues when accelerate is installed but no GPU is available. import torch - # Check for GPU (CUDA) or Apple Silicon (MPS) - has_gpu = torch.cuda.is_available() or (hasattr(torch.backends, "mps") and torch.backends.mps.is_available()) - - if has_gpu: - device = None # Let sentence-transformers auto-detect GPU/MPS - else: + # Force CPU mode if configured (used in daemon mode to avoid MPS/XPC issues on macOS) + if self.force_cpu: device = "cpu" + logger.info("Embeddings: forcing CPU mode") + else: + # Check for GPU (CUDA) or Apple Silicon (MPS) + # Wrap in try-except to gracefully handle any device detection issues + # (e.g., in CI environments or when PyTorch is built without GPU support) + device = "cpu" # Default to CPU + try: + has_gpu = torch.cuda.is_available() or ( + hasattr(torch.backends, "mps") and torch.backends.mps.is_available() + ) + if has_gpu: + device = None # Let sentence-transformers auto-detect GPU/MPS + except Exception as e: + logger.warning(f"Failed to detect GPU/MPS, falling back to CPU: {e}") self._model = SentenceTransformer( self.model_name, @@ -199,12 +214,19 @@ class LocalSTEmbeddings(Embeddings): ) # Determine device based on hardware availability - has_gpu = torch.cuda.is_available() or (hasattr(torch.backends, "mps") and torch.backends.mps.is_available()) - - if has_gpu: - device = None # Let sentence-transformers auto-detect GPU/MPS - else: + if self.force_cpu: device = "cpu" + else: + # Wrap in try-except to gracefully handle any device detection issues + device = "cpu" # Default to CPU + try: + has_gpu = torch.cuda.is_available() or ( + hasattr(torch.backends, "mps") and torch.backends.mps.is_available() + ) + if has_gpu: + device = None # Let sentence-transformers auto-detect GPU/MPS + except Exception as e: + logger.warning(f"Failed to detect GPU/MPS during reinit, falling back to CPU: {e}") self._model = SentenceTransformer( self.model_name, @@ -770,24 +792,28 @@ class LiteLLMEmbeddings(Embeddings): def create_embeddings_from_env() -> Embeddings: """ - Create an Embeddings instance based on environment variables. + Create an Embeddings instance based on configuration. - See hindsight_api.config for environment variable names and defaults. + Reads configuration via get_config() to ensure consistency across the codebase. Returns: Configured Embeddings instance """ - provider = os.environ.get(ENV_EMBEDDINGS_PROVIDER, DEFAULT_EMBEDDINGS_PROVIDER).lower() + from ..config import get_config + + config = get_config() + provider = config.embeddings_provider.lower() if provider == "tei": - url = os.environ.get(ENV_EMBEDDINGS_TEI_URL) + url = config.embeddings_tei_url if not url: raise ValueError(f"{ENV_EMBEDDINGS_TEI_URL} is required when {ENV_EMBEDDINGS_PROVIDER} is 'tei'") return RemoteTEIEmbeddings(base_url=url) elif provider == "local": - model = os.environ.get(ENV_EMBEDDINGS_LOCAL_MODEL) - model_name = model or DEFAULT_EMBEDDINGS_LOCAL_MODEL - return LocalSTEmbeddings(model_name=model_name) + return LocalSTEmbeddings( + model_name=config.embeddings_local_model, + force_cpu=config.embeddings_local_force_cpu, + ) elif provider == "openai": # Use dedicated embeddings API key, or fall back to LLM API key api_key = os.environ.get(ENV_EMBEDDINGS_OPENAI_API_KEY) or os.environ.get(ENV_LLM_API_KEY) diff --git a/hindsight-api/hindsight_api/main.py b/hindsight-api/hindsight_api/main.py index 695bd172..bd31c71e 100644 --- a/hindsight-api/hindsight_api/main.py +++ b/hindsight-api/hindsight_api/main.py @@ -140,6 +140,13 @@ def main(): args.port = DEFAULT_DAEMON_PORT args.host = "127.0.0.1" # Only bind to localhost for security + # Force CPU mode for daemon to avoid macOS MPS/XPC issues + # MPS (Metal Performance Shaders) has unstable XPC connections in background processes + # that can cause assertion failures and process crashes at the C++ level + # (which Python exception handlers cannot catch) + os.environ["HINDSIGHT_API_EMBEDDINGS_LOCAL_FORCE_CPU"] = "1" + os.environ["HINDSIGHT_API_RERANKER_LOCAL_FORCE_CPU"] = "1" + # Check if another daemon is already running daemon_lock = DaemonLock() if not daemon_lock.acquire(): @@ -191,11 +198,14 @@ def main(): consolidation_llm_base_url=config.consolidation_llm_base_url, embeddings_provider=config.embeddings_provider, embeddings_local_model=config.embeddings_local_model, + embeddings_local_force_cpu=config.embeddings_local_force_cpu, embeddings_tei_url=config.embeddings_tei_url, embeddings_openai_base_url=config.embeddings_openai_base_url, embeddings_cohere_base_url=config.embeddings_cohere_base_url, reranker_provider=config.reranker_provider, reranker_local_model=config.reranker_local_model, + reranker_local_force_cpu=config.reranker_local_force_cpu, + reranker_local_max_concurrent=config.reranker_local_max_concurrent, reranker_tei_url=config.reranker_tei_url, reranker_tei_batch_size=config.reranker_tei_batch_size, reranker_tei_max_concurrent=config.reranker_tei_max_concurrent, diff --git a/hindsight-api/tests/test_tei_cross_encoder.py b/hindsight-api/tests/test_tei_cross_encoder.py index ea3b1b2e..fcfb295b 100644 --- a/hindsight-api/tests/test_tei_cross_encoder.py +++ b/hindsight-api/tests/test_tei_cross_encoder.py @@ -527,6 +527,7 @@ class TestRemoteTEICrossEncoderConfig: """Test creating encoder from environment variables.""" import os + from hindsight_api.config import clear_config_cache from hindsight_api.engine.cross_encoder import create_cross_encoder_from_env with patch.dict( @@ -538,6 +539,7 @@ class TestRemoteTEICrossEncoderConfig: "HINDSIGHT_API_RERANKER_TEI_MAX_CONCURRENT": "16", }, ): + clear_config_cache() # Clear cache to pick up patched env vars encoder = create_cross_encoder_from_env() assert isinstance(encoder, RemoteTEICrossEncoder) @@ -545,6 +547,8 @@ class TestRemoteTEICrossEncoderConfig: assert encoder.batch_size == 256 assert encoder.max_concurrent == 16 + clear_config_cache() # Clear cache after test + # ============================================================================ # TEI Reranker Performance Benchmark Tests