holetron-lab/fleet-memory
1
0
Fork 0
Code Releases Packages Activity Actions

fix: pytorch init failures (#175)

Browse source
This commit is contained in:
Nicolò Boschi 2026-01-19 15:02:20 +01:00 committed by GitHub
parent fe4ed1db73
commit 0284595909
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
1 changed files with 6 additions and 1 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

7
hindsight-api/hindsight_api/engine/cross_encoder.py
View file

@ -136,7 +136,12 @@ class LocalSTCrossEncoder(CrossEncoderModel):
# 3. Data transfer overhead to GPU outweighs compute benefit
# 4. CPU inference is actually faster for this workload
logger.info(f"Reranker: initializing local provider with model {self.model_name}")
self._model = CrossEncoder(self.model_name)
# Disable lazy loading (meta tensors) which causes issues with newer transformers/accelerate.
# Setting low_cpu_mem_usage=False and device_map=None ensures tensors are fully materialized.
self._model = CrossEncoder(
self.model_name,
model_kwargs={"low_cpu_mem_usage": False, "device_map": None},
)
# Initialize shared executor (limited workers naturally limits concurrency)
if LocalSTCrossEncoder._executor is None: