From d2bfa84bca60cfb4dfc16c06eb4b62f758fb182c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=B2=20Boschi?= Date: Mon, 16 Mar 2026 17:35:09 +0100 Subject: [PATCH] docs: add config vars for local reranker FP16 and bucket batching (#589) * docs: add config vars for local reranker FP16 and bucket batching (#588) * fix: add missing reranker local fields to CLI config override and fix ty type error - Add reranker_local_fp16, reranker_local_bucket_batching, reranker_local_batch_size to the manual HindsightConfig() constructor call in main.py (CLI override block) - Replace direct module attribute assignment with setattr() in the transformers 5.x monkey-patch so ty can resolve it without raising unresolved-attribute --- .../hindsight_api/engine/cross_encoder.py | 15 +++++++++------ hindsight-api-slim/hindsight_api/main.py | 3 +++ hindsight-docs/docs/developer/configuration.md | 3 +++ 3 files changed, 15 insertions(+), 6 deletions(-) diff --git a/hindsight-api-slim/hindsight_api/engine/cross_encoder.py b/hindsight-api-slim/hindsight_api/engine/cross_encoder.py index f1ac9e0b..bc0609ca 100644 --- a/hindsight-api-slim/hindsight_api/engine/cross_encoder.py +++ b/hindsight-api-slim/hindsight_api/engine/cross_encoder.py @@ -195,10 +195,15 @@ class LocalSTCrossEncoder(CrossEncoderModel): # create_position_ids_from_input_ids as a module-level function; the custom # code in these models still references it. This monkey-patch restores it. try: - from transformers.models.xlm_roberta.modeling_xlm_roberta import XLMRobertaEmbeddings import transformers.models.xlm_roberta.modeling_xlm_roberta as xlm_module - if not hasattr(xlm_module, 'create_position_ids_from_input_ids'): - xlm_module.create_position_ids_from_input_ids = XLMRobertaEmbeddings.create_position_ids_from_input_ids + from transformers.models.xlm_roberta.modeling_xlm_roberta import XLMRobertaEmbeddings + + if not hasattr(xlm_module, "create_position_ids_from_input_ids"): + setattr( + xlm_module, + "create_position_ids_from_input_ids", + XLMRobertaEmbeddings.create_position_ids_from_input_ids, + ) logger.info("Reranker: applied transformers 5.x compatibility patch for XLM-RoBERTa") except Exception: pass @@ -260,9 +265,7 @@ class LocalSTCrossEncoder(CrossEncoderModel): sorted_indices = sorted(range(len(pairs)), key=lambda i: lengths[i]) sorted_pairs = [pairs[i] for i in sorted_indices] - sorted_scores = self._model.predict( - sorted_pairs, batch_size=self.batch_size, show_progress_bar=False - ) + sorted_scores = self._model.predict(sorted_pairs, batch_size=self.batch_size, show_progress_bar=False) sorted_scores = sorted_scores.tolist() if hasattr(sorted_scores, "tolist") else list(sorted_scores) # Restore original order diff --git a/hindsight-api-slim/hindsight_api/main.py b/hindsight-api-slim/hindsight_api/main.py index a48dac14..c0e80066 100644 --- a/hindsight-api-slim/hindsight_api/main.py +++ b/hindsight-api-slim/hindsight_api/main.py @@ -219,6 +219,9 @@ def main(): reranker_local_force_cpu=config.reranker_local_force_cpu, reranker_local_max_concurrent=config.reranker_local_max_concurrent, reranker_local_trust_remote_code=config.reranker_local_trust_remote_code, + reranker_local_fp16=config.reranker_local_fp16, + reranker_local_bucket_batching=config.reranker_local_bucket_batching, + reranker_local_batch_size=config.reranker_local_batch_size, reranker_tei_url=config.reranker_tei_url, reranker_tei_batch_size=config.reranker_tei_batch_size, reranker_tei_max_concurrent=config.reranker_tei_max_concurrent, diff --git a/hindsight-docs/docs/developer/configuration.md b/hindsight-docs/docs/developer/configuration.md index 012746b7..5fd4d939 100644 --- a/hindsight-docs/docs/developer/configuration.md +++ b/hindsight-docs/docs/developer/configuration.md @@ -414,6 +414,9 @@ Supported OpenAI embedding dimensions: | `HINDSIGHT_API_RERANKER_LOCAL_MODEL` | Model for local provider | `cross-encoder/ms-marco-MiniLM-L-6-v2` | | `HINDSIGHT_API_RERANKER_LOCAL_MAX_CONCURRENT` | Max concurrent local reranking (prevents CPU thrashing under load) | `4` | | `HINDSIGHT_API_RERANKER_LOCAL_TRUST_REMOTE_CODE` | Allow loading models with custom code (security risk, disabled by default) | `false` | +| `HINDSIGHT_API_RERANKER_LOCAL_FP16` | Half-precision (FP16) inference for the local reranker. 27–36% faster on MPS; quality-identical. Disabled by default to avoid regressions on non-MPS deployments — some CPUs lack native FP16 support. | `false` | +| `HINDSIGHT_API_RERANKER_LOCAL_BUCKET_BATCHING` | Sort pairs by token length before batching to reduce padding waste. 36–54% faster across models; quality-identical by construction. | `false` | +| `HINDSIGHT_API_RERANKER_LOCAL_BATCH_SIZE` | Batch size for local reranker `predict()`. Optimal value varies by hardware and model (smaller batches can outperform larger ones on MPS). | `32` | | `HINDSIGHT_API_RERANKER_TEI_URL` | TEI server URL | - | | `HINDSIGHT_API_RERANKER_TEI_BATCH_SIZE` | Batch size for TEI reranking | `128` | | `HINDSIGHT_API_RERANKER_TEI_MAX_CONCURRENT` | Max concurrent TEI reranking requests | `8` |