docs: add config vars for local reranker FP16 and bucket batching (#589)

* docs: add config vars for local reranker FP16 and bucket batching (#588)

* fix: add missing reranker local fields to CLI config override and fix ty type error

- Add reranker_local_fp16, reranker_local_bucket_batching, reranker_local_batch_size
  to the manual HindsightConfig() constructor call in main.py (CLI override block)
- Replace direct module attribute assignment with setattr() in the transformers 5.x
  monkey-patch so ty can resolve it without raising unresolved-attribute

hindsight-api-slim/hindsight_api/engine/cross_encoder.py

@@ -195,10 +195,15 @@ class LocalSTCrossEncoder(CrossEncoderModel):
         # create_position_ids_from_input_ids as a module-level function; the custom
         # code in these models still references it. This monkey-patch restores it.
         try:
-            from transformers.models.xlm_roberta.modeling_xlm_roberta import XLMRobertaEmbeddings
             import transformers.models.xlm_roberta.modeling_xlm_roberta as xlm_module
-            if not hasattr(xlm_module, 'create_position_ids_from_input_ids'):
-                xlm_module.create_position_ids_from_input_ids = XLMRobertaEmbeddings.create_position_ids_from_input_ids
+            from transformers.models.xlm_roberta.modeling_xlm_roberta import XLMRobertaEmbeddings
+
+            if not hasattr(xlm_module, "create_position_ids_from_input_ids"):
+                setattr(
+                    xlm_module,
+                    "create_position_ids_from_input_ids",
+                    XLMRobertaEmbeddings.create_position_ids_from_input_ids,
+                )
                 logger.info("Reranker: applied transformers 5.x compatibility patch for XLM-RoBERTa")
         except Exception:
             pass
@@ -260,9 +265,7 @@ class LocalSTCrossEncoder(CrossEncoderModel):
             sorted_indices = sorted(range(len(pairs)), key=lambda i: lengths[i])
             sorted_pairs = [pairs[i] for i in sorted_indices]
 
-            sorted_scores = self._model.predict(
-                sorted_pairs, batch_size=self.batch_size, show_progress_bar=False
-            )
+            sorted_scores = self._model.predict(sorted_pairs, batch_size=self.batch_size, show_progress_bar=False)
             sorted_scores = sorted_scores.tolist() if hasattr(sorted_scores, "tolist") else list(sorted_scores)
 
             # Restore original order

hindsight-api-slim/hindsight_api/main.py

@@ -219,6 +219,9 @@ def main():
             reranker_local_force_cpu=config.reranker_local_force_cpu,
             reranker_local_max_concurrent=config.reranker_local_max_concurrent,
             reranker_local_trust_remote_code=config.reranker_local_trust_remote_code,
+            reranker_local_fp16=config.reranker_local_fp16,
+            reranker_local_bucket_batching=config.reranker_local_bucket_batching,
+            reranker_local_batch_size=config.reranker_local_batch_size,
             reranker_tei_url=config.reranker_tei_url,
             reranker_tei_batch_size=config.reranker_tei_batch_size,
             reranker_tei_max_concurrent=config.reranker_tei_max_concurrent,

hindsight-docs/docs/developer/configuration.md

@@ -414,6 +414,9 @@ Supported OpenAI embedding dimensions:
 | `HINDSIGHT_API_RERANKER_LOCAL_MODEL` | Model for local provider | `cross-encoder/ms-marco-MiniLM-L-6-v2` |
 | `HINDSIGHT_API_RERANKER_LOCAL_MAX_CONCURRENT` | Max concurrent local reranking (prevents CPU thrashing under load) | `4` |
 | `HINDSIGHT_API_RERANKER_LOCAL_TRUST_REMOTE_CODE` | Allow loading models with custom code (security risk, disabled by default) | `false` |
+| `HINDSIGHT_API_RERANKER_LOCAL_FP16` | Half-precision (FP16) inference for the local reranker. 27–36% faster on MPS; quality-identical. Disabled by default to avoid regressions on non-MPS deployments — some CPUs lack native FP16 support. | `false` |
+| `HINDSIGHT_API_RERANKER_LOCAL_BUCKET_BATCHING` | Sort pairs by token length before batching to reduce padding waste. 36–54% faster across models; quality-identical by construction. | `false` |
+| `HINDSIGHT_API_RERANKER_LOCAL_BATCH_SIZE` | Batch size for local reranker `predict()`. Optimal value varies by hardware and model (smaller batches can outperform larger ones on MPS). | `32` |
 | `HINDSIGHT_API_RERANKER_TEI_URL` | TEI server URL | - |
 | `HINDSIGHT_API_RERANKER_TEI_BATCH_SIZE` | Batch size for TEI reranking | `128` |
 | `HINDSIGHT_API_RERANKER_TEI_MAX_CONCURRENT` | Max concurrent TEI reranking requests | `8` |