fix: replace additive combined scoring with multiplicative CE boosts (#494)
This commit is contained in:
Nicolò Boschi
GitHub
parent
ad2cf72aab
commit
aa8e5475c4
6 changed files with 230 additions and 369 deletions
|
|
@ -187,7 +187,7 @@ from .response_models import RecallResult as RecallResultModel
|
|||
from .retain import bank_utils, embedding_utils
|
||||
from .retain.types import RetainContentDict
|
||||
from .search import think_utils
|
||||
from .search.reranking import CrossEncoderReranker
|
||||
from .search.reranking import CrossEncoderReranker, apply_combined_scoring
|
||||
from .search.tags import TagsMatch, build_tags_where_clause
|
||||
from .task_backend import BrokerTaskBackend, SyncTaskBackend, TaskBackend
|
||||
|
||||
|
|
@ -2858,57 +2858,12 @@ class MemoryEngine(MemoryEngineInterface):
|
|||
rerank_span.set_attribute("hindsight.pre_filtered_count", pre_filtered_count)
|
||||
rerank_span.end()
|
||||
|
||||
# Step 4.5: Combine cross-encoder score with retrieval signals
|
||||
# This preserves retrieval work (RRF, temporal, recency) instead of pure cross-encoder ranking
|
||||
# Step 4.5: Combine cross-encoder score with retrieval signals via multiplicative boosts.
|
||||
# See apply_combined_scoring for the full rationale and formula.
|
||||
if scored_results:
|
||||
# Normalize RRF scores to [0, 1] range using min-max normalization
|
||||
rrf_scores = [sr.candidate.rrf_score for sr in scored_results]
|
||||
max_rrf = max(rrf_scores) if rrf_scores else 0.0
|
||||
min_rrf = min(rrf_scores) if rrf_scores else 0.0
|
||||
rrf_range = max_rrf - min_rrf # Don't force to 1.0, let fallback handle it
|
||||
|
||||
# Calculate recency based on occurred_start (more recent = higher score)
|
||||
now = utcnow()
|
||||
for sr in scored_results:
|
||||
# Normalize RRF score (0-1 range, 0.5 if all same)
|
||||
if rrf_range > 0:
|
||||
sr.rrf_normalized = (sr.candidate.rrf_score - min_rrf) / rrf_range
|
||||
else:
|
||||
# All RRF scores are the same, use neutral value
|
||||
sr.rrf_normalized = 0.5
|
||||
|
||||
# Calculate recency (decay over 365 days, minimum 0.1)
|
||||
sr.recency = 0.5 # default for missing dates
|
||||
if sr.retrieval.occurred_start:
|
||||
occurred = sr.retrieval.occurred_start
|
||||
if hasattr(occurred, "tzinfo") and occurred.tzinfo is None:
|
||||
occurred = occurred.replace(tzinfo=UTC)
|
||||
days_ago = (now - occurred).total_seconds() / 86400
|
||||
sr.recency = max(0.1, 1.0 - (days_ago / 365)) # Linear decay over 1 year
|
||||
|
||||
# Get temporal proximity if available (already 0-1)
|
||||
sr.temporal = (
|
||||
sr.retrieval.temporal_proximity if sr.retrieval.temporal_proximity is not None else 0.5
|
||||
)
|
||||
|
||||
# Weighted combination
|
||||
# Cross-encoder: 60% (semantic relevance)
|
||||
# RRF: 20% (retrieval consensus)
|
||||
# Temporal proximity: 10% (time relevance for temporal queries)
|
||||
# Recency: 10% (prefer recent facts)
|
||||
sr.combined_score = (
|
||||
0.6 * sr.cross_encoder_score_normalized
|
||||
+ 0.2 * sr.rrf_normalized
|
||||
+ 0.1 * sr.temporal
|
||||
+ 0.1 * sr.recency
|
||||
)
|
||||
sr.weight = sr.combined_score # Update weight for final ranking
|
||||
|
||||
# Re-sort by combined score
|
||||
apply_combined_scoring(scored_results, now=utcnow())
|
||||
scored_results.sort(key=lambda x: x.weight, reverse=True)
|
||||
log_buffer.append(
|
||||
" [4.6] Combined scoring: cross_encoder(0.6) + rrf(0.2) + temporal(0.1) + recency(0.1)"
|
||||
)
|
||||
log_buffer.append(" [4.6] Combined scoring: ce * recency_boost(0.2) * temporal_boost(0.2)")
|
||||
|
||||
# Add reranked results to tracer AFTER combined scoring (so normalized values are included)
|
||||
if tracer:
|
||||
|
|
|
|||
|
|
@ -2,8 +2,72 @@
|
|||
Cross-encoder neural reranking for search results.
|
||||
"""
|
||||
|
||||
from datetime import datetime, timezone
|
||||
|
||||
from .types import MergedCandidate, ScoredResult
|
||||
|
||||
UTC = timezone.utc
|
||||
|
||||
# Multiplicative boost alphas for recency and temporal proximity.
|
||||
# Each signal contributes at most ±(alpha/2) relative adjustment to the base CE score,
|
||||
# so the max combined boost is (1 + alpha/2)^2 ≈ +21% and min is (1 - alpha/2)^2 ≈ -19%.
|
||||
_RECENCY_ALPHA: float = 0.2
|
||||
_TEMPORAL_ALPHA: float = 0.2
|
||||
|
||||
|
||||
def apply_combined_scoring(
|
||||
scored_results: list[ScoredResult],
|
||||
now: datetime,
|
||||
recency_alpha: float = _RECENCY_ALPHA,
|
||||
temporal_alpha: float = _TEMPORAL_ALPHA,
|
||||
) -> None:
|
||||
"""Apply combined scoring to a list of ScoredResults in-place.
|
||||
|
||||
Uses the cross-encoder score as the primary relevance signal, with recency
|
||||
and temporal proximity applied as multiplicative boosts. This ensures the
|
||||
influence of these secondary signals is always proportional to the base
|
||||
relevance score, regardless of the cross-encoder model's score calibration.
|
||||
|
||||
Formula::
|
||||
|
||||
recency_boost = 1 + recency_alpha * (recency - 0.5) # in [1-α/2, 1+α/2]
|
||||
temporal_boost = 1 + temporal_alpha * (temporal - 0.5) # in [1-α/2, 1+α/2]
|
||||
combined_score = cross_encoder_score_normalized * recency_boost * temporal_boost
|
||||
|
||||
Temporal proximity is treated as neutral (0.5) when not set by temporal retrieval,
|
||||
so temporal_boost collapses to 1.0 for non-temporal queries.
|
||||
|
||||
Args:
|
||||
scored_results: Results from the cross-encoder reranker. Mutated in place.
|
||||
now: Current UTC datetime for recency calculation.
|
||||
recency_alpha: Max relative recency adjustment (default 0.2 → ±10%).
|
||||
temporal_alpha: Max relative temporal adjustment (default 0.2 → ±10%).
|
||||
"""
|
||||
if now.tzinfo is None:
|
||||
now = now.replace(tzinfo=UTC)
|
||||
|
||||
for sr in scored_results:
|
||||
# Recency: linear decay over 365 days → [0.1, 1.0]; neutral 0.5 if no date.
|
||||
sr.recency = 0.5
|
||||
if sr.retrieval.occurred_start:
|
||||
occurred = sr.retrieval.occurred_start
|
||||
if occurred.tzinfo is None:
|
||||
occurred = occurred.replace(tzinfo=UTC)
|
||||
days_ago = (now - occurred).total_seconds() / 86400
|
||||
sr.recency = max(0.1, min(1.0, 1.0 - (days_ago / 365)))
|
||||
|
||||
# Temporal proximity: meaningful only for temporal queries; neutral otherwise.
|
||||
sr.temporal = sr.retrieval.temporal_proximity if sr.retrieval.temporal_proximity is not None else 0.5
|
||||
|
||||
# RRF: kept at 0.0 for trace continuity but excluded from scoring.
|
||||
# RRF is batch-relative (min-max normalised) and redundant after reranking.
|
||||
sr.rrf_normalized = 0.0
|
||||
|
||||
recency_boost = 1.0 + recency_alpha * (sr.recency - 0.5)
|
||||
temporal_boost = 1.0 + temporal_alpha * (sr.temporal - 0.5)
|
||||
sr.combined_score = sr.cross_encoder_score_normalized * recency_boost * temporal_boost
|
||||
sr.weight = sr.combined_score
|
||||
|
||||
|
||||
class CrossEncoderReranker:
|
||||
"""
|
||||
|
|
|
|||
|
|
@ -43,8 +43,8 @@ dependencies = [
|
|||
"cohere>=5.0.0",
|
||||
"flashrank>=0.2.0",
|
||||
"litellm>=1.0.0",
|
||||
"markitdown[pdf,docx,pptx,xlsx,xls]>=0.1.4", # File to markdown conversion
|
||||
"obstore>=0.4.0", # S3/GCS/Azure object storage client (Rust-backed)
|
||||
"markitdown[pdf,docx,pptx,xlsx,xls]>=0.1.4", # File to markdown conversion
|
||||
"obstore>=0.4.0", # S3/GCS/Azure object storage client (Rust-backed)
|
||||
# Local ML models for embeddings/reranking - can be excluded in Docker with INCLUDE_LOCAL_MODELS=false
|
||||
"sentence-transformers>=3.3.0",
|
||||
"transformers>=4.53.0", # Security fixes for ReDoS vulnerabilities
|
||||
|
|
@ -62,6 +62,7 @@ dependencies = [
|
|||
"authlib>=1.6.6", # Account takeover vulnerability fix
|
||||
"aiohttp>=3.13.3", # Multiple DoS vulnerabilities
|
||||
"claude-agent-sdk>=0.1.27",
|
||||
"einops>=0.8.2",
|
||||
]
|
||||
|
||||
[project.optional-dependencies]
|
||||
|
|
|
|||
|
|
@ -1,334 +1,171 @@
|
|||
"""
|
||||
Tests for combined scoring functionality.
|
||||
Tests for combined scoring (apply_combined_scoring).
|
||||
|
||||
Verifies that:
|
||||
1. RRF scores are properly normalized to [0, 1] range
|
||||
2. Combined scoring formula is applied correctly
|
||||
3. Tracer captures normalized values (not raw values)
|
||||
The function applies multiplicative recency/temporal boosts to the cross-encoder
|
||||
score so that the relative influence of these signals is proportional to the base
|
||||
relevance score, independent of the cross-encoder model's score calibration.
|
||||
"""
|
||||
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import pytest
|
||||
from datetime import datetime, timezone
|
||||
from hindsight_api.engine.search.types import RetrievalResult, MergedCandidate, ScoredResult
|
||||
from hindsight_api.engine.memory_engine import Budget
|
||||
from hindsight_api import RequestContext
|
||||
|
||||
from hindsight_api.engine.search.reranking import apply_combined_scoring, _RECENCY_ALPHA, _TEMPORAL_ALPHA
|
||||
from hindsight_api.engine.search.types import MergedCandidate, RetrievalResult, ScoredResult
|
||||
|
||||
UTC = timezone.utc
|
||||
NOW = datetime(2024, 6, 1, tzinfo=UTC)
|
||||
|
||||
|
||||
class TestRRFNormalization:
|
||||
"""Test that RRF scores are properly normalized."""
|
||||
def _make_result(
|
||||
ce_norm: float,
|
||||
occurred_start: datetime | None = None,
|
||||
temporal_proximity: float | None = None,
|
||||
) -> ScoredResult:
|
||||
retrieval = MagicMock(spec=RetrievalResult)
|
||||
retrieval.occurred_start = occurred_start
|
||||
retrieval.temporal_proximity = temporal_proximity
|
||||
|
||||
def test_rrf_normalized_range(self):
|
||||
"""RRF normalized values should be in [0, 1] range, not raw [0.04, 0.06]."""
|
||||
# Simulate RRF scores like what we get from actual retrieval
|
||||
raw_rrf_scores = [0.0607, 0.0550, 0.0480, 0.0390]
|
||||
candidate = MagicMock(spec=MergedCandidate)
|
||||
candidate.retrieval = retrieval
|
||||
candidate.rrf_score = 0.05
|
||||
|
||||
max_rrf = max(raw_rrf_scores)
|
||||
min_rrf = min(raw_rrf_scores)
|
||||
rrf_range = max_rrf - min_rrf
|
||||
|
||||
normalized = []
|
||||
for score in raw_rrf_scores:
|
||||
if rrf_range > 0:
|
||||
norm = (score - min_rrf) / rrf_range
|
||||
else:
|
||||
norm = 0.5
|
||||
normalized.append(norm)
|
||||
|
||||
# Verify normalized values are in [0, 1]
|
||||
for i, norm in enumerate(normalized):
|
||||
assert 0.0 <= norm <= 1.0, f"Normalized RRF {norm} not in [0, 1] for raw {raw_rrf_scores[i]}"
|
||||
|
||||
# Highest raw should be 1.0
|
||||
assert normalized[0] == 1.0, f"Highest RRF should normalize to 1.0, got {normalized[0]}"
|
||||
|
||||
# Lowest raw should be 0.0
|
||||
assert normalized[-1] == 0.0, f"Lowest RRF should normalize to 0.0, got {normalized[-1]}"
|
||||
|
||||
def test_rrf_all_same_scores(self):
|
||||
"""When all RRF scores are the same, normalized should be 0.5 (neutral)."""
|
||||
raw_rrf_scores = [0.0500, 0.0500, 0.0500]
|
||||
|
||||
max_rrf = max(raw_rrf_scores)
|
||||
min_rrf = min(raw_rrf_scores)
|
||||
rrf_range = max_rrf - min_rrf
|
||||
|
||||
normalized = []
|
||||
for score in raw_rrf_scores:
|
||||
if rrf_range > 0:
|
||||
norm = (score - min_rrf) / rrf_range
|
||||
else:
|
||||
norm = 0.5 # Neutral value when all same
|
||||
normalized.append(norm)
|
||||
|
||||
# All should be 0.5 when scores are identical
|
||||
for norm in normalized:
|
||||
assert norm == 0.5, f"Expected 0.5 for identical scores, got {norm}"
|
||||
return ScoredResult(
|
||||
candidate=candidate,
|
||||
cross_encoder_score=1.0,
|
||||
cross_encoder_score_normalized=ce_norm,
|
||||
weight=ce_norm,
|
||||
)
|
||||
|
||||
|
||||
class TestCombinedScoringFormula:
|
||||
"""Test that the combined scoring formula is applied correctly."""
|
||||
class TestBoostFormula:
|
||||
def test_neutral_signals_leave_score_unchanged(self):
|
||||
"""recency=0.5 and temporal=0.5 both produce boost=1.0, so weight == ce."""
|
||||
sr = _make_result(ce_norm=0.6)
|
||||
apply_combined_scoring([sr], now=NOW)
|
||||
assert abs(sr.weight - 0.6) < 1e-9
|
||||
|
||||
def test_combined_score_calculation(self):
|
||||
"""Verify the weighted combination: 0.6*CE + 0.2*RRF + 0.1*temporal + 0.1*recency."""
|
||||
# Test case 1: All components at 1.0
|
||||
ce_norm = 1.0
|
||||
rrf_norm = 1.0
|
||||
temporal = 1.0
|
||||
recency = 1.0
|
||||
def test_max_recency_boost(self):
|
||||
"""A memory from today (recency≈1.0) should boost by (1 + alpha*0.5)."""
|
||||
sr = _make_result(ce_norm=0.5, occurred_start=NOW)
|
||||
apply_combined_scoring([sr], now=NOW)
|
||||
expected = 0.5 * (1.0 + _RECENCY_ALPHA * 0.5) * 1.0 # temporal neutral
|
||||
assert abs(sr.weight - expected) < 1e-6
|
||||
|
||||
expected = 0.6 * ce_norm + 0.2 * rrf_norm + 0.1 * temporal + 0.1 * recency
|
||||
assert expected == 1.0, f"All 1.0 should give 1.0, got {expected}"
|
||||
def test_min_recency_penalty(self):
|
||||
"""A memory from >365 days ago (recency=0.1) should penalise score."""
|
||||
old = NOW - timedelta(days=400)
|
||||
sr = _make_result(ce_norm=0.5, occurred_start=old)
|
||||
apply_combined_scoring([sr], now=NOW)
|
||||
expected = 0.5 * (1.0 + _RECENCY_ALPHA * (0.1 - 0.5)) * 1.0
|
||||
assert abs(sr.weight - expected) < 1e-6
|
||||
|
||||
# Test case 2: All components at 0.0
|
||||
ce_norm = 0.0
|
||||
rrf_norm = 0.0
|
||||
temporal = 0.0
|
||||
recency = 0.0
|
||||
def test_max_temporal_boost(self):
|
||||
"""temporal_proximity=1.0 should boost by (1 + alpha*0.5)."""
|
||||
sr = _make_result(ce_norm=0.5, temporal_proximity=1.0)
|
||||
apply_combined_scoring([sr], now=NOW)
|
||||
expected = 0.5 * 1.0 * (1.0 + _TEMPORAL_ALPHA * 0.5) # recency neutral
|
||||
assert abs(sr.weight - expected) < 1e-6
|
||||
|
||||
expected = 0.6 * ce_norm + 0.2 * rrf_norm + 0.1 * temporal + 0.1 * recency
|
||||
assert expected == 0.0, f"All 0.0 should give 0.0, got {expected}"
|
||||
def test_temporal_none_is_neutral(self):
|
||||
"""temporal_proximity=None must be treated as 0.5 (no boost/penalty)."""
|
||||
sr_none = _make_result(ce_norm=0.5, temporal_proximity=None)
|
||||
sr_half = _make_result(ce_norm=0.5, temporal_proximity=0.5)
|
||||
apply_combined_scoring([sr_none], now=NOW)
|
||||
apply_combined_scoring([sr_half], now=NOW)
|
||||
assert abs(sr_none.weight - sr_half.weight) < 1e-9
|
||||
|
||||
# Test case 3: High CE, low RRF (cross-encoder finds something retrieval missed)
|
||||
ce_norm = 0.999
|
||||
rrf_norm = 0.0 # Lowest in set
|
||||
temporal = 0.5
|
||||
recency = 0.5
|
||||
def test_both_signals_combined(self):
|
||||
"""Both boosts are applied multiplicatively."""
|
||||
sr = _make_result(ce_norm=0.5, occurred_start=NOW, temporal_proximity=1.0)
|
||||
apply_combined_scoring([sr], now=NOW)
|
||||
recency_boost = 1.0 + _RECENCY_ALPHA * (1.0 - 0.5)
|
||||
temporal_boost = 1.0 + _TEMPORAL_ALPHA * (1.0 - 0.5)
|
||||
expected = 0.5 * recency_boost * temporal_boost
|
||||
assert abs(sr.weight - expected) < 1e-6
|
||||
|
||||
expected = 0.6 * ce_norm + 0.2 * rrf_norm + 0.1 * temporal + 0.1 * recency
|
||||
# 0.5994 + 0.0 + 0.05 + 0.05 = 0.6994
|
||||
assert abs(expected - 0.6994) < 0.001, f"Expected ~0.6994, got {expected}"
|
||||
def test_boost_is_proportional_to_ce(self):
|
||||
"""The absolute boost from recency scales with the CE score."""
|
||||
sr_high = _make_result(ce_norm=0.9, occurred_start=NOW)
|
||||
sr_low = _make_result(ce_norm=0.3, occurred_start=NOW)
|
||||
apply_combined_scoring([sr_high, sr_low], now=NOW)
|
||||
|
||||
# Test case 4: Medium CE, high RRF (retrieval consensus)
|
||||
ce_norm = 0.8
|
||||
rrf_norm = 1.0 # Highest in set
|
||||
temporal = 0.5
|
||||
recency = 0.5
|
||||
# Both get the same recency boost factor — absolute gain is proportional to CE
|
||||
boost_factor = 1.0 + _RECENCY_ALPHA * 0.5
|
||||
assert abs(sr_high.weight - 0.9 * boost_factor) < 1e-6
|
||||
assert abs(sr_low.weight - 0.3 * boost_factor) < 1e-6
|
||||
|
||||
expected = 0.6 * ce_norm + 0.2 * rrf_norm + 0.1 * temporal + 0.1 * recency
|
||||
# 0.48 + 0.2 + 0.05 + 0.05 = 0.78
|
||||
assert abs(expected - 0.78) < 0.001, f"Expected ~0.78, got {expected}"
|
||||
def test_boost_capped(self):
|
||||
"""Max boost: recency=1.0 + temporal=1.0 gives ≤21% uplift on CE."""
|
||||
sr = _make_result(ce_norm=1.0, occurred_start=NOW, temporal_proximity=1.0)
|
||||
apply_combined_scoring([sr], now=NOW)
|
||||
assert sr.weight <= 1.0 * (1 + _RECENCY_ALPHA / 2) * (1 + _TEMPORAL_ALPHA / 2) + 1e-9
|
||||
|
||||
def test_rrf_contribution_is_significant(self):
|
||||
"""Verify RRF actually contributes to the final score (not negligible)."""
|
||||
# Same CE, different RRF
|
||||
ce_norm = 0.8
|
||||
temporal = 0.5
|
||||
recency = 0.5
|
||||
def test_rrf_normalized_always_zero(self):
|
||||
"""RRF is excluded from scoring; rrf_normalized is set to 0.0 for trace clarity."""
|
||||
sr = _make_result(ce_norm=0.5)
|
||||
apply_combined_scoring([sr], now=NOW)
|
||||
assert sr.rrf_normalized == 0.0
|
||||
|
||||
# Low RRF
|
||||
score_low_rrf = 0.6 * ce_norm + 0.2 * 0.0 + 0.1 * temporal + 0.1 * recency
|
||||
def test_combined_score_equals_weight(self):
|
||||
"""combined_score and weight must stay in sync."""
|
||||
sr = _make_result(ce_norm=0.7, occurred_start=NOW, temporal_proximity=0.8)
|
||||
apply_combined_scoring([sr], now=NOW)
|
||||
assert sr.combined_score == sr.weight
|
||||
|
||||
# High RRF
|
||||
score_high_rrf = 0.6 * ce_norm + 0.2 * 1.0 + 0.1 * temporal + 0.1 * recency
|
||||
def test_model_calibration_independence(self):
|
||||
"""
|
||||
A low-calibration model (low CE scores) and a high-calibration model
|
||||
(high CE scores) should produce the same ranking for identical content.
|
||||
|
||||
# Difference should be 0.2 (20% contribution)
|
||||
diff = score_high_rrf - score_low_rrf
|
||||
assert abs(diff - 0.2) < 0.001, f"RRF should contribute 0.2 difference, got {diff}"
|
||||
With additive scoring the recency term would dominate for low-CE models;
|
||||
with multiplicative boosting the relative ranking is stable.
|
||||
"""
|
||||
recent = NOW - timedelta(days=10)
|
||||
old = NOW - timedelta(days=300)
|
||||
|
||||
# High-calibration model: clear winner is #1 (more relevant, slightly older)
|
||||
h_relevant = _make_result(ce_norm=0.85, occurred_start=old)
|
||||
h_recent = _make_result(ce_norm=0.60, occurred_start=recent)
|
||||
apply_combined_scoring([h_relevant, h_recent], now=NOW)
|
||||
assert h_relevant.weight > h_recent.weight, "High-CE model: relevance should win"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_trace_has_normalized_rrf(memory, request_context):
|
||||
"""Integration test: verify trace contains normalized RRF values, not raw."""
|
||||
bank_id = f"test_scoring_{datetime.now(timezone.utc).timestamp()}"
|
||||
# Low-calibration model: same relative difference, just compressed scores
|
||||
l_relevant = _make_result(ce_norm=0.34, occurred_start=old)
|
||||
l_recent = _make_result(ce_norm=0.24, occurred_start=recent)
|
||||
apply_combined_scoring([l_relevant, l_recent], now=NOW)
|
||||
assert l_relevant.weight > l_recent.weight, "Low-CE model: relevance should still win"
|
||||
|
||||
try:
|
||||
# Store multiple memories to ensure different RRF scores
|
||||
await memory.retain_async(
|
||||
bank_id=bank_id,
|
||||
content="Python is a programming language created by Guido van Rossum",
|
||||
context="tech facts",
|
||||
request_context=request_context,
|
||||
)
|
||||
await memory.retain_async(
|
||||
bank_id=bank_id,
|
||||
content="JavaScript was created by Brendan Eich at Netscape",
|
||||
context="tech facts",
|
||||
request_context=request_context,
|
||||
)
|
||||
await memory.retain_async(
|
||||
bank_id=bank_id,
|
||||
content="The Eiffel Tower is located in Paris, France",
|
||||
context="geography facts",
|
||||
request_context=request_context,
|
||||
)
|
||||
await memory.retain_async(
|
||||
bank_id=bank_id,
|
||||
content="Mount Everest is the tallest mountain on Earth",
|
||||
context="geography facts",
|
||||
request_context=request_context,
|
||||
)
|
||||
def test_no_occurred_start_defaults_recency_neutral(self):
|
||||
"""Missing occurred_start → recency=0.5 → no boost/penalty."""
|
||||
sr = _make_result(ce_norm=0.5, occurred_start=None)
|
||||
apply_combined_scoring([sr], now=NOW)
|
||||
assert sr.recency == 0.5
|
||||
assert abs(sr.weight - 0.5) < 1e-9
|
||||
|
||||
# Search with tracing
|
||||
result = await memory.recall_async(
|
||||
bank_id=bank_id,
|
||||
query="programming languages",
|
||||
fact_type=["world"],
|
||||
budget=Budget.LOW,
|
||||
max_tokens=1024,
|
||||
enable_trace=True,
|
||||
request_context=request_context,
|
||||
)
|
||||
def test_timezone_naive_occurred_start_handled(self):
|
||||
"""Naive datetimes in occurred_start should not raise."""
|
||||
naive_date = datetime(2024, 1, 1) # no tzinfo
|
||||
sr = _make_result(ce_norm=0.5, occurred_start=naive_date)
|
||||
apply_combined_scoring([sr], now=NOW) # must not raise
|
||||
assert 0.0 < sr.weight < 1.0
|
||||
|
||||
assert result.trace is not None, "Trace should be present"
|
||||
trace = result.trace
|
||||
def test_custom_alpha_values(self):
|
||||
"""Custom alpha parameters are respected."""
|
||||
sr = _make_result(ce_norm=0.5, occurred_start=NOW)
|
||||
apply_combined_scoring([sr], now=NOW, recency_alpha=0.4, temporal_alpha=0.0)
|
||||
expected = 0.5 * (1.0 + 0.4 * 0.5) * 1.0
|
||||
assert abs(sr.weight - expected) < 1e-6
|
||||
|
||||
# Check reranked results have proper score_components
|
||||
assert "reranked" in trace, "Trace should have reranked results"
|
||||
assert len(trace["reranked"]) > 0, "Should have reranked results"
|
||||
def test_future_event_recency_capped_at_one(self):
|
||||
"""Events in the future must not produce recency > 1.0, keeping boost within bounds."""
|
||||
future = NOW + timedelta(days=180)
|
||||
sr = _make_result(ce_norm=0.5, occurred_start=future)
|
||||
apply_combined_scoring([sr], now=NOW)
|
||||
assert sr.recency == 1.0
|
||||
expected_max_boost = 1.0 + _RECENCY_ALPHA * 0.5
|
||||
assert sr.weight <= 0.5 * expected_max_boost + 1e-9
|
||||
|
||||
has_valid_rrf = False
|
||||
has_valid_temporal = False
|
||||
has_valid_recency = False
|
||||
|
||||
for r in trace["reranked"]:
|
||||
sc = r.get("score_components", {})
|
||||
|
||||
# Check RRF normalized is present and in valid range
|
||||
if "rrf_normalized" in sc:
|
||||
rrf_norm = sc["rrf_normalized"]
|
||||
assert 0.0 <= rrf_norm <= 1.0, f"rrf_normalized {rrf_norm} should be in [0, 1]"
|
||||
# Should NOT be raw RRF score (which would be ~0.04-0.06)
|
||||
# A normalized value of exactly 0.0 or 1.0 is valid (min/max of set)
|
||||
# But raw scores like 0.0607 should never appear as normalized
|
||||
if rrf_norm > 0.1: # Any value > 0.1 is likely properly normalized
|
||||
has_valid_rrf = True
|
||||
|
||||
# Check temporal is present and in valid range
|
||||
if "temporal" in sc:
|
||||
temporal = sc["temporal"]
|
||||
assert 0.0 <= temporal <= 1.0, f"temporal {temporal} should be in [0, 1]"
|
||||
has_valid_temporal = True
|
||||
|
||||
# Check recency is present and in valid range
|
||||
if "recency" in sc:
|
||||
recency = sc["recency"]
|
||||
assert 0.0 <= recency <= 1.0, f"recency {recency} should be in [0, 1]"
|
||||
has_valid_recency = True
|
||||
|
||||
# At least some results should have these components
|
||||
# (might not have rrf > 0.1 if all scores are same, which is fine)
|
||||
assert has_valid_temporal, "Should have temporal scores in trace"
|
||||
assert has_valid_recency, "Should have recency scores in trace"
|
||||
|
||||
print("\n✓ Combined scoring trace test passed!")
|
||||
print(f" - Reranked results: {len(trace['reranked'])}")
|
||||
if trace["reranked"]:
|
||||
sc = trace["reranked"][0].get("score_components", {})
|
||||
print(f" - First result score components: {sc}")
|
||||
|
||||
finally:
|
||||
await memory.delete_bank(bank_id, request_context=request_context)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_rrf_normalized_not_raw_in_trace(memory, request_context):
|
||||
"""Verify that raw RRF scores (0.04-0.06 range) don't appear as normalized values."""
|
||||
bank_id = f"test_rrf_raw_{datetime.now(timezone.utc).timestamp()}"
|
||||
|
||||
try:
|
||||
# Store enough memories to get varied RRF scores
|
||||
for i in range(5):
|
||||
await memory.retain_async(
|
||||
bank_id=bank_id,
|
||||
content=f"Test fact number {i} about various topics",
|
||||
context="test context",
|
||||
request_context=request_context,
|
||||
)
|
||||
|
||||
result = await memory.recall_async(
|
||||
bank_id=bank_id,
|
||||
query="test fact",
|
||||
fact_type=["world"],
|
||||
budget=Budget.LOW,
|
||||
max_tokens=512,
|
||||
enable_trace=True,
|
||||
request_context=request_context,
|
||||
)
|
||||
|
||||
trace = result.trace
|
||||
assert trace is not None
|
||||
|
||||
# Check that rrf_normalized values are NOT in the raw range
|
||||
raw_rrf_range = (0.01, 0.08) # Raw RRF scores are typically in this range
|
||||
|
||||
for r in trace.get("reranked", []):
|
||||
sc = r.get("score_components", {})
|
||||
|
||||
if "rrf_normalized" in sc and "rrf_score" in sc:
|
||||
rrf_norm = sc["rrf_normalized"]
|
||||
rrf_raw = sc["rrf_score"]
|
||||
|
||||
# Raw should be in the typical range
|
||||
assert raw_rrf_range[0] <= rrf_raw <= raw_rrf_range[1], \
|
||||
f"Raw RRF {rrf_raw} should be in typical range {raw_rrf_range}"
|
||||
|
||||
# Normalized should either be:
|
||||
# - 0.0 (min in set)
|
||||
# - 1.0 (max in set)
|
||||
# - 0.5 (all same)
|
||||
# - Something in between (0.0 to 1.0)
|
||||
# But NOT the same as raw (which would indicate no normalization)
|
||||
if len(trace["reranked"]) > 1:
|
||||
# If we have multiple results, normalized should differ from raw
|
||||
# (unless by coincidence, which is very unlikely)
|
||||
assert rrf_norm != rrf_raw, \
|
||||
f"Normalized RRF ({rrf_norm}) should differ from raw ({rrf_raw})"
|
||||
|
||||
print("\n✓ RRF raw vs normalized test passed!")
|
||||
|
||||
finally:
|
||||
await memory.delete_bank(bank_id, request_context=request_context)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_combined_score_matches_components(memory, request_context):
|
||||
"""Verify the final score actually equals the weighted sum of components."""
|
||||
bank_id = f"test_combined_{datetime.now(timezone.utc).timestamp()}"
|
||||
|
||||
try:
|
||||
await memory.retain_async(
|
||||
bank_id=bank_id,
|
||||
content="The quick brown fox jumps over the lazy dog",
|
||||
context="test",
|
||||
request_context=request_context,
|
||||
)
|
||||
await memory.retain_async(
|
||||
bank_id=bank_id,
|
||||
content="A quick test of the emergency broadcast system",
|
||||
context="test",
|
||||
request_context=request_context,
|
||||
)
|
||||
|
||||
result = await memory.recall_async(
|
||||
bank_id=bank_id,
|
||||
query="quick test",
|
||||
fact_type=["world"],
|
||||
budget=Budget.LOW,
|
||||
max_tokens=512,
|
||||
enable_trace=True,
|
||||
request_context=request_context,
|
||||
)
|
||||
|
||||
trace = result.trace
|
||||
assert trace is not None
|
||||
|
||||
for r in trace.get("reranked", []):
|
||||
sc = r.get("score_components", {})
|
||||
final_score = r.get("rerank_score", 0)
|
||||
|
||||
# Get components (use defaults if missing)
|
||||
ce = sc.get("cross_encoder_score_normalized", 0)
|
||||
rrf = sc.get("rrf_normalized", 0.5)
|
||||
tmp = sc.get("temporal", 0.5)
|
||||
rec = sc.get("recency", 0.5)
|
||||
|
||||
# Calculate expected score
|
||||
expected = 0.6 * ce + 0.2 * rrf + 0.1 * tmp + 0.1 * rec
|
||||
|
||||
# Allow small floating point difference
|
||||
assert abs(final_score - expected) < 0.01, \
|
||||
f"Final score {final_score} doesn't match expected {expected} from components"
|
||||
|
||||
print("\n✓ Combined score verification test passed!")
|
||||
|
||||
finally:
|
||||
await memory.delete_bank(bank_id, request_context=request_context)
|
||||
def test_empty_list_is_noop(self):
|
||||
apply_combined_scoring([], now=NOW) # must not raise
|
||||
|
|
|
|||
|
|
@ -871,7 +871,7 @@ export function SearchDebugView() {
|
|||
</div>
|
||||
<div className="text-sm text-muted-foreground mt-0.5">
|
||||
<span className="font-mono text-xs">
|
||||
0.6×cross_encoder + 0.2×rrf + 0.1×temporal + 0.1×recency
|
||||
ce × recency_boost(±10%) × temporal_boost(±10%)
|
||||
</span>
|
||||
</div>
|
||||
</div>
|
||||
|
|
@ -924,24 +924,17 @@ export function SearchDebugView() {
|
|||
= {(r.rerank_score || r.score || 0).toFixed(4)}
|
||||
</span>
|
||||
{sc.cross_encoder_score_normalized !== undefined && (
|
||||
<span title="Cross-encoder (60%)">
|
||||
<span title="Cross-encoder score (primary relevance signal)">
|
||||
CE: {sc.cross_encoder_score_normalized.toFixed(3)}
|
||||
</span>
|
||||
)}
|
||||
{sc.rrf_normalized !== undefined && (
|
||||
<span
|
||||
title={`RRF normalized (20%) - raw: ${sc.rrf_score?.toFixed(4) || "N/A"}`}
|
||||
>
|
||||
RRF: {sc.rrf_normalized.toFixed(3)}
|
||||
</span>
|
||||
)}
|
||||
{sc.temporal !== undefined && (
|
||||
<span title="Temporal proximity (10%)">
|
||||
{sc.temporal !== undefined && sc.temporal !== 0.5 && (
|
||||
<span title="Temporal proximity boost (±10% — only active for temporal queries)">
|
||||
Tmp: {sc.temporal.toFixed(3)}
|
||||
</span>
|
||||
)}
|
||||
{sc.recency !== undefined && (
|
||||
<span title="Recency (10%)">
|
||||
<span title="Recency boost (±10% — based on memory age)">
|
||||
Rec: {sc.recency.toFixed(3)}
|
||||
</span>
|
||||
)}
|
||||
|
|
|
|||
11
uv.lock
11
uv.lock
|
|
@ -839,6 +839,15 @@ wheels = [
|
|||
{ url = "https://files.pythonhosted.org/packages/11/a8/c6a4b901d17399c77cd81fb001ce8961e9f5e04d3daf27e8925cb012e163/docutils-0.22.3-py3-none-any.whl", hash = "sha256:bd772e4aca73aff037958d44f2be5229ded4c09927fcf8690c577b66234d6ceb", size = 633032 },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "einops"
|
||||
version = "0.8.2"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/2c/77/850bef8d72ffb9219f0b1aac23fbc1bf7d038ee6ea666f331fa273031aa2/einops-0.8.2.tar.gz", hash = "sha256:609da665570e5e265e27283aab09e7f279ade90c4f01bcfca111f3d3e13f2827", size = 56261 }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/2a/09/f8d8f8f31e4483c10a906437b4ce31bdf3d6d417b73fe33f1a8b59e34228/einops-0.8.2-py3-none-any.whl", hash = "sha256:54058201ac7087911181bfec4af6091bb59380360f069276601256a76af08193", size = 65638 },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "email-validator"
|
||||
version = "2.3.0"
|
||||
|
|
@ -1457,6 +1466,7 @@ dependencies = [
|
|||
{ name = "cohere" },
|
||||
{ name = "cryptography" },
|
||||
{ name = "dateparser" },
|
||||
{ name = "einops" },
|
||||
{ name = "fastapi", extra = ["standard"] },
|
||||
{ name = "fastmcp" },
|
||||
{ name = "filelock" },
|
||||
|
|
@ -1535,6 +1545,7 @@ requires-dist = [
|
|||
{ name = "cohere", specifier = ">=5.0.0" },
|
||||
{ name = "cryptography", specifier = ">=46.0.5" },
|
||||
{ name = "dateparser", specifier = ">=1.2.2" },
|
||||
{ name = "einops", specifier = ">=0.8.2" },
|
||||
{ name = "fastapi", extras = ["standard"], specifier = ">=0.120.3" },
|
||||
{ name = "fastmcp", specifier = ">=2.14.0" },
|
||||
{ name = "filelock", specifier = ">=3.20.1" },
|
||||
|
|
|
|||
Loading…
Reference in a new issue