From 5fddd9a79c8ed4eb0b411e7333c93404c9c9b669 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=B2=20Boschi?= Date: Tue, 24 Feb 2026 09:48:23 +0100 Subject: [PATCH] feat: add reflect mode to LoComo benchmark and improve reflect agent (#428) * feat: add reflect mode to LoComo benchmark and improve reflect agent - Replace think mode with reflect mode in LoComo benchmark using reflect_async with Budget.HIGH - Add --question-index CLI flag to run a single question by its index - Track and display original question index in logs and visualizer - Update visualizer to show reflect mode results Reflect agent improvements: - tool_recall: always fetch chunks (max_chunk_tokens=1000 min, non-optional) - tool_search_observations: use include_source_facts=True instead of separate DB query - Use model_dump() throughout to avoid manual error-prone dict conversion - Enforce minimum 1000 tokens for max_tokens and max_chunk_tokens in _execute_tool - Fix NoneType error when LLM passes null for mental_model_ids/observation_ids arrays - Add non-conversational constraint to system prompt to prevent follow-up questions - Fix recall_fn Callable type hint to include max_chunk_tokens parameter - Fix main.py missing reranker_zeroentropy fields in HindsightConfig constructor * fix: update tests for reflect tool API changes - source_memory_ids -> source_fact_ids in test_search_observations (MemoryFact.model_dump() field name) - Remove proof_count check (not in MemoryFact, was ObservationResult-specific) - Remove max_results param from tool_recall call (no longer supported) - Fix recall_result["count"] -> len(recall_result["memories"]) --- .../hindsight_api/engine/memory_engine.py | 28 ++-- .../hindsight_api/engine/reflect/agent.py | 19 ++- .../hindsight_api/engine/reflect/prompts.py | 5 +- .../hindsight_api/engine/reflect/tools.py | 95 +++-------- .../engine/reflect/tools_schema.py | 4 + hindsight-api/tests/test_consolidation.py | 17 +- .../benchmarks/common/benchmark_runner.py | 21 ++- .../benchmarks/locomo/locomo_benchmark.py | 155 +++++++++++------- hindsight-dev/benchmarks/visualizer/main.py | 14 +- 9 files changed, 176 insertions(+), 182 deletions(-) diff --git a/hindsight-api/hindsight_api/engine/memory_engine.py b/hindsight-api/hindsight_api/engine/memory_engine.py index 82a892a6..3fbb2a1a 100644 --- a/hindsight-api/hindsight_api/engine/memory_engine.py +++ b/hindsight-api/hindsight_api/engine/memory_engine.py @@ -4332,9 +4332,16 @@ class MemoryEngine(MemoryEngineInterface): pending_consolidation=pending_consolidation, ) - async def recall_fn(q: str, max_tokens: int = 4096) -> dict[str, Any]: + async def recall_fn(q: str, max_tokens: int = 4096, max_chunk_tokens: int = 1000) -> dict[str, Any]: return await tool_recall( - self, bank_id, q, request_context, max_tokens=max_tokens, tags=tags, tags_match=tags_match + self, + bank_id, + q, + request_context, + max_tokens=max_tokens, + tags=tags, + tags_match=tags_match, + max_chunk_tokens=max_chunk_tokens, ) async def expand_fn(memory_ids: list[str], depth: str) -> dict[str, Any]: @@ -4443,16 +4450,16 @@ class MemoryEngine(MemoryEngineInterface): if used_memory_ids_set and memory_id not in used_memory_ids_set: continue # Skip memories not actually used by the agent seen_memory_ids.add(memory_id) - fact_type = memory_data.get("type", "world") + fact_type = memory_data.get("fact_type", "world") if fact_type in based_on: based_on[fact_type].append( MemoryFact( id=memory_id, text=memory_data.get("text", ""), fact_type=fact_type, - context=None, - occurred_start=memory_data.get("occurred"), - occurred_end=memory_data.get("occurred"), + context=memory_data.get("context"), + occurred_start=memory_data.get("occurred_start"), + occurred_end=memory_data.get("occurred_end"), ) ) elif tc.tool == "search_observations" and "observations" in tc.output: @@ -4462,14 +4469,7 @@ class MemoryEngine(MemoryEngineInterface): if used_observation_ids_set and obs_id not in used_observation_ids_set: continue # Skip observations not actually used by the agent seen_memory_ids.add(obs_id) - based_on["observation"].append( - MemoryFact( - id=obs_id, - text=obs_data.get("text", ""), - fact_type="observation", - context=None, - ) - ) + based_on["observation"].append(MemoryFact(**obs_data)) # Extract mental models from tool outputs - only include models the agent actually used # agent_result.used_mental_model_ids contains validated IDs from the done action diff --git a/hindsight-api/hindsight_api/engine/reflect/agent.py b/hindsight-api/hindsight_api/engine/reflect/agent.py index 435c584c..fd3353a6 100644 --- a/hindsight-api/hindsight_api/engine/reflect/agent.py +++ b/hindsight-api/hindsight_api/engine/reflect/agent.py @@ -266,7 +266,7 @@ async def run_reflect_agent( bank_profile: dict[str, Any], search_mental_models_fn: Callable[[str, int], Awaitable[dict[str, Any]]], search_observations_fn: Callable[[str, int], Awaitable[dict[str, Any]]], - recall_fn: Callable[[str, int], Awaitable[dict[str, Any]]], + recall_fn: Callable[[str, int, int], Awaitable[dict[str, Any]]], expand_fn: Callable[[list[str], str], Awaitable[dict[str, Any]]], context: str | None = None, max_iterations: int = DEFAULT_MAX_ITERATIONS, @@ -819,9 +819,9 @@ async def _process_done_tool( answer = "No answer provided." # Validate IDs (only include IDs that were actually retrieved) - used_memory_ids = [mid for mid in args.get("memory_ids", []) if mid in available_memory_ids] - used_mental_model_ids = [mid for mid in args.get("mental_model_ids", []) if mid in available_mental_model_ids] - used_observation_ids = [oid for oid in args.get("observation_ids", []) if oid in available_observation_ids] + used_memory_ids = [mid for mid in (args.get("memory_ids") or []) if mid in available_memory_ids] + used_mental_model_ids = [mid for mid in (args.get("mental_model_ids") or []) if mid in available_mental_model_ids] + used_observation_ids = [oid for oid in (args.get("observation_ids") or []) if oid in available_observation_ids] # Generate structured output if schema provided structured_output = None @@ -857,7 +857,7 @@ async def _execute_tool_with_timing( tc: "LLMToolCall", search_mental_models_fn: Callable[[str, int], Awaitable[dict[str, Any]]], search_observations_fn: Callable[[str, int], Awaitable[dict[str, Any]]], - recall_fn: Callable[[str, int], Awaitable[dict[str, Any]]], + recall_fn: Callable[[str, int, int], Awaitable[dict[str, Any]]], expand_fn: Callable[[list[str], str], Awaitable[dict[str, Any]]], ) -> tuple[dict[str, Any], int]: """Execute a tool call and return result with timing.""" @@ -929,7 +929,7 @@ async def _execute_tool( args: dict[str, Any], search_mental_models_fn: Callable[[str, int], Awaitable[dict[str, Any]]], search_observations_fn: Callable[[str, int], Awaitable[dict[str, Any]]], - recall_fn: Callable[[str, int], Awaitable[dict[str, Any]]], + recall_fn: Callable[[str, int, int], Awaitable[dict[str, Any]]], expand_fn: Callable[[list[str], str], Awaitable[dict[str, Any]]], ) -> dict[str, Any]: """Execute a single tool by name.""" @@ -955,7 +955,8 @@ async def _execute_tool( if not query: return {"error": "recall requires a query parameter"} max_tokens = max(int(args.get("max_tokens") or 2048), 1000) # Default 2048, min 1000 - return await recall_fn(query, max_tokens) + max_chunk_tokens = max(int(args.get("max_chunk_tokens") or 1000), 1000) # Always enabled, min 1000 + return await recall_fn(query, max_tokens, max_chunk_tokens) elif tool_name == "expand": memory_ids = args.get("memory_ids", []) @@ -983,9 +984,9 @@ def _summarize_input(tool_name: str, args: dict[str, Any]) -> str: elif tool_name == "recall": query = args.get("query", "") query_preview = f"'{query[:30]}...'" if len(query) > 30 else f"'{query}'" - # Show actual value used (default 2048, min 1000) max_tokens = max(int(args.get("max_tokens") or 2048), 1000) - return f"(query={query_preview}, max_tokens={max_tokens})" + max_chunk_tokens = max(int(args.get("max_chunk_tokens") or 1000), 1000) + return f"(query={query_preview}, max_tokens={max_tokens}, max_chunk_tokens={max_chunk_tokens})" elif tool_name == "expand": memory_ids = args.get("memory_ids", []) depth = args.get("depth", "chunk") diff --git a/hindsight-api/hindsight_api/engine/reflect/prompts.py b/hindsight-api/hindsight_api/engine/reflect/prompts.py index 2c93088a..08aebee8 100644 --- a/hindsight-api/hindsight_api/engine/reflect/prompts.py +++ b/hindsight-api/hindsight_api/engine/reflect/prompts.py @@ -286,6 +286,7 @@ def build_system_prompt_for_tools( "- Format for clarity and readability with proper spacing and hierarchy", "- NEVER include memory IDs, UUIDs, or 'Memory references' in the answer text", "- Put IDs ONLY in the memory_ids/mental_model_ids/observation_ids arrays, not in the answer", + "- CRITICAL: This is a NON-CONVERSATIONAL system. NEVER ask follow-up questions, offer further assistance, or suggest next steps. Your answer must be complete and self-contained. The user cannot reply.", ] ) @@ -481,4 +482,6 @@ CRITICAL: Output ONLY the final synthesized answer. Do NOT include: - Meta-commentary about what you're doing ("I'll search...", "Let me analyze...") - Explanations of your reasoning process - Descriptions of your approach -Just provide the direct answer with proper markdown formatting.""" +Just provide the direct answer with proper markdown formatting. + +CRITICAL: This is a NON-CONVERSATIONAL system. NEVER ask follow-up questions, offer to search again, suggest alternatives, or end with anything like "Would you like me to..." or "Let me know if...". The user cannot reply. Your answer must be complete and self-contained.""" diff --git a/hindsight-api/hindsight_api/engine/reflect/tools.py b/hindsight-api/hindsight_api/engine/reflect/tools.py index 95848c9c..00488712 100644 --- a/hindsight-api/hindsight_api/engine/reflect/tools.py +++ b/hindsight-api/hindsight_api/engine/reflect/tools.py @@ -129,7 +129,7 @@ async def tool_search_observations( pending_consolidation: int = 0, ) -> dict[str, Any]: """ - Search consolidated observations using recall with include_observations. + Search consolidated observations using recall with include_source_facts. Observations are auto-generated from memories. Returns freshness info so the agent knows if it should also verify with recall(). @@ -146,72 +146,24 @@ async def tool_search_observations( pending_consolidation: Number of memories waiting to be consolidated Returns: - Dict with matching observations including freshness info + Dict with matching observations including freshness info and source memories """ - from ..memory_engine import fq_table - - # Use recall to search observations (they come back in results field when fact_type=["observation"]) result = await memory_engine.recall_async( bank_id=bank_id, query=query, - fact_type=["observation"], # Only retrieve observations - max_tokens=max_tokens, # Token budget controls how many observations are returned + fact_type=["observation"], + max_tokens=max_tokens, enable_trace=False, request_context=request_context, tags=tags, tags_match=tags_match, + include_source_facts=True, + max_source_facts_tokens=-1, # No token limit — include all source facts _connection_budget=1, _quiet=True, ) - observations = [] - - # When fact_type=["observation"], results come back in `results` field as MemoryFact objects - # We need to fetch additional fields (proof_count, source_memory_ids) from the database - if result.results: - obs_ids = [m.id for m in result.results] - - # Fetch proof_count and source_memory_ids for these observations - pool = await memory_engine._get_pool() - async with pool.acquire() as conn: - obs_rows = await conn.fetch( - f""" - SELECT id, proof_count, source_memory_ids - FROM {fq_table("memory_units")} - WHERE id = ANY($1::uuid[]) - """, - obs_ids, - ) - obs_data = {str(row["id"]): row for row in obs_rows} - - for m in result.results: - # Get additional data from DB lookup - extra = obs_data.get(m.id, {}) - proof_count = extra.get("proof_count", 1) if extra else 1 - source_ids = extra.get("source_memory_ids", []) if extra else [] - # Convert UUIDs to strings - source_memory_ids = [str(sid) for sid in (source_ids or [])] - - # Determine staleness - is_stale = False - staleness_reason = None - if pending_consolidation > 0: - is_stale = True - staleness_reason = f"{pending_consolidation} memories pending consolidation" - - observations.append( - { - "id": str(m.id), - "text": m.text, - "proof_count": proof_count, - "source_memory_ids": source_memory_ids, - "tags": m.tags or [], - "is_stale": is_stale, - "staleness_reason": staleness_reason, - } - ) - - # Return freshness info (more understandable than raw pending_consolidation count) + is_stale = pending_consolidation > 0 if pending_consolidation == 0: freshness = "up_to_date" elif pending_consolidation < 10: @@ -221,8 +173,10 @@ async def tool_search_observations( return { "query": query, - "count": len(observations), - "observations": observations, + "count": len(result.results), + "observations": [m.model_dump() for m in result.results], + "source_facts": {k: v.model_dump() for k, v in (result.source_facts or {}).items()}, + "is_stale": is_stale, "freshness": freshness, } @@ -233,10 +187,10 @@ async def tool_recall( query: str, request_context: "RequestContext", max_tokens: int = 2048, - max_results: int = 50, tags: list[str] | None = None, tags_match: str = "any", connection_budget: int = 1, + max_chunk_tokens: int = 1000, ) -> dict[str, Any]: """ Search memories using TEMPR retrieval. @@ -250,18 +204,19 @@ async def tool_recall( query: Search query request_context: Request context for authentication max_tokens: Maximum tokens for results (default 2048) - max_results: Maximum number of results tags: Filter by tags (includes untagged memories) tags_match: How to match tags - "any" (OR), "all" (AND), or "exact" connection_budget: Max DB connections for this recall (default 1 for internal ops) + max_chunk_tokens: Maximum tokens for raw source chunk text (default 1000, always included) Returns: - Dict with list of matching memories + Dict with list of matching memories including raw chunk text """ + include_chunks = True result = await memory_engine.recall_async( bank_id=bank_id, query=query, - fact_type=["experience", "world"], # Exclude opinions and observations + fact_type=["experience", "world"], max_tokens=max_tokens, enable_trace=False, request_context=request_context, @@ -269,24 +224,14 @@ async def tool_recall( tags_match=tags_match, _connection_budget=connection_budget, _quiet=True, # Suppress logging for internal operations + include_chunks=include_chunks, + max_chunk_tokens=max_chunk_tokens, ) - memories = [] - for m in result.results[:max_results]: - memories.append( - { - "id": str(m.id), - "text": m.text, - "type": m.fact_type, - "entities": m.entities or [], - "occurred": m.occurred_start, # Already ISO format string - } - ) - return { "query": query, - "count": len(memories), - "memories": memories, + "memories": [m.model_dump() for m in result.results], + "chunks": {k: v.model_dump() for k, v in (result.chunks or {}).items()}, } diff --git a/hindsight-api/hindsight_api/engine/reflect/tools_schema.py b/hindsight-api/hindsight_api/engine/reflect/tools_schema.py index cd952f01..15c14fe1 100644 --- a/hindsight-api/hindsight_api/engine/reflect/tools_schema.py +++ b/hindsight-api/hindsight_api/engine/reflect/tools_schema.py @@ -96,6 +96,10 @@ TOOL_RECALL = { "type": "integer", "description": "Optional limit on result size (default 2048). Use higher values for broader searches.", }, + "max_chunk_tokens": { + "type": "integer", + "description": "Maximum tokens for raw source chunk text included alongside each memory fact (default 1000, min 1000). Chunks provide the surrounding context the fact was extracted from. Increase for broader context.", + }, }, "required": ["reason", "query"], }, diff --git a/hindsight-api/tests/test_consolidation.py b/hindsight-api/tests/test_consolidation.py index 0d80eef5..1379453e 100644 --- a/hindsight-api/tests/test_consolidation.py +++ b/hindsight-api/tests/test_consolidation.py @@ -1435,22 +1435,20 @@ class TestObservationDrillDown: assert result["count"] > 0, "Expected at least one observation" - # Verify source_memory_ids and proof_count are present + # Verify source_fact_ids is present (MemoryFact field name for source memories) obs = result["observations"][0] - assert "source_memory_ids" in obs, "Observation should have source_memory_ids" - assert "proof_count" in obs, "Observation should have proof_count" - assert obs["proof_count"] >= 1, "proof_count should be at least 1" + assert "source_fact_ids" in obs, "Observation should have source_fact_ids" - # If source_memory_ids exist, verify they can be used with expand - if obs["source_memory_ids"]: - assert len(obs["source_memory_ids"]) >= 1, "Should have at least one source memory" + # If source_fact_ids exist, verify they can be used with expand + if obs["source_fact_ids"]: + assert len(obs["source_fact_ids"]) >= 1, "Should have at least one source memory" # Use expand tool to get source memory details async with memory._pool.acquire() as conn: expand_result = await tool_expand( conn=conn, bank_id=bank_id, - memory_ids=obs["source_memory_ids"][:2], # Take first 2 + memory_ids=obs["source_fact_ids"][:2], # Take first 2 depth="chunk", ) @@ -1717,11 +1715,10 @@ class TestHierarchicalRetrieval: query="What was the quarterly revenue?", request_context=request_context, max_tokens=2048, - max_results=10, ) # Should have raw facts with specific numbers - assert recall_result["count"] >= 1, "Recall should find the raw facts" + assert len(recall_result["memories"]) >= 1, "Recall should find the raw facts" # Check that we get the actual numbers from the original memories all_memory_text = " ".join([m["text"] for m in recall_result["memories"]]) diff --git a/hindsight-dev/benchmarks/common/benchmark_runner.py b/hindsight-dev/benchmarks/common/benchmark_runner.py index 10c9ec39..1d98bc25 100644 --- a/hindsight-dev/benchmarks/common/benchmark_runner.py +++ b/hindsight-dev/benchmarks/common/benchmark_runner.py @@ -643,9 +643,13 @@ class BenchmarkRunner: for q in category_5_questions: logging.debug(f" Skipped category=5 question: {q.get('question', 'N/A')[:100]}") - # Filter out category 5 and questions without answers - qa_pairs = [pair for pair in qa_pairs if pair.get("category") != 5 and pair.get("answer")] - questions_to_eval = qa_pairs[:max_questions] if max_questions else qa_pairs + # Filter out category 5 and questions without answers, preserving original indices + indexed_pairs = [ + (orig_idx, pair) + for orig_idx, pair in enumerate(qa_pairs) + if pair.get("category") != 5 and pair.get("answer") + ] + indexed_pairs_to_eval = indexed_pairs[:max_questions] if max_questions else indexed_pairs with Progress( SpinnerColumn(), @@ -655,11 +659,12 @@ class BenchmarkRunner: console=console, ) as progress: task = progress.add_task( - f"[cyan]Evaluating QA for {item_id} - {len(questions_to_eval)} questions", total=len(questions_to_eval) + f"[cyan]Evaluating QA for {item_id} - {len(indexed_pairs_to_eval)} questions", + total=len(indexed_pairs_to_eval), ) # Create tasks for all questions - async def process_question(qa): + async def process_question(orig_idx: int, qa: dict): async with semaphore: question = qa["question"] correct_answer = qa["answer"] @@ -683,6 +688,7 @@ class BenchmarkRunner: ] return { + "question_index": orig_idx, "question": question, "correct_answer": correct_answer, "predicted_answer": predicted_answer, @@ -696,9 +702,10 @@ class BenchmarkRunner: logging.exception(f"Failed to answer question: {question[:100]}") # Mark as invalid if answer generation failed console.print( - f" [red]✗[/red] Failed to answer question: {question[:50]}... Error: {str(e)[:100]}" + f" [red]✗[/red] Failed to answer question [{orig_idx}]: {question[:50]}... Error: {str(e)[:100]}" ) return { + "question_index": orig_idx, "question": question, "correct_answer": correct_answer, "predicted_answer": "ERROR: Failed to generate answer", @@ -709,7 +716,7 @@ class BenchmarkRunner: "error": str(e), } - question_tasks = [process_question(qa) for qa in questions_to_eval] + question_tasks = [process_question(orig_idx, qa) for orig_idx, qa in indexed_pairs_to_eval] # Use as_completed to update progress as results come in results = [] diff --git a/hindsight-dev/benchmarks/locomo/locomo_benchmark.py b/hindsight-dev/benchmarks/locomo/locomo_benchmark.py index 95950e91..2d2bea73 100644 --- a/hindsight-dev/benchmarks/locomo/locomo_benchmark.py +++ b/hindsight-dev/benchmarks/locomo/locomo_benchmark.py @@ -185,27 +185,23 @@ Answer: return f"Error generating answer: {str(e)}", "Error occurred during answer generation.", None -class LoComoThinkAnswerGenerator(LLMAnswerGenerator): - """LoComo answer generator using the think API instead of search + LLM. +class LoComoReflectAnswerGenerator(LLMAnswerGenerator): + """LoComo answer generator using the reflect API instead of search + LLM. - This generator performs its own retrieval internally via the think API, + This generator performs its own retrieval internally via the reflect API, so it doesn't need external search to be performed by the benchmark runner. """ - def __init__(self, memory: "MemoryEngine", agent_id: str, thinking_budget: int = 500): - """Initialize with memory instance and agent_id. + def __init__(self, memory: "MemoryEngine"): + """Initialize with memory instance. Args: memory: MemoryEngine instance - agent_id: Agent identifier for think queries - thinking_budget: Budget for memory exploration """ self.memory = memory - self.agent_id = agent_id - self.thinking_budget = thinking_budget def needs_external_search(self) -> bool: - """Think API does its own retrieval, so no external search needed.""" + """Reflect API does its own retrieval, so no external search needed.""" return False async def generate_answer( @@ -217,72 +213,88 @@ class LoComoThinkAnswerGenerator(LLMAnswerGenerator): bank_id: Optional[str] = None, ) -> Tuple[str, str, Optional[List[Dict[str, Any]]]]: """ - Generate answer using the integrated think API. + Generate answer using the integrated reflect API. - The think API performs both search and answer generation in a single call, - combining agent facts, world facts, and opinions to formulate a response. + The reflect API performs both search and answer generation in a single call, + combining world facts, experience facts, and mental models to formulate a response. Args: question: Question to answer - recall_result: Not used (empty dict), as think does its own retrieval - question_date: Date when the question was asked (currently not used by think API) - question_type: Question category (unused in think API) - bank_id: Not used - think API uses self.agent_id from constructor + recall_result: Not used (empty dict), as reflect does its own retrieval + question_date: Date when the question was asked (currently not used by reflect API) + question_type: Question category (unused in reflect API) + bank_id: Bank ID to query Returns: Tuple of (answer, reasoning, retrieved_memories) - - retrieved_memories: Combined list of all facts from based_on (world, agent, opinion) + - retrieved_memories: Combined list of all facts from based_on """ + from hindsight_api.models import RequestContext + try: - # Use the think API which does both search and answer generation - result = await self.memory.think_async( - agent_id=self.agent_id, - query=question, - thinking_budget=self.thinking_budget, + question_date_str = "" + if question_date: + question_date_str = f"\n# CURRENT DATE:\nThe question is being asked on: {question_date.strftime('%Y-%m-%d %H:%M:%S')} UTC\n" + + query = f""" +# CONTEXT: +You have access to facts and entities from a conversation. +{question_date_str} +# INSTRUCTIONS: +1. Search thoroughly across all available memories before answering - do not stop at the first result +2. Keep searching with different queries until you have a comprehensive answer +3. Carefully analyze all provided memories +4. Pay special attention to the timestamps to determine the answer +5. If the question asks about a specific event or fact, look for direct evidence in the memories +6. If the memories contain contradictory information or multiple instances of an event, say them all +7. Always convert relative time references to specific dates, months, or years. +8. Be as specific as possible when talking about people, places, and events +9. If the answer is not explicitly stated in the memories, use logical reasoning based on the information available to answer (e.g. calculate duration of an event from different memories). + +Question: {question} +""" + + from hindsight_api.engine.memory_engine import Budget + + result = await self.memory.reflect_async( + bank_id=bank_id, + query=query, + budget=Budget.HIGH, + request_context=RequestContext(), ) - # Extract answer and reasoning answer = result.text - # Extract memories from based_on + # Flatten all facts from based_on into retrieved_memories based_on = result.based_on - world_facts = based_on.get("world", []) - agent_facts = based_on.get("agent", []) - opinion_facts = based_on.get("opinion", []) - - # Combine all facts into retrieved_memories retrieved_memories = [] + for facts in based_on.values(): + if isinstance(facts, list): + for fact in facts: + if hasattr(fact, "model_dump"): + retrieved_memories.append(fact.model_dump()) + elif isinstance(fact, dict): + retrieved_memories.append(fact) - # Add world facts - for fact in world_facts: - retrieved_memories.append(fact.model_dump()) - - for fact in agent_facts: - retrieved_memories.append(fact.model_dump()) - for fact in opinion_facts: - retrieved_memories.append(fact.model_dump()) - # Build reasoning summary - num_world = len(world_facts) - num_agent = len(agent_facts) - num_opinion = len(opinion_facts) - - reasoning = f"Think API: {num_world} world facts, {num_agent} agent facts, {num_opinion} opinions" + counts = {k: len(v) for k, v in based_on.items() if isinstance(v, list)} + reasoning = "Reflect API: " + ", ".join(f"{v} {k}" for k, v in counts.items()) return answer, reasoning, retrieved_memories except Exception as e: - return f"Error generating answer: {str(e)}", "Error occurred during think API call.", [] + return f"Error generating answer: {str(e)}", "Error occurred during reflect API call.", [] async def run_benchmark( max_conversations: int = None, max_questions_per_conv: int = None, skip_ingestion: bool = False, - use_think: bool = False, + use_reflect: bool = False, conversation: str = None, api_url: str = None, max_concurrent_questions_override: int = None, only_failed: bool = False, only_invalid: bool = False, + question_index: int = None, ): """ Run the LoComo benchmark. @@ -291,11 +303,12 @@ async def run_benchmark( max_conversations: Maximum number of conversations to evaluate (None for all) max_questions_per_conv: Maximum questions per conversation (None for all) skip_ingestion: Whether to skip ingestion and use existing data - use_think: Whether to use the think API instead of search + LLM + use_reflect: Whether to use the reflect API instead of search + LLM conversation: Specific conversation ID to run (e.g., "conv-26") api_url: Optional API URL to connect to (default: use local memory) only_failed: If True, only run conversations that have failed questions (is_correct=False) only_invalid: If True, only run conversations that have invalid questions (is_invalid=True) + question_index: Run only the question at this index (0-based) within each conversation """ from rich.console import Console @@ -305,7 +318,7 @@ async def run_benchmark( failed_conversation_ids = set() invalid_conversation_ids = set() if only_failed or only_invalid: - suffix = "_think" if use_think else "" + suffix = "_reflect" if use_reflect else "" results_filename = f"benchmark_results{suffix}.json" results_path = Path(__file__).parent / "results" / results_filename @@ -358,11 +371,9 @@ async def run_benchmark( memory = await create_memory_engine() # Select answer generator based on mode - from hindsight_api.engine.memory_engine import Budget - - if use_think: - console.print("[blue]Mode: think (using think API)[/blue]") - answer_generator = LoComoThinkAnswerGenerator(memory=memory, agent_id="locomo", thinking_budget=500) + if use_reflect: + console.print("[blue]Mode: reflect (using reflect API)[/blue]") + answer_generator = LoComoReflectAnswerGenerator(memory=memory) max_concurrent_questions = max_concurrent_questions_override or 4 eval_semaphore_size = 4 else: @@ -398,8 +409,25 @@ async def run_benchmark( dataset.load = filtered_load + # Filter to a single question by index if requested + if question_index is not None: + original_get_qa_pairs = dataset.get_qa_pairs + + def filtered_get_qa_pairs(item: Dict) -> List[Dict[str, Any]]: + pairs = original_get_qa_pairs(item) + if question_index >= len(pairs): + console.print( + f"[red]Error: question index {question_index} out of range (conversation has {len(pairs)} questions)[/red]" + ) + return [] + selected = pairs[question_index] + console.print(f"[cyan]Running single question [{question_index}]: {selected['question']}[/cyan]") + return [selected] + + dataset.get_qa_pairs = filtered_get_qa_pairs + # Determine output filename based on mode - suffix = "_think" if use_think else "" + suffix = "_reflect" if use_reflect else "" results_filename = f"benchmark_results{suffix}.json" output_path = Path(__file__).parent / "results" / results_filename @@ -440,12 +468,12 @@ async def run_benchmark( console.print(f"\n[green]✓[/green] Results saved incrementally to {output_path}") # Generate markdown table - generate_markdown_table(results, use_think=use_think) + generate_markdown_table(results, use_reflect=use_reflect) return results -def generate_markdown_table(results: dict, use_think: bool = False): +def generate_markdown_table(results: dict, use_reflect: bool = False): """ Generate a markdown table with benchmark results. @@ -463,7 +491,7 @@ def generate_markdown_table(results: dict, use_think: bool = False): # Build markdown content lines = [] - mode_str = " (Think Mode)" if use_think else "" + mode_str = " (Reflect Mode)" if use_reflect else "" lines.append(f"# LoComo Benchmark Results{mode_str}") lines.append("") @@ -514,7 +542,7 @@ def generate_markdown_table(results: dict, use_think: bool = False): ) # Write to file with suffix - suffix = "_think" if use_think else "" + suffix = "_reflect" if use_reflect else "" output_file = Path(__file__).parent / "results" / f"results_table{suffix}.md" output_file.parent.mkdir(parents=True, exist_ok=True) output_file.write_text("\n".join(lines)) @@ -531,7 +559,7 @@ if __name__ == "__main__": parser.add_argument("--max-conversations", type=int, default=None, help="Maximum conversations to evaluate") parser.add_argument("--max-questions", type=int, default=None, help="Maximum questions per conversation") parser.add_argument("--skip-ingestion", action="store_true", help="Skip ingestion and use existing data") - parser.add_argument("--use-think", action="store_true", help="Use think API instead of search + LLM") + parser.add_argument("--use-reflect", action="store_true", help="Use reflect API instead of search + LLM") parser.add_argument( "--conversation", type=str, default=None, help='Run only specific conversation (e.g., "conv-26")' ) @@ -557,6 +585,12 @@ if __name__ == "__main__": action="store_true", help="Only run conversations that have invalid questions (is_invalid=True). Requires existing results file.", ) + parser.add_argument( + "--question-index", + type=int, + default=None, + help="Run only the question at this 0-based index within each conversation (e.g., 11)", + ) args = parser.parse_args() @@ -569,11 +603,12 @@ if __name__ == "__main__": max_conversations=args.max_conversations, max_questions_per_conv=args.max_questions, skip_ingestion=args.skip_ingestion, - use_think=args.use_think, + use_reflect=args.use_reflect, conversation=args.conversation, api_url=args.api_url, max_concurrent_questions_override=args.max_concurrent_questions, only_failed=args.only_failed, only_invalid=args.only_invalid, + question_index=args.question_index, ) ) diff --git a/hindsight-dev/benchmarks/visualizer/main.py b/hindsight-dev/benchmarks/visualizer/main.py index 184a7bb2..aa555a64 100644 --- a/hindsight-dev/benchmarks/visualizer/main.py +++ b/hindsight-dev/benchmarks/visualizer/main.py @@ -83,7 +83,7 @@ app, rt = fast_app() def load_locomo_results(mode: str = "search") -> dict[str, Any] | None: """Load LoComo benchmark results.""" - filename = "benchmark_results_think.json" if mode == "think" else "benchmark_results.json" + filename = "benchmark_results_reflect.json" if mode == "reflect" else "benchmark_results.json" results_path = BENCHMARKS_DIR / "locomo" / "results" / filename if not results_path.exists(): @@ -136,7 +136,7 @@ def get(): Select( Option("-- Choose a benchmark --", value="", selected=True), Option("LoComo (search mode)", value="/locomo/search"), - Option("LoComo (think mode)", value="/locomo/think"), + Option("LoComo (reflect mode)", value="/locomo/reflect"), Option("LongMemEval", value="/longmemeval"), onchange="if(this.value) window.location.href = this.value;", cls="w-full px-3 py-2 border border-input rounded-md bg-background text-foreground focus:outline-none focus:ring-2 focus:ring-ring", @@ -234,7 +234,7 @@ def get_locomo(mode: str, filter_type: str = "all", category_filter: str = "all" elif result.get("is_correct"): category_stats[category]["correct"] += 1 - mode_label = " (Think Mode)" if mode == "think" else " (Search Mode)" + mode_label = " (Reflect Mode)" if mode == "reflect" else " (Search Mode)" # Overall stats stats_html = Div( @@ -550,7 +550,7 @@ def get_locomo_item(mode: str, item_idx: int, filter_type: str = "all", category category_stats[cat_id]["invalid"] = stats.get("invalid", 0) # Overall stats for this item - mode_label = " (Think Mode)" if mode == "think" else " (Search Mode)" + mode_label = " (Reflect Mode)" if mode == "reflect" else " (Search Mode)" stats_html = Div( H3(f"{item_id}{mode_label} - Performance", cls="text-2xl font-bold text-foreground mb-6"), Div( @@ -759,6 +759,7 @@ def get_locomo_item(mode: str, item_idx: int, filter_type: str = "all", category correct_answer = result.get("correct_answer", "") predicted_answer = result.get("predicted_answer", "") category = get_category_name(result.get("category", "Unknown")) + question_index = result.get("question_index", q_idx) icon = "⚠️" if is_invalid else ("✅" if is_correct else "❌") border_class = ( @@ -771,7 +772,7 @@ def get_locomo_item(mode: str, item_idx: int, filter_type: str = "all", category Div( # Header Div( - P(f"{icon} Question {q_idx + 1}", cls="text-lg font-semibold text-foreground"), + P(f"{icon} Question #{question_index}", cls="text-lg font-semibold text-foreground"), P(f"Category: {category}", cls="text-sm text-muted-foreground"), cls="mb-4", ), @@ -1317,6 +1318,7 @@ def get_longmemeval_item(item_idx: int, filter_type: str = "all"): correct_answer = result.get("correct_answer", "") predicted_answer = result.get("predicted_answer", "") category = result.get("category", "Unknown") + question_index = result.get("question_index", q_idx) icon = "⚠️" if is_invalid else ("✅" if is_correct else "❌") border_class = ( @@ -1329,7 +1331,7 @@ def get_longmemeval_item(item_idx: int, filter_type: str = "all"): Div( # Header Div( - P(f"{icon} Question {q_idx + 1}", cls="text-lg font-semibold text-foreground"), + P(f"{icon} Question #{question_index}", cls="text-lg font-semibold text-foreground"), P(f"Category: {category}", cls="text-sm text-muted-foreground"), cls="mb-4", ),