chore: remove dead code (#245)
* chore: remove dead code * chore: remove extract_opinions from test and regenerate openapi - Remove extract_opinions parameter from test_fact_extraction_analysis - Regenerate OpenAPI spec after removing entity observations code * chore: update generated files and apply formatting - Regenerate Python and TypeScript client SDKs after main merge - Apply ruff formatting to llm_wrapper.py * fix: accept and filter deprecated 'opinion' fact type in recall The dead code removal eliminated support for the 'opinion' fact type, but existing clients may still pass it. Instead of rejecting it with a ValueError, silently filter it out before validation to maintain backward compatibility.
This commit is contained in:
Nicolò Boschi
GitHub
parent
0da77ce2c9
commit
ab5e31f203
25 changed files with 1234 additions and 2079 deletions
|
|
@ -92,8 +92,7 @@ class RecallRequest(BaseModel):
|
|||
query: str
|
||||
types: list[str] | None = Field(
|
||||
default=None,
|
||||
description="List of fact types to recall: 'world', 'experience', 'observation'. Defaults to world and experience if not specified. "
|
||||
"Note: 'opinion' is accepted but ignored (opinions are excluded from recall).",
|
||||
description="List of fact types to recall: 'world', 'experience', 'observation'. Defaults to world and experience if not specified.",
|
||||
)
|
||||
budget: Budget = Budget.MID
|
||||
max_tokens: int = 4096
|
||||
|
|
@ -504,13 +503,6 @@ class ReflectRequest(BaseModel):
|
|||
)
|
||||
|
||||
|
||||
class OpinionItem(BaseModel):
|
||||
"""Model for an opinion with confidence score."""
|
||||
|
||||
text: str
|
||||
confidence: float
|
||||
|
||||
|
||||
class ReflectFact(BaseModel):
|
||||
"""A fact used in think response."""
|
||||
|
||||
|
|
@ -529,7 +521,7 @@ class ReflectFact(BaseModel):
|
|||
|
||||
id: str | None = None
|
||||
text: str
|
||||
type: str | None = None # fact type: world, experience, opinion
|
||||
type: str | None = None # fact type: world, experience, observation
|
||||
context: str | None = None
|
||||
occurred_start: str | None = None
|
||||
occurred_end: str | None = None
|
||||
|
|
@ -1707,9 +1699,7 @@ def _register_routes(app: FastAPI):
|
|||
description="Recall memory using semantic similarity and spreading activation.\n\n"
|
||||
"The type parameter is optional and must be one of:\n"
|
||||
"- `world`: General knowledge about people, places, events, and things that happen\n"
|
||||
"- `experience`: Memories about experience, conversations, actions taken, and tasks performed\n"
|
||||
"- `opinion`: The bank's formed beliefs, perspectives, and viewpoints\n\n"
|
||||
"Set `include_entities=true` to get entity observations alongside recall results.",
|
||||
"- `experience`: Memories about experience, conversations, actions taken, and tasks performed",
|
||||
operation_id="recall_memories",
|
||||
tags=["Memory"],
|
||||
)
|
||||
|
|
@ -1723,10 +1713,8 @@ def _register_routes(app: FastAPI):
|
|||
metrics = get_metrics_collector()
|
||||
|
||||
try:
|
||||
# Default to world and experience if not specified (exclude observation and opinion)
|
||||
# Filter out 'opinion' even if requested - opinions are excluded from recall
|
||||
# Default to world and experience if not specified (exclude observation)
|
||||
fact_types = request.types if request.types else list(VALID_RECALL_FACT_TYPES)
|
||||
fact_types = [ft for ft in fact_types if ft != "opinion"]
|
||||
|
||||
# Parse query_timestamp if provided
|
||||
question_date = None
|
||||
|
|
@ -1858,8 +1846,7 @@ def _register_routes(app: FastAPI):
|
|||
"2. Retrieves world facts relevant to the query\n"
|
||||
"3. Retrieves existing opinions (bank's perspectives)\n"
|
||||
"4. Uses LLM to formulate a contextual answer\n"
|
||||
"5. Extracts and stores any new opinions formed\n"
|
||||
"6. Returns plain text answer, the facts used, and new opinions",
|
||||
"5. Returns plain text answer and the facts used",
|
||||
operation_id="reflect",
|
||||
tags=["Memory"],
|
||||
)
|
||||
|
|
|
|||
|
|
@ -119,7 +119,6 @@ ENV_RETAIN_CHUNK_SIZE = "HINDSIGHT_API_RETAIN_CHUNK_SIZE"
|
|||
ENV_RETAIN_EXTRACT_CAUSAL_LINKS = "HINDSIGHT_API_RETAIN_EXTRACT_CAUSAL_LINKS"
|
||||
ENV_RETAIN_EXTRACTION_MODE = "HINDSIGHT_API_RETAIN_EXTRACTION_MODE"
|
||||
ENV_RETAIN_CUSTOM_INSTRUCTIONS = "HINDSIGHT_API_RETAIN_CUSTOM_INSTRUCTIONS"
|
||||
ENV_RETAIN_OBSERVATIONS_ASYNC = "HINDSIGHT_API_RETAIN_OBSERVATIONS_ASYNC"
|
||||
|
||||
# Observations settings (consolidated knowledge from facts)
|
||||
ENV_ENABLE_OBSERVATIONS = "HINDSIGHT_API_ENABLE_OBSERVATIONS"
|
||||
|
|
@ -210,7 +209,6 @@ DEFAULT_RETAIN_EXTRACT_CAUSAL_LINKS = True # Extract causal links between facts
|
|||
DEFAULT_RETAIN_EXTRACTION_MODE = "concise" # Extraction mode: "concise", "verbose", or "custom"
|
||||
RETAIN_EXTRACTION_MODES = ("concise", "verbose", "custom") # Allowed extraction modes
|
||||
DEFAULT_RETAIN_CUSTOM_INSTRUCTIONS = None # Custom extraction guidelines (only used when mode="custom")
|
||||
DEFAULT_RETAIN_OBSERVATIONS_ASYNC = False # Run observation generation async (after retain completes)
|
||||
|
||||
# Observations defaults (consolidated knowledge from facts)
|
||||
DEFAULT_ENABLE_OBSERVATIONS = True # Observations enabled by default
|
||||
|
|
@ -397,7 +395,6 @@ class HindsightConfig:
|
|||
retain_extract_causal_links: bool
|
||||
retain_extraction_mode: str
|
||||
retain_custom_instructions: str | None
|
||||
retain_observations_async: bool
|
||||
|
||||
# Observations settings (consolidated knowledge from facts)
|
||||
enable_observations: bool
|
||||
|
|
@ -565,10 +562,6 @@ class HindsightConfig:
|
|||
os.getenv(ENV_RETAIN_EXTRACTION_MODE, DEFAULT_RETAIN_EXTRACTION_MODE)
|
||||
),
|
||||
retain_custom_instructions=os.getenv(ENV_RETAIN_CUSTOM_INSTRUCTIONS) or DEFAULT_RETAIN_CUSTOM_INSTRUCTIONS,
|
||||
retain_observations_async=os.getenv(
|
||||
ENV_RETAIN_OBSERVATIONS_ASYNC, str(DEFAULT_RETAIN_OBSERVATIONS_ASYNC)
|
||||
).lower()
|
||||
== "true",
|
||||
# Observations settings (consolidated knowledge from facts)
|
||||
enable_observations=os.getenv(ENV_ENABLE_OBSERVATIONS, str(DEFAULT_ENABLE_OBSERVATIONS)).lower() == "true",
|
||||
consolidation_batch_size=int(
|
||||
|
|
|
|||
|
|
@ -442,49 +442,6 @@ class MemoryEngineInterface(ABC):
|
|||
"""
|
||||
...
|
||||
|
||||
@abstractmethod
|
||||
async def get_entity_observations(
|
||||
self,
|
||||
bank_id: str,
|
||||
entity_id: str,
|
||||
*,
|
||||
limit: int = 10,
|
||||
request_context: "RequestContext",
|
||||
) -> list[Any]:
|
||||
"""
|
||||
Get observations for an entity.
|
||||
|
||||
Args:
|
||||
bank_id: The memory bank ID.
|
||||
entity_id: The entity ID.
|
||||
limit: Maximum observations.
|
||||
request_context: Request context for authentication.
|
||||
|
||||
Returns:
|
||||
List of EntityObservation objects.
|
||||
"""
|
||||
...
|
||||
|
||||
@abstractmethod
|
||||
async def regenerate_entity_observations(
|
||||
self,
|
||||
bank_id: str,
|
||||
entity_id: str,
|
||||
entity_name: str,
|
||||
*,
|
||||
request_context: "RequestContext",
|
||||
) -> None:
|
||||
"""
|
||||
Regenerate observations for an entity.
|
||||
|
||||
Args:
|
||||
bank_id: The memory bank ID.
|
||||
entity_id: The entity ID.
|
||||
entity_name: The entity's canonical name.
|
||||
request_context: Request context for authentication.
|
||||
"""
|
||||
...
|
||||
|
||||
# =========================================================================
|
||||
# Statistics & Operations
|
||||
# =========================================================================
|
||||
|
|
|
|||
|
|
@ -150,7 +150,7 @@ class LLMProvider:
|
|||
# Strip google/ prefix from model name — native SDK uses bare names
|
||||
# e.g. "google/gemini-2.0-flash-lite-001" -> "gemini-2.0-flash-lite-001"
|
||||
if self.model.startswith("google/"):
|
||||
self.model = self.model[len("google/"):]
|
||||
self.model = self.model[len("google/") :]
|
||||
|
||||
logger.info(
|
||||
f"Vertex AI: project={self._vertexai_project_id}, region={self._vertexai_region}, "
|
||||
|
|
|
|||
|
|
@ -1191,8 +1191,8 @@ class MemoryEngine(MemoryEngineInterface):
|
|||
context: Context about when/why this memory was formed
|
||||
event_date: When the event occurred (defaults to now)
|
||||
document_id: Optional document ID for tracking (always upserts if document already exists)
|
||||
fact_type_override: Override fact type ('world', 'experience', 'opinion')
|
||||
confidence_score: Confidence score for opinions (0.0 to 1.0)
|
||||
fact_type_override: Override fact type ('world', 'experience')
|
||||
confidence_score: Confidence score (0.0 to 1.0)
|
||||
request_context: Request context for authentication.
|
||||
|
||||
Returns:
|
||||
|
|
@ -1247,8 +1247,8 @@ class MemoryEngine(MemoryEngineInterface):
|
|||
- "document_id" (optional): Document ID for this specific content item
|
||||
document_id: **DEPRECATED** - Use "document_id" key in each content dict instead.
|
||||
Applies the same document_id to ALL content items that don't specify their own.
|
||||
fact_type_override: Override fact type for all facts ('world', 'experience', 'opinion')
|
||||
confidence_score: Confidence score for opinions (0.0 to 1.0)
|
||||
fact_type_override: Override fact type for all facts ('world', 'experience')
|
||||
confidence_score: Confidence score (0.0 to 1.0)
|
||||
return_usage: If True, returns tuple of (unit_ids, TokenUsage). Default False for backward compatibility.
|
||||
|
||||
Returns:
|
||||
|
|
@ -1570,16 +1570,16 @@ class MemoryEngine(MemoryEngineInterface):
|
|||
if fact_type is None:
|
||||
fact_type = list(VALID_RECALL_FACT_TYPES)
|
||||
|
||||
# Validate fact types early
|
||||
# Filter out 'opinion' early (deprecated, silently ignore)
|
||||
fact_type = [ft for ft in fact_type if ft != "opinion"]
|
||||
|
||||
# Validate fact types
|
||||
invalid_types = set(fact_type) - VALID_RECALL_FACT_TYPES
|
||||
if invalid_types:
|
||||
raise ValueError(
|
||||
f"Invalid fact type(s): {', '.join(sorted(invalid_types))}. "
|
||||
f"Must be one of: {', '.join(sorted(VALID_RECALL_FACT_TYPES))}"
|
||||
)
|
||||
|
||||
# Filter out 'opinion' - opinions are no longer returned from recall
|
||||
fact_type = [ft for ft in fact_type if ft != "opinion"]
|
||||
if not fact_type:
|
||||
# All requested types were opinions - return empty result
|
||||
return RecallResultModel(results=[], entities={}, chunks={})
|
||||
|
|
@ -2235,44 +2235,15 @@ class MemoryEngine(MemoryEngineInterface):
|
|||
)
|
||||
top_results_dicts.append(result_dict)
|
||||
|
||||
# Get entities for each fact if include_entities is requested
|
||||
fact_entity_map = {} # unit_id -> list of (entity_id, entity_name)
|
||||
if include_entities and top_scored:
|
||||
unit_ids = [uuid.UUID(sr.id) for sr in top_scored]
|
||||
if unit_ids:
|
||||
async with acquire_with_retry(pool) as entity_conn:
|
||||
entity_rows = await entity_conn.fetch(
|
||||
f"""
|
||||
SELECT ue.unit_id, e.id as entity_id, e.canonical_name
|
||||
FROM {fq_table("unit_entities")} ue
|
||||
JOIN {fq_table("entities")} e ON ue.entity_id = e.id
|
||||
WHERE ue.unit_id = ANY($1::uuid[])
|
||||
""",
|
||||
unit_ids,
|
||||
)
|
||||
for row in entity_rows:
|
||||
unit_id = str(row["unit_id"])
|
||||
if unit_id not in fact_entity_map:
|
||||
fact_entity_map[unit_id] = []
|
||||
fact_entity_map[unit_id].append(
|
||||
{"entity_id": str(row["entity_id"]), "canonical_name": row["canonical_name"]}
|
||||
)
|
||||
|
||||
# Convert results to MemoryFact objects
|
||||
memory_facts = []
|
||||
for result_dict in top_results_dicts:
|
||||
result_id = str(result_dict.get("id"))
|
||||
# Get entity names for this fact
|
||||
entity_names = None
|
||||
if include_entities and result_id in fact_entity_map:
|
||||
entity_names = [e["canonical_name"] for e in fact_entity_map[result_id]]
|
||||
|
||||
memory_facts.append(
|
||||
MemoryFact(
|
||||
id=result_id,
|
||||
id=str(result_dict.get("id")),
|
||||
text=result_dict.get("text"),
|
||||
fact_type=result_dict.get("fact_type", "world"),
|
||||
entities=entity_names,
|
||||
entities=None, # Entity observations removed
|
||||
context=result_dict.get("context"),
|
||||
occurred_start=result_dict.get("occurred_start"),
|
||||
occurred_end=result_dict.get("occurred_end"),
|
||||
|
|
@ -2283,38 +2254,12 @@ class MemoryEngine(MemoryEngineInterface):
|
|||
)
|
||||
)
|
||||
|
||||
# Fetch entity observations if requested
|
||||
# Entity observations removed - always set to None
|
||||
entities_dict = None
|
||||
total_entity_tokens = 0
|
||||
total_chunk_tokens = 0
|
||||
if include_entities and fact_entity_map:
|
||||
# Collect unique entities in order of fact relevance (preserving order from top_scored)
|
||||
# Use a list to maintain order, but track seen entities to avoid duplicates
|
||||
entities_ordered = [] # list of (entity_id, entity_name) tuples
|
||||
seen_entity_ids = set()
|
||||
|
||||
# Iterate through facts in relevance order
|
||||
for sr in top_scored:
|
||||
unit_id = sr.id
|
||||
if unit_id in fact_entity_map:
|
||||
for entity in fact_entity_map[unit_id]:
|
||||
entity_id = entity["entity_id"]
|
||||
entity_name = entity["canonical_name"]
|
||||
if entity_id not in seen_entity_ids:
|
||||
entities_ordered.append((entity_id, entity_name))
|
||||
seen_entity_ids.add(entity_id)
|
||||
|
||||
# Return entities with empty observations (summaries now live in mental models)
|
||||
entities_dict = {}
|
||||
for entity_id, entity_name in entities_ordered:
|
||||
entities_dict[entity_name] = EntityState(
|
||||
entity_id=entity_id,
|
||||
canonical_name=entity_name,
|
||||
observations=[], # Mental models provide this now
|
||||
)
|
||||
|
||||
# Fetch chunks if requested
|
||||
chunks_dict = None
|
||||
total_chunk_tokens = 0
|
||||
if include_chunks and top_scored:
|
||||
from .response_models import ChunkInfo
|
||||
|
||||
|
|
@ -2383,7 +2328,6 @@ class MemoryEngine(MemoryEngineInterface):
|
|||
# Log final recall stats
|
||||
total_time = time.time() - recall_start
|
||||
num_chunks = len(chunks_dict) if chunks_dict else 0
|
||||
num_entities = len(entities_dict) if entities_dict else 0
|
||||
# Include wait times in log if significant
|
||||
wait_parts = []
|
||||
if semaphore_wait > 0.01:
|
||||
|
|
@ -2392,7 +2336,7 @@ class MemoryEngine(MemoryEngineInterface):
|
|||
wait_parts.append(f"conn={max_conn_wait:.3f}s")
|
||||
wait_info = f" | waits: {', '.join(wait_parts)}" if wait_parts else ""
|
||||
log_buffer.append(
|
||||
f"[RECALL {recall_id}] Complete: {len(top_scored)} facts ({total_tokens} tok), {num_chunks} chunks ({total_chunk_tokens} tok), {num_entities} entities ({total_entity_tokens} tok) | {fact_type_summary} | {total_time:.3f}s{wait_info}"
|
||||
f"[RECALL {recall_id}] Complete: {len(top_scored)} facts ({total_tokens} tok), {num_chunks} chunks ({total_chunk_tokens} tok) | {fact_type_summary} | {total_time:.3f}s{wait_info}"
|
||||
)
|
||||
if not quiet:
|
||||
logger.info("\n" + "\n".join(log_buffer))
|
||||
|
|
@ -3566,7 +3510,6 @@ class MemoryEngine(MemoryEngineInterface):
|
|||
ReflectResult containing:
|
||||
- text: Plain text answer
|
||||
- based_on: Empty dict (agent retrieves facts dynamically)
|
||||
- new_opinions: Empty list
|
||||
- structured_output: None (not yet supported for agentic reflect)
|
||||
"""
|
||||
# Use cached LLM config
|
||||
|
|
@ -3891,7 +3834,6 @@ class MemoryEngine(MemoryEngineInterface):
|
|||
result = ReflectResult(
|
||||
text=agent_result.text,
|
||||
based_on=based_on,
|
||||
new_opinions=[], # Learnings stored as mental models
|
||||
structured_output=agent_result.structured_output,
|
||||
usage=usage,
|
||||
tool_trace=tool_trace_result,
|
||||
|
|
@ -3920,32 +3862,6 @@ class MemoryEngine(MemoryEngineInterface):
|
|||
|
||||
return result
|
||||
|
||||
async def get_entity_observations(
|
||||
self,
|
||||
bank_id: str,
|
||||
entity_id: str,
|
||||
*,
|
||||
limit: int = 10,
|
||||
request_context: "RequestContext",
|
||||
) -> list[Any]:
|
||||
"""
|
||||
Get observations for an entity.
|
||||
|
||||
NOTE: Entity observations/summaries have been moved to mental models.
|
||||
This method returns an empty list. Use mental models for entity summaries.
|
||||
|
||||
Args:
|
||||
bank_id: bank IDentifier
|
||||
entity_id: Entity UUID to get observations for
|
||||
limit: Ignored (kept for backwards compatibility)
|
||||
request_context: Request context for authentication.
|
||||
|
||||
Returns:
|
||||
Empty list (observations now in mental models)
|
||||
"""
|
||||
await self._authenticate_tenant(request_context)
|
||||
return []
|
||||
|
||||
async def list_entities(
|
||||
self,
|
||||
bank_id: str,
|
||||
|
|
@ -4132,36 +4048,6 @@ class MemoryEngine(MemoryEngineInterface):
|
|||
await self._authenticate_tenant(request_context)
|
||||
return EntityState(entity_id=entity_id, canonical_name=entity_name, observations=[])
|
||||
|
||||
async def regenerate_entity_observations(
|
||||
self,
|
||||
bank_id: str,
|
||||
entity_id: str,
|
||||
entity_name: str,
|
||||
*,
|
||||
version: str | None = None,
|
||||
conn=None,
|
||||
request_context: "RequestContext",
|
||||
) -> list[str]:
|
||||
"""
|
||||
Regenerate observations for an entity.
|
||||
|
||||
NOTE: Entity observations/summaries have been moved to mental models.
|
||||
This method is now a no-op and returns an empty list.
|
||||
|
||||
Args:
|
||||
bank_id: bank IDentifier
|
||||
entity_id: Entity UUID
|
||||
entity_name: Canonical name of the entity
|
||||
version: Entity's last_seen timestamp when task was created (for deduplication)
|
||||
conn: Optional database connection (ignored)
|
||||
request_context: Request context for authentication.
|
||||
|
||||
Returns:
|
||||
Empty list (observations now in mental models)
|
||||
"""
|
||||
await self._authenticate_tenant(request_context)
|
||||
return []
|
||||
|
||||
# =========================================================================
|
||||
# Statistics & Operations (for HTTP API layer)
|
||||
# =========================================================================
|
||||
|
|
@ -4272,9 +4158,6 @@ class MemoryEngine(MemoryEngineInterface):
|
|||
if not entity_row:
|
||||
return None
|
||||
|
||||
# Get observations for the entity
|
||||
observations = await self.get_entity_observations(bank_id, entity_id, limit=20, request_context=request_context)
|
||||
|
||||
return {
|
||||
"id": str(entity_row["id"]),
|
||||
"canonical_name": entity_row["canonical_name"],
|
||||
|
|
@ -4282,7 +4165,7 @@ class MemoryEngine(MemoryEngineInterface):
|
|||
"first_seen": entity_row["first_seen"].isoformat() if entity_row["first_seen"] else None,
|
||||
"last_seen": entity_row["last_seen"].isoformat() if entity_row["last_seen"] else None,
|
||||
"metadata": entity_row["metadata"] or {},
|
||||
"observations": observations,
|
||||
"observations": [],
|
||||
}
|
||||
|
||||
def _parse_observations(self, observations_raw: list):
|
||||
|
|
|
|||
|
|
@ -263,7 +263,6 @@ class ReflectResult(BaseModel):
|
|||
}
|
||||
],
|
||||
},
|
||||
"new_opinions": ["Machine learning has great potential in healthcare"],
|
||||
"structured_output": {"summary": "ML in healthcare", "confidence": 0.9},
|
||||
"usage": {"input_tokens": 1500, "output_tokens": 500, "total_tokens": 2000},
|
||||
}
|
||||
|
|
@ -272,9 +271,8 @@ class ReflectResult(BaseModel):
|
|||
|
||||
text: str = Field(description="The formulated answer text")
|
||||
based_on: dict[str, Any] = Field(
|
||||
description="Facts used to formulate the answer, organized by type (world, experience, opinion, mental_models, directives)"
|
||||
description="Facts used to formulate the answer, organized by type (world, experience, mental_models, directives)"
|
||||
)
|
||||
new_opinions: list[str] = Field(default_factory=list, description="List of newly formed opinions during reflection")
|
||||
structured_output: dict[str, Any] | None = Field(
|
||||
default=None,
|
||||
description="Structured output parsed according to the provided response schema. Only present when response_schema was provided.",
|
||||
|
|
@ -297,24 +295,6 @@ class ReflectResult(BaseModel):
|
|||
)
|
||||
|
||||
|
||||
class Opinion(BaseModel):
|
||||
"""
|
||||
An opinion with confidence score.
|
||||
|
||||
Opinions represent the bank's formed perspectives on topics,
|
||||
with a confidence level indicating strength of belief.
|
||||
"""
|
||||
|
||||
model_config = ConfigDict(
|
||||
json_schema_extra={
|
||||
"example": {"text": "Machine learning has great potential in healthcare", "confidence": 0.85}
|
||||
}
|
||||
)
|
||||
|
||||
text: str = Field(description="The opinion text")
|
||||
confidence: float = Field(description="Confidence score between 0.0 and 1.0")
|
||||
|
||||
|
||||
class EntityObservation(BaseModel):
|
||||
"""
|
||||
An observation about an entity.
|
||||
|
|
|
|||
|
|
@ -693,7 +693,6 @@ async def _extract_facts_from_chunk(
|
|||
context: str,
|
||||
llm_config: "LLMConfig",
|
||||
agent_name: str = None,
|
||||
extract_opinions: bool = False,
|
||||
) -> tuple[list[dict[str, str]], TokenUsage]:
|
||||
"""
|
||||
Extract facts from a single chunk (internal helper for parallel processing).
|
||||
|
|
@ -707,17 +706,9 @@ async def _extract_facts_from_chunk(
|
|||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
memory_bank_context = f"\n- Your name: {agent_name}" if agent_name and extract_opinions else ""
|
||||
|
||||
# Determine which fact types to extract based on the flag
|
||||
# Determine which fact types to extract
|
||||
# Note: We use "assistant" in the prompt but convert to "bank" for storage
|
||||
if extract_opinions:
|
||||
# Opinion extraction uses a separate prompt (not this one)
|
||||
fact_types_instruction = "Extract ONLY 'opinion' type facts (formed opinions, beliefs, and perspectives). DO NOT extract 'world' or 'assistant' facts."
|
||||
else:
|
||||
fact_types_instruction = (
|
||||
"Extract ONLY 'world' and 'assistant' type facts. DO NOT extract opinions - those are extracted separately."
|
||||
)
|
||||
fact_types_instruction = "Extract ONLY 'world' and 'assistant' type facts."
|
||||
|
||||
# Check config for extraction mode and causal link extraction
|
||||
config = get_config()
|
||||
|
|
@ -770,7 +761,6 @@ async def _extract_facts_from_chunk(
|
|||
# Format event_date with day of week for better temporal reasoning
|
||||
event_date_formatted = event_date.strftime("%A, %B %d, %Y") # e.g., "Monday, June 10, 2024"
|
||||
user_message = f"""Extract facts from the following text chunk.
|
||||
{memory_bank_context}
|
||||
|
||||
Chunk: {chunk_index + 1}/{total_chunks}
|
||||
Event Date: {event_date_formatted} ({event_date.isoformat()})
|
||||
|
|
@ -1029,7 +1019,6 @@ async def _extract_facts_with_auto_split(
|
|||
context: str,
|
||||
llm_config: LLMConfig,
|
||||
agent_name: str = None,
|
||||
extract_opinions: bool = False,
|
||||
) -> tuple[list[dict[str, str]], TokenUsage]:
|
||||
"""
|
||||
Extract facts from a chunk with automatic splitting if output exceeds token limits.
|
||||
|
|
@ -1045,7 +1034,6 @@ async def _extract_facts_with_auto_split(
|
|||
context: Context about the conversation/document
|
||||
llm_config: LLM configuration to use
|
||||
agent_name: Optional agent name (memory owner)
|
||||
extract_opinions: If True, extract ONLY opinions. If False, extract world and agent facts (no opinions)
|
||||
|
||||
Returns:
|
||||
Tuple of (facts list, token usage) extracted from the chunk (possibly from sub-chunks)
|
||||
|
|
@ -1064,7 +1052,6 @@ async def _extract_facts_with_auto_split(
|
|||
context=context,
|
||||
llm_config=llm_config,
|
||||
agent_name=agent_name,
|
||||
extract_opinions=extract_opinions,
|
||||
)
|
||||
except OutputTooLongError:
|
||||
# Output exceeded token limits - split the chunk in half and retry
|
||||
|
|
@ -1109,7 +1096,6 @@ async def _extract_facts_with_auto_split(
|
|||
context=context,
|
||||
llm_config=llm_config,
|
||||
agent_name=agent_name,
|
||||
extract_opinions=extract_opinions,
|
||||
),
|
||||
_extract_facts_with_auto_split(
|
||||
chunk=second_half,
|
||||
|
|
@ -1119,7 +1105,6 @@ async def _extract_facts_with_auto_split(
|
|||
context=context,
|
||||
llm_config=llm_config,
|
||||
agent_name=agent_name,
|
||||
extract_opinions=extract_opinions,
|
||||
),
|
||||
]
|
||||
|
||||
|
|
@ -1143,7 +1128,6 @@ async def extract_facts_from_text(
|
|||
llm_config: LLMConfig,
|
||||
agent_name: str,
|
||||
context: str = "",
|
||||
extract_opinions: bool = False,
|
||||
) -> tuple[list[Fact], list[tuple[str, int]], TokenUsage]:
|
||||
"""
|
||||
Extract semantic facts from conversational or narrative text using LLM.
|
||||
|
|
@ -1160,7 +1144,6 @@ async def extract_facts_from_text(
|
|||
context: Context about the conversation/document
|
||||
llm_config: LLM configuration to use
|
||||
agent_name: Agent name (memory owner)
|
||||
extract_opinions: If True, extract ONLY opinions. If False, extract world and bank facts (no opinions)
|
||||
|
||||
Returns:
|
||||
Tuple of (facts, chunks, usage) where:
|
||||
|
|
@ -1188,7 +1171,6 @@ async def extract_facts_from_text(
|
|||
context=context,
|
||||
llm_config=llm_config,
|
||||
agent_name=agent_name,
|
||||
extract_opinions=extract_opinions,
|
||||
)
|
||||
for i, chunk in enumerate(chunks)
|
||||
]
|
||||
|
|
@ -1220,7 +1202,7 @@ SECONDS_PER_FACT = 10
|
|||
|
||||
|
||||
async def extract_facts_from_contents(
|
||||
contents: list[RetainContent], llm_config, agent_name: str, extract_opinions: bool = False
|
||||
contents: list[RetainContent], llm_config, agent_name: str
|
||||
) -> tuple[list[ExtractedFactType], list[ChunkMetadata], TokenUsage]:
|
||||
"""
|
||||
Extract facts from multiple content items in parallel.
|
||||
|
|
@ -1235,7 +1217,6 @@ async def extract_facts_from_contents(
|
|||
contents: List of RetainContent objects to process
|
||||
llm_config: LLM configuration for fact extraction
|
||||
agent_name: Name of the agent (for agent-related fact detection)
|
||||
extract_opinions: If True, extract only opinions; otherwise world/bank facts
|
||||
|
||||
Returns:
|
||||
Tuple of (extracted_facts, chunks_metadata, usage)
|
||||
|
|
@ -1254,7 +1235,6 @@ async def extract_facts_from_contents(
|
|||
context=item.context,
|
||||
llm_config=llm_config,
|
||||
agent_name=agent_name,
|
||||
extract_opinions=extract_opinions,
|
||||
)
|
||||
fact_extraction_tasks.append(task)
|
||||
|
||||
|
|
|
|||
|
|
@ -101,11 +101,8 @@ async def retain_batch(
|
|||
|
||||
# Step 1: Extract facts from all contents
|
||||
step_start = time.time()
|
||||
extract_opinions = fact_type_override == "opinion"
|
||||
|
||||
extracted_facts, chunks, usage = await fact_extraction.extract_facts_from_contents(
|
||||
contents, llm_config, agent_name, extract_opinions
|
||||
)
|
||||
extracted_facts, chunks, usage = await fact_extraction.extract_facts_from_contents(contents, llm_config, agent_name)
|
||||
log_buffer.append(
|
||||
f"[1] Extract facts: {len(extracted_facts)} facts, {len(chunks)} chunks from {len(contents)} contents in {time.time() - step_start:.3f}s"
|
||||
)
|
||||
|
|
|
|||
|
|
@ -19,7 +19,6 @@ async def extract_facts(
|
|||
context: str = "",
|
||||
llm_config: "LLMConfig" = None,
|
||||
agent_name: str = None,
|
||||
extract_opinions: bool = False,
|
||||
) -> tuple[list["Fact"], list[tuple[str, int]]]:
|
||||
"""
|
||||
Extract semantic facts from text using LLM.
|
||||
|
|
@ -36,7 +35,6 @@ async def extract_facts(
|
|||
context: Context about the conversation/document
|
||||
llm_config: LLM configuration to use
|
||||
agent_name: Optional agent name to help identify agent-related facts
|
||||
extract_opinions: If True, extract ONLY opinions. If False, extract world and agent facts (no opinions)
|
||||
|
||||
Returns:
|
||||
Tuple of (facts, chunks) where:
|
||||
|
|
@ -55,7 +53,6 @@ async def extract_facts(
|
|||
context=context,
|
||||
llm_config=llm_config,
|
||||
agent_name=agent_name,
|
||||
extract_opinions=extract_opinions,
|
||||
)
|
||||
|
||||
if not facts:
|
||||
|
|
|
|||
|
|
@ -239,7 +239,6 @@ def main():
|
|||
retain_extract_causal_links=config.retain_extract_causal_links,
|
||||
retain_extraction_mode=config.retain_extraction_mode,
|
||||
retain_custom_instructions=config.retain_custom_instructions,
|
||||
retain_observations_async=config.retain_observations_async,
|
||||
enable_observations=config.enable_observations,
|
||||
consolidation_batch_size=config.consolidation_batch_size,
|
||||
consolidation_max_tokens=config.consolidation_max_tokens,
|
||||
|
|
|
|||
|
|
@ -189,7 +189,7 @@ class MetricsCollectorBase:
|
|||
Args:
|
||||
provider: LLM provider name (openai, anthropic, gemini, groq, ollama, lmstudio)
|
||||
model: Model name
|
||||
scope: Scope identifier (e.g., "memory", "reflect", "entity_observation")
|
||||
scope: Scope identifier (e.g., "memory", "reflect", "consolidation")
|
||||
duration: Call duration in seconds
|
||||
input_tokens: Number of input/prompt tokens
|
||||
output_tokens: Number of output/completion tokens
|
||||
|
|
@ -321,7 +321,7 @@ class MetricsCollector(MetricsCollectorBase):
|
|||
pass
|
||||
|
||||
Args:
|
||||
operation: Operation name (retain, recall, reflect, entity_observation)
|
||||
operation: Operation name (retain, recall, reflect, consolidation)
|
||||
bank_id: Memory bank ID
|
||||
source: Source of the operation (api, reflect, internal)
|
||||
budget: Optional budget level (low, mid, high)
|
||||
|
|
@ -371,7 +371,7 @@ class MetricsCollector(MetricsCollectorBase):
|
|||
Args:
|
||||
provider: LLM provider name (openai, anthropic, gemini, groq, ollama, lmstudio)
|
||||
model: Model name
|
||||
scope: Scope identifier (e.g., "memory", "reflect", "entity_observation")
|
||||
scope: Scope identifier (e.g., "memory", "reflect", "consolidation")
|
||||
duration: Call duration in seconds
|
||||
input_tokens: Number of input/prompt tokens
|
||||
output_tokens: Number of output/completion tokens
|
||||
|
|
|
|||
|
|
@ -58,7 +58,6 @@ async def test_fact_extraction_basic_analysis(llm_config):
|
|||
llm_config=llm_config,
|
||||
agent_name="test-agent",
|
||||
context="Friday Standup meeting",
|
||||
extract_opinions=False,
|
||||
)
|
||||
|
||||
duration = time.time() - start_time
|
||||
|
|
|
|||
|
|
@ -358,7 +358,7 @@ class TestLLMMetrics:
|
|||
collector.record_llm_call(
|
||||
provider="gemini",
|
||||
model="gemini-pro",
|
||||
scope="entity_observation",
|
||||
scope="memory",
|
||||
duration=2.0,
|
||||
success=True,
|
||||
)
|
||||
|
|
@ -369,11 +369,11 @@ class TestLLMMetrics:
|
|||
assert call_args[0][0] == 1
|
||||
assert call_args[0][1]["provider"] == "gemini"
|
||||
assert call_args[0][1]["model"] == "gemini-pro"
|
||||
assert call_args[0][1]["scope"] == "entity_observation"
|
||||
assert call_args[0][1]["scope"] == "memory"
|
||||
|
||||
def test_record_llm_call_different_scopes(self, collector):
|
||||
"""Test recording LLM calls with different scopes."""
|
||||
scopes = ["memory", "reflect", "entity_observation", "answer"]
|
||||
scopes = ["memory", "reflect", "consolidation", "answer"]
|
||||
|
||||
for scope in scopes:
|
||||
collector.llm_duration.record.reset_mock()
|
||||
|
|
|
|||
|
|
@ -469,7 +469,6 @@ async def test_mixed_language_entities(memory, request_context):
|
|||
budget=Budget.MID,
|
||||
max_tokens=1000,
|
||||
fact_type=["world"],
|
||||
include_entities=True,
|
||||
request_context=request_context,
|
||||
)
|
||||
|
||||
|
|
|
|||
|
|
@ -91,156 +91,13 @@ async def test_entity_extraction_on_retain(memory, request_context):
|
|||
await conn.execute("DELETE FROM entities WHERE bank_id = $1", bank_id)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_regenerate_entity_observations(memory, request_context):
|
||||
"""
|
||||
Test explicit regeneration of summary for an entity.
|
||||
"""
|
||||
bank_id = f"test_regen_obs_{datetime.now(timezone.utc).timestamp()}"
|
||||
|
||||
try:
|
||||
# Store facts about an entity
|
||||
await memory.retain_async(
|
||||
bank_id=bank_id,
|
||||
content="Sarah is a product manager who loves user research and data analysis.",
|
||||
context="work info",
|
||||
event_date=datetime(2024, 1, 15, tzinfo=timezone.utc),
|
||||
request_context=request_context,
|
||||
)
|
||||
|
||||
await memory.wait_for_background_tasks()
|
||||
|
||||
# Find the Sarah entity
|
||||
pool = await memory._get_pool()
|
||||
async with pool.acquire() as conn:
|
||||
entity_row = await conn.fetchrow(
|
||||
"""
|
||||
SELECT id, canonical_name
|
||||
FROM entities
|
||||
WHERE bank_id = $1 AND LOWER(canonical_name) LIKE '%sarah%'
|
||||
LIMIT 1
|
||||
""",
|
||||
bank_id
|
||||
)
|
||||
|
||||
if entity_row:
|
||||
entity_id = str(entity_row['id'])
|
||||
entity_name = entity_row['canonical_name']
|
||||
|
||||
# Manually regenerate summary (via observations API for backwards compat)
|
||||
created_ids = await memory.regenerate_entity_observations(
|
||||
bank_id=bank_id,
|
||||
entity_id=entity_id,
|
||||
entity_name=entity_name,
|
||||
request_context=request_context,
|
||||
)
|
||||
|
||||
print(f"\n=== Regenerated Summary ===")
|
||||
print(f"Created {len(created_ids)} summary for {entity_name}")
|
||||
|
||||
# Get entity state
|
||||
state = await memory.get_entity_state(
|
||||
bank_id, entity_id, entity_name, request_context=request_context
|
||||
)
|
||||
for obs in state.observations:
|
||||
print(f" - {obs.text}")
|
||||
|
||||
# Verify summary was created
|
||||
if len(created_ids) > 0:
|
||||
assert len(state.observations) == 1, "Should have exactly 1 observation (the summary)"
|
||||
print(f"Summary regenerated successfully")
|
||||
else:
|
||||
print(f"Note: No summary was regenerated")
|
||||
|
||||
else:
|
||||
print(f"Note: No 'Sarah' entity was extracted")
|
||||
|
||||
finally:
|
||||
# Cleanup
|
||||
pool = await memory._get_pool()
|
||||
async with pool.acquire() as conn:
|
||||
await conn.execute("DELETE FROM memory_units WHERE bank_id = $1", bank_id)
|
||||
await conn.execute("DELETE FROM entities WHERE bank_id = $1", bank_id)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_entity_state_retrieval(memory, request_context):
|
||||
"""
|
||||
Test retrieving entity state with facts.
|
||||
"""
|
||||
bank_id = f"test_entity_state_{datetime.now(timezone.utc).timestamp()}"
|
||||
|
||||
try:
|
||||
# Store facts
|
||||
await memory.retain_async(
|
||||
bank_id=bank_id,
|
||||
content="Alice works at Google as a senior software engineer.",
|
||||
context="work info",
|
||||
event_date=datetime(2024, 1, 15, tzinfo=timezone.utc),
|
||||
request_context=request_context,
|
||||
)
|
||||
await memory.retain_async(
|
||||
bank_id=bank_id,
|
||||
content="Alice loves hiking and outdoor photography.",
|
||||
context="hobbies",
|
||||
event_date=datetime(2024, 1, 16, tzinfo=timezone.utc),
|
||||
request_context=request_context,
|
||||
)
|
||||
|
||||
# Find the Alice entity
|
||||
pool = await memory._get_pool()
|
||||
async with pool.acquire() as conn:
|
||||
entity_row = await conn.fetchrow(
|
||||
"""
|
||||
SELECT id, canonical_name
|
||||
FROM entities
|
||||
WHERE bank_id = $1 AND LOWER(canonical_name) LIKE '%alice%'
|
||||
LIMIT 1
|
||||
""",
|
||||
bank_id
|
||||
)
|
||||
|
||||
assert entity_row is not None, "Alice entity should have been extracted"
|
||||
|
||||
entity_id = str(entity_row['id'])
|
||||
entity_name = entity_row['canonical_name']
|
||||
|
||||
# Check fact count
|
||||
async with pool.acquire() as conn:
|
||||
fact_count = await conn.fetchval(
|
||||
"SELECT COUNT(*) FROM unit_entities WHERE entity_id = $1",
|
||||
entity_row['id']
|
||||
)
|
||||
|
||||
print(f"\n=== Entity State Test ===")
|
||||
print(f"Entity: {entity_name} (id: {entity_id})")
|
||||
print(f"Linked facts: {fact_count}")
|
||||
|
||||
# Get entity state
|
||||
state = await memory.get_entity_state(
|
||||
bank_id, entity_id, entity_name, request_context=request_context
|
||||
)
|
||||
|
||||
assert state.entity_id == entity_id
|
||||
assert state.canonical_name == entity_name
|
||||
print(f"Entity state retrieved successfully")
|
||||
|
||||
finally:
|
||||
# Cleanup
|
||||
pool = await memory._get_pool()
|
||||
async with pool.acquire() as conn:
|
||||
await conn.execute("DELETE FROM memory_units WHERE bank_id = $1", bank_id)
|
||||
await conn.execute("DELETE FROM entities WHERE bank_id = $1", bank_id)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_search_with_include_entities(memory, request_context):
|
||||
"""
|
||||
Test that search with include_entities=True returns entity information.
|
||||
Test that recall accepts include_entities parameter for backwards compatibility.
|
||||
|
||||
This test verifies that:
|
||||
1. Entities are extracted after retain
|
||||
2. Entity info is returned in recall results with include_entities=True
|
||||
Note: Entity observations have been deprecated. This test verifies the parameter
|
||||
is still accepted without errors.
|
||||
"""
|
||||
bank_id = f"test_search_ent_{datetime.now(timezone.utc).timestamp()}"
|
||||
|
||||
|
|
@ -249,10 +106,6 @@ async def test_search_with_include_entities(memory, request_context):
|
|||
contents = [
|
||||
"Alice is a data scientist who works on recommendation systems at Netflix.",
|
||||
"Alice presented her research at the ML conference last month.",
|
||||
"Alice is an expert in deep learning and neural networks.",
|
||||
"Alice graduated from Stanford with a PhD in Computer Science.",
|
||||
"Alice leads a team of 5 data scientists at Netflix.",
|
||||
"Alice published a paper on collaborative filtering algorithms.",
|
||||
]
|
||||
|
||||
for i, content in enumerate(contents):
|
||||
|
|
@ -267,7 +120,7 @@ async def test_search_with_include_entities(memory, request_context):
|
|||
# Wait for background tasks
|
||||
await memory.wait_for_background_tasks()
|
||||
|
||||
# Search with include_entities=True
|
||||
# Search with include_entities=True (should be accepted for backwards compatibility)
|
||||
result = await memory.recall_async(
|
||||
bank_id=bank_id,
|
||||
query="What does Alice do?",
|
||||
|
|
@ -279,98 +132,9 @@ async def test_search_with_include_entities(memory, request_context):
|
|||
request_context=request_context,
|
||||
)
|
||||
|
||||
print(f"\n=== Search Results ===")
|
||||
print(f"Found {len(result.results)} facts")
|
||||
for fact in result.results:
|
||||
print(f" - {fact.text}")
|
||||
if fact.entities:
|
||||
print(f" Entities: {', '.join(fact.entities)}")
|
||||
|
||||
# Verify results
|
||||
# Verify recall works
|
||||
assert len(result.results) > 0, "Should find some facts"
|
||||
|
||||
# Check if entities are included in facts
|
||||
facts_with_entities = [f for f in result.results if f.entities]
|
||||
assert len(facts_with_entities) > 0, "Some facts should have entity information"
|
||||
print(f"{len(facts_with_entities)} facts have entity information")
|
||||
|
||||
# Check if entity info is returned
|
||||
if result.entities:
|
||||
print(f"Entity info included for {len(result.entities)} entities")
|
||||
|
||||
# Verify Alice entity is in results
|
||||
alice_found = False
|
||||
for name, state in result.entities.items():
|
||||
assert state.canonical_name == name, "Entity canonical_name should match key"
|
||||
assert state.entity_id, "Entity should have an ID"
|
||||
if "alice" in name.lower():
|
||||
alice_found = True
|
||||
print(f"Alice entity found: {name}")
|
||||
|
||||
assert alice_found, "Alice entity should be in recall results"
|
||||
|
||||
finally:
|
||||
# Cleanup
|
||||
pool = await memory._get_pool()
|
||||
async with pool.acquire() as conn:
|
||||
await conn.execute("DELETE FROM memory_units WHERE bank_id = $1", bank_id)
|
||||
await conn.execute("DELETE FROM entities WHERE bank_id = $1", bank_id)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_get_entity_state(memory, request_context):
|
||||
"""
|
||||
Test getting the full state of an entity.
|
||||
"""
|
||||
bank_id = f"test_entity_state_{datetime.now(timezone.utc).timestamp()}"
|
||||
|
||||
try:
|
||||
# Store facts
|
||||
await memory.retain_async(
|
||||
bank_id=bank_id,
|
||||
content="Bob is a frontend developer who specializes in React and TypeScript.",
|
||||
context="work info",
|
||||
event_date=datetime(2024, 1, 15, tzinfo=timezone.utc),
|
||||
request_context=request_context,
|
||||
)
|
||||
|
||||
await memory.wait_for_background_tasks()
|
||||
|
||||
# Find entity
|
||||
pool = await memory._get_pool()
|
||||
async with pool.acquire() as conn:
|
||||
entity_row = await conn.fetchrow(
|
||||
"""
|
||||
SELECT id, canonical_name
|
||||
FROM entities
|
||||
WHERE bank_id = $1 AND LOWER(canonical_name) LIKE '%bob%'
|
||||
LIMIT 1
|
||||
""",
|
||||
bank_id
|
||||
)
|
||||
|
||||
if entity_row:
|
||||
entity_id = str(entity_row['id'])
|
||||
entity_name = entity_row['canonical_name']
|
||||
|
||||
# Get entity state
|
||||
state = await memory.get_entity_state(
|
||||
bank_id=bank_id,
|
||||
entity_id=entity_id,
|
||||
entity_name=entity_name,
|
||||
limit=10,
|
||||
request_context=request_context,
|
||||
)
|
||||
|
||||
print(f"\n=== Entity State for {entity_name} ===")
|
||||
print(f"Entity ID: {state.entity_id}")
|
||||
print(f"Canonical Name: {state.canonical_name}")
|
||||
print(f"Observations: {len(state.observations)}")
|
||||
for obs in state.observations:
|
||||
print(f" - {obs.text}")
|
||||
|
||||
assert state.entity_id == entity_id, "Entity ID should match"
|
||||
assert state.canonical_name == entity_name, "Canonical name should match"
|
||||
print(f"Found {len(result.results)} facts")
|
||||
|
||||
finally:
|
||||
# Cleanup
|
||||
|
|
|
|||
|
|
@ -16,7 +16,6 @@ async def test_retain_with_chunks(memory, request_context):
|
|||
Test that retain function:
|
||||
1. Stores facts with associated chunks
|
||||
2. Recall returns chunk_id for each fact
|
||||
3. Recall with include_entities=True also works (for compatibility)
|
||||
"""
|
||||
bank_id = f"test_chunks_{datetime.now(timezone.utc).timestamp()}"
|
||||
document_id = "test_doc_123"
|
||||
|
|
@ -56,7 +55,6 @@ async def test_retain_with_chunks(memory, request_context):
|
|||
budget=Budget.LOW,
|
||||
max_tokens=500,
|
||||
fact_type=["world"], # Search for world facts
|
||||
include_entities=False, # Disable entities for simpler test
|
||||
include_chunks=True, # Enable chunks
|
||||
max_chunk_tokens=8192,
|
||||
request_context=request_context,
|
||||
|
|
@ -146,7 +144,6 @@ async def test_chunks_and_entities_follow_fact_order(memory, request_context):
|
|||
budget=Budget.MID,
|
||||
max_tokens=1000,
|
||||
fact_type=["world"],
|
||||
include_entities=True,
|
||||
include_chunks=True,
|
||||
max_chunk_tokens=8192,
|
||||
request_context=request_context,
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
"""
|
||||
Test think function for opinion generation and consistency.
|
||||
Test reflect (think) function.
|
||||
"""
|
||||
import pytest
|
||||
from datetime import datetime, timezone
|
||||
|
|
@ -7,131 +7,6 @@ from hindsight_api.engine.memory_engine import Budget
|
|||
from hindsight_api import RequestContext
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_think_opinion_consistency(memory, request_context):
|
||||
"""
|
||||
Test that think function:
|
||||
1. Generates an opinion
|
||||
2. Stores the opinion in the database
|
||||
3. Returns consistent response on subsequent calls with the same query
|
||||
"""
|
||||
bank_id = f"test_think_{datetime.now(timezone.utc).timestamp()}"
|
||||
|
||||
try:
|
||||
|
||||
# Store some initial facts to give context for opinion formation
|
||||
await memory.retain_async(
|
||||
bank_id=bank_id,
|
||||
content="Alice is a software engineer who has worked on 5 major projects. She always delivers on time and writes clean, well-documented code.",
|
||||
context="performance review",
|
||||
event_date=datetime(2024, 1, 15, tzinfo=timezone.utc),
|
||||
request_context=request_context,
|
||||
)
|
||||
|
||||
await memory.retain_async(
|
||||
bank_id=bank_id,
|
||||
content="Bob recently joined the team. He missed his first deadline and his code had many bugs.",
|
||||
context="performance review",
|
||||
event_date=datetime(2024, 2, 1, tzinfo=timezone.utc),
|
||||
request_context=request_context,
|
||||
)
|
||||
|
||||
# First think call - should generate opinions
|
||||
query = "Who is a more reliable engineer?"
|
||||
result1 = await memory.reflect_async(
|
||||
bank_id=bank_id,
|
||||
query=query,
|
||||
budget=Budget.LOW,
|
||||
request_context=request_context,
|
||||
)
|
||||
|
||||
print(f"\n=== First Think Call ===")
|
||||
print(f"Answer: {result1.text}")
|
||||
|
||||
# Verify we got an answer
|
||||
assert result1.text, "First think call should return an answer"
|
||||
assert result1.based_on, "Should return based_on facts"
|
||||
|
||||
# Wait for background opinion processing tasks to complete
|
||||
await memory.wait_for_background_tasks()
|
||||
|
||||
# Search for stored opinions to verify they were actually saved
|
||||
pool = await memory._get_pool()
|
||||
async with pool.acquire() as conn:
|
||||
stored_opinions = await conn.fetch(
|
||||
"""
|
||||
SELECT id, text, confidence_score, fact_type
|
||||
FROM memory_units
|
||||
WHERE bank_id = $1 AND fact_type = 'opinion'
|
||||
ORDER BY created_at DESC
|
||||
""",
|
||||
bank_id
|
||||
)
|
||||
|
||||
print(f"\n=== Stored Opinions in Database ===")
|
||||
print(f"Total opinions stored: {len(stored_opinions)}")
|
||||
for op in stored_opinions:
|
||||
print(f" - {op['text']} (confidence: {op['confidence_score']:.2f})")
|
||||
|
||||
# Verify opinions were actually written to database
|
||||
# NOTE: Opinion extraction may not always detect opinions depending on the LLM response format
|
||||
if len(stored_opinions) > 0:
|
||||
assert all(op['fact_type'] == 'opinion' for op in stored_opinions), "All stored items should have fact_type='opinion'"
|
||||
print(f"✓ Opinions were successfully stored in database")
|
||||
else:
|
||||
print(f"⚠ Note: No opinions were extracted/stored (this can happen if the LLM response format doesn't trigger opinion extraction)")
|
||||
|
||||
# Second think call - should use the stored opinions
|
||||
result2 = await memory.reflect_async(
|
||||
bank_id=bank_id,
|
||||
query=query,
|
||||
budget=Budget.LOW,
|
||||
request_context=request_context,
|
||||
)
|
||||
|
||||
print(f"\n=== Second Think Call ===")
|
||||
print(f"Answer: {result2.text}")
|
||||
print(f"Existing opinions used: {len(result2.based_on.get('opinion', []))}")
|
||||
for opinion in result2.based_on.get('opinion', []):
|
||||
print(f" - {opinion.text}")
|
||||
|
||||
# Verify second call also got an answer
|
||||
assert result2.text, "Second think call should return an answer"
|
||||
|
||||
# Verify second call used the stored opinions (if any were stored)
|
||||
if len(stored_opinions) > 0:
|
||||
assert len(result2.based_on.get('opinion', [])) > 0, "Second call should retrieve stored opinions"
|
||||
|
||||
# The responses should be consistent (both should mention the same person as more reliable)
|
||||
# We'll do a basic check that they're not contradictory
|
||||
text1_lower = result1.text.lower()
|
||||
text2_lower = result2.text.lower()
|
||||
|
||||
print(f"\n=== Consistency Check ===")
|
||||
|
||||
# Check if Alice is mentioned as more reliable in first response
|
||||
if 'alice' in text1_lower and ('reliable' in text1_lower or 'better' in text1_lower):
|
||||
print("First response favors Alice")
|
||||
# Second response should also favor Alice (consistency)
|
||||
assert 'alice' in text2_lower, "Second response should also mention Alice"
|
||||
print("Second response also mentions Alice - CONSISTENT ✓")
|
||||
|
||||
# Check if Bob is mentioned
|
||||
if 'bob' in text1_lower:
|
||||
print("First response mentions Bob")
|
||||
if 'bob' in text2_lower:
|
||||
print("Second response also mentions Bob - CONSISTENT ✓")
|
||||
|
||||
print(f"\n✅ Test passed - opinions were formed, stored, and used consistently")
|
||||
|
||||
finally:
|
||||
# Clean up agent data
|
||||
try:
|
||||
await memory.delete_bank(bank_id, request_context=request_context)
|
||||
except Exception as e:
|
||||
print(f"Warning: Error during cleanup: {e}")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_think_without_prior_context(memory, request_context):
|
||||
"""
|
||||
|
|
|
|||
|
|
@ -1644,7 +1644,7 @@ class MemoryApi:
|
|||
) -> RecallResponse:
|
||||
"""Recall memory
|
||||
|
||||
Recall memory using semantic similarity and spreading activation. The type parameter is optional and must be one of: - `world`: General knowledge about people, places, events, and things that happen - `experience`: Memories about experience, conversations, actions taken, and tasks performed - `opinion`: The bank's formed beliefs, perspectives, and viewpoints Set `include_entities=true` to get entity observations alongside recall results.
|
||||
Recall memory using semantic similarity and spreading activation. The type parameter is optional and must be one of: - `world`: General knowledge about people, places, events, and things that happen - `experience`: Memories about experience, conversations, actions taken, and tasks performed
|
||||
|
||||
:param bank_id: (required)
|
||||
:type bank_id: str
|
||||
|
|
@ -1720,7 +1720,7 @@ class MemoryApi:
|
|||
) -> ApiResponse[RecallResponse]:
|
||||
"""Recall memory
|
||||
|
||||
Recall memory using semantic similarity and spreading activation. The type parameter is optional and must be one of: - `world`: General knowledge about people, places, events, and things that happen - `experience`: Memories about experience, conversations, actions taken, and tasks performed - `opinion`: The bank's formed beliefs, perspectives, and viewpoints Set `include_entities=true` to get entity observations alongside recall results.
|
||||
Recall memory using semantic similarity and spreading activation. The type parameter is optional and must be one of: - `world`: General knowledge about people, places, events, and things that happen - `experience`: Memories about experience, conversations, actions taken, and tasks performed
|
||||
|
||||
:param bank_id: (required)
|
||||
:type bank_id: str
|
||||
|
|
@ -1796,7 +1796,7 @@ class MemoryApi:
|
|||
) -> RESTResponseType:
|
||||
"""Recall memory
|
||||
|
||||
Recall memory using semantic similarity and spreading activation. The type parameter is optional and must be one of: - `world`: General knowledge about people, places, events, and things that happen - `experience`: Memories about experience, conversations, actions taken, and tasks performed - `opinion`: The bank's formed beliefs, perspectives, and viewpoints Set `include_entities=true` to get entity observations alongside recall results.
|
||||
Recall memory using semantic similarity and spreading activation. The type parameter is optional and must be one of: - `world`: General knowledge about people, places, events, and things that happen - `experience`: Memories about experience, conversations, actions taken, and tasks performed
|
||||
|
||||
:param bank_id: (required)
|
||||
:type bank_id: str
|
||||
|
|
@ -1950,7 +1950,7 @@ class MemoryApi:
|
|||
) -> ReflectResponse:
|
||||
"""Reflect and generate answer
|
||||
|
||||
Reflect and formulate an answer using bank identity, world facts, and opinions. This endpoint: 1. Retrieves experience (conversations and events) 2. Retrieves world facts relevant to the query 3. Retrieves existing opinions (bank's perspectives) 4. Uses LLM to formulate a contextual answer 5. Extracts and stores any new opinions formed 6. Returns plain text answer, the facts used, and new opinions
|
||||
Reflect and formulate an answer using bank identity, world facts, and opinions. This endpoint: 1. Retrieves experience (conversations and events) 2. Retrieves world facts relevant to the query 3. Retrieves existing opinions (bank's perspectives) 4. Uses LLM to formulate a contextual answer 5. Returns plain text answer and the facts used
|
||||
|
||||
:param bank_id: (required)
|
||||
:type bank_id: str
|
||||
|
|
@ -2026,7 +2026,7 @@ class MemoryApi:
|
|||
) -> ApiResponse[ReflectResponse]:
|
||||
"""Reflect and generate answer
|
||||
|
||||
Reflect and formulate an answer using bank identity, world facts, and opinions. This endpoint: 1. Retrieves experience (conversations and events) 2. Retrieves world facts relevant to the query 3. Retrieves existing opinions (bank's perspectives) 4. Uses LLM to formulate a contextual answer 5. Extracts and stores any new opinions formed 6. Returns plain text answer, the facts used, and new opinions
|
||||
Reflect and formulate an answer using bank identity, world facts, and opinions. This endpoint: 1. Retrieves experience (conversations and events) 2. Retrieves world facts relevant to the query 3. Retrieves existing opinions (bank's perspectives) 4. Uses LLM to formulate a contextual answer 5. Returns plain text answer and the facts used
|
||||
|
||||
:param bank_id: (required)
|
||||
:type bank_id: str
|
||||
|
|
@ -2102,7 +2102,7 @@ class MemoryApi:
|
|||
) -> RESTResponseType:
|
||||
"""Reflect and generate answer
|
||||
|
||||
Reflect and formulate an answer using bank identity, world facts, and opinions. This endpoint: 1. Retrieves experience (conversations and events) 2. Retrieves world facts relevant to the query 3. Retrieves existing opinions (bank's perspectives) 4. Uses LLM to formulate a contextual answer 5. Extracts and stores any new opinions formed 6. Returns plain text answer, the facts used, and new opinions
|
||||
Reflect and formulate an answer using bank identity, world facts, and opinions. This endpoint: 1. Retrieves experience (conversations and events) 2. Retrieves world facts relevant to the query 3. Retrieves existing opinions (bank's perspectives) 4. Uses LLM to formulate a contextual answer 5. Returns plain text answer and the facts used
|
||||
|
||||
:param bank_id: (required)
|
||||
:type bank_id: str
|
||||
|
|
|
|||
|
|
@ -236,9 +236,6 @@ export const getMemory = <ThrowOnError extends boolean = false>(
|
|||
* The type parameter is optional and must be one of:
|
||||
* - `world`: General knowledge about people, places, events, and things that happen
|
||||
* - `experience`: Memories about experience, conversations, actions taken, and tasks performed
|
||||
* - `opinion`: The bank's formed beliefs, perspectives, and viewpoints
|
||||
*
|
||||
* Set `include_entities=true` to get entity observations alongside recall results.
|
||||
*/
|
||||
export const recallMemories = <ThrowOnError extends boolean = false>(
|
||||
options: Options<RecallMemoriesData, ThrowOnError>,
|
||||
|
|
@ -266,8 +263,7 @@ export const recallMemories = <ThrowOnError extends boolean = false>(
|
|||
* 2. Retrieves world facts relevant to the query
|
||||
* 3. Retrieves existing opinions (bank's perspectives)
|
||||
* 4. Uses LLM to formulate a contextual answer
|
||||
* 5. Extracts and stores any new opinions formed
|
||||
* 6. Returns plain text answer, the facts used, and new opinions
|
||||
* 5. Returns plain text answer and the facts used
|
||||
*/
|
||||
export const reflect = <ThrowOnError extends boolean = false>(
|
||||
options: Options<ReflectData, ThrowOnError>,
|
||||
|
|
|
|||
|
|
@ -1178,7 +1178,7 @@ export type RecallRequest = {
|
|||
/**
|
||||
* Types
|
||||
*
|
||||
* List of fact types to recall: 'world', 'experience', 'observation'. Defaults to world and experience if not specified. Note: 'opinion' is accepted but ignored (opinions are excluded from recall).
|
||||
* List of fact types to recall: 'world', 'experience', 'observation'. Defaults to world and experience if not specified.
|
||||
*/
|
||||
types?: Array<string> | null;
|
||||
budget?: Budget;
|
||||
|
|
|
|||
|
|
@ -130,21 +130,21 @@ const TRAIT_LABELS: Record<
|
|||
skepticism: {
|
||||
label: "Skepticism",
|
||||
shortLabel: "S",
|
||||
description: "How skeptical vs trusting when forming opinions",
|
||||
description: "How skeptical vs trusting when forming observations",
|
||||
lowLabel: "Trusting",
|
||||
highLabel: "Skeptical",
|
||||
},
|
||||
literalism: {
|
||||
label: "Literalism",
|
||||
shortLabel: "L",
|
||||
description: "How literally to interpret information when forming opinions",
|
||||
description: "How literally to interpret information when forming observations",
|
||||
lowLabel: "Flexible",
|
||||
highLabel: "Literal",
|
||||
},
|
||||
empathy: {
|
||||
label: "Empathy",
|
||||
shortLabel: "E",
|
||||
description: "How much to consider emotional context when forming opinions",
|
||||
description: "How much to consider emotional context when forming observations",
|
||||
lowLabel: "Detached",
|
||||
highLabel: "Empathetic",
|
||||
},
|
||||
|
|
@ -718,7 +718,9 @@ export function BankProfileView() {
|
|||
<Brain className="w-5 h-5 text-primary" />
|
||||
Disposition Profile
|
||||
</CardTitle>
|
||||
<CardDescription>Traits that shape how opinions are formed via Reflect</CardDescription>
|
||||
<CardDescription>
|
||||
Traits that shape how observations are formed via Reflect
|
||||
</CardDescription>
|
||||
</CardHeader>
|
||||
<CardContent>
|
||||
{profile && (
|
||||
|
|
|
|||
|
|
@ -368,31 +368,6 @@ export function ThinkView() {
|
|||
</CardContent>
|
||||
</Card>
|
||||
|
||||
{/* New Opinions Formed */}
|
||||
{result.new_opinions && result.new_opinions.length > 0 && (
|
||||
<Card className="border-green-200 dark:border-green-800">
|
||||
<CardHeader className="bg-green-50 dark:bg-green-950">
|
||||
<CardTitle className="flex items-center gap-2">
|
||||
<Sparkles className="w-5 h-5" />
|
||||
New Opinions Formed
|
||||
</CardTitle>
|
||||
<CardDescription>New beliefs generated from this interaction</CardDescription>
|
||||
</CardHeader>
|
||||
<CardContent className="pt-6">
|
||||
<div className="space-y-3">
|
||||
{result.new_opinions.map((opinion: any, i: number) => (
|
||||
<div key={i} className="p-3 bg-muted rounded-lg border border-border">
|
||||
<div className="font-semibold text-foreground">{opinion.text}</div>
|
||||
<div className="text-sm text-muted-foreground mt-1">
|
||||
Confidence: {opinion.confidence?.toFixed(2)}
|
||||
</div>
|
||||
</div>
|
||||
))}
|
||||
</div>
|
||||
</CardContent>
|
||||
</Card>
|
||||
)}
|
||||
|
||||
{/* Directive */}
|
||||
<Card className="border-blue-200 dark:border-blue-800">
|
||||
<CardHeader className="py-4">
|
||||
|
|
|
|||
|
|
@ -474,7 +474,6 @@ Observations are consolidated knowledge synthesized from facts.
|
|||
| `HINDSIGHT_API_ENABLE_OBSERVATIONS` | Enable observation consolidation | `true` |
|
||||
| `HINDSIGHT_API_CONSOLIDATION_BATCH_SIZE` | Memories to load per batch (internal optimization) | `50` |
|
||||
| `HINDSIGHT_API_CONSOLIDATION_MAX_TOKENS` | Max tokens for recall when finding related observations during consolidation | `1024` |
|
||||
| `HINDSIGHT_API_RETAIN_OBSERVATIONS_ASYNC` | Run observation generation asynchronously (after retain completes) | `false` |
|
||||
|
||||
### Reflect
|
||||
|
||||
|
|
|
|||
|
|
@ -343,7 +343,7 @@
|
|||
"Memory"
|
||||
],
|
||||
"summary": "Recall memory",
|
||||
"description": "Recall memory using semantic similarity and spreading activation.\n\nThe type parameter is optional and must be one of:\n- `world`: General knowledge about people, places, events, and things that happen\n- `experience`: Memories about experience, conversations, actions taken, and tasks performed\n- `opinion`: The bank's formed beliefs, perspectives, and viewpoints\n\nSet `include_entities=true` to get entity observations alongside recall results.",
|
||||
"description": "Recall memory using semantic similarity and spreading activation.\n\nThe type parameter is optional and must be one of:\n- `world`: General knowledge about people, places, events, and things that happen\n- `experience`: Memories about experience, conversations, actions taken, and tasks performed",
|
||||
"operationId": "recall_memories",
|
||||
"parameters": [
|
||||
{
|
||||
|
|
@ -412,7 +412,7 @@
|
|||
"Memory"
|
||||
],
|
||||
"summary": "Reflect and generate answer",
|
||||
"description": "Reflect and formulate an answer using bank identity, world facts, and opinions.\n\nThis endpoint:\n1. Retrieves experience (conversations and events)\n2. Retrieves world facts relevant to the query\n3. Retrieves existing opinions (bank's perspectives)\n4. Uses LLM to formulate a contextual answer\n5. Extracts and stores any new opinions formed\n6. Returns plain text answer, the facts used, and new opinions",
|
||||
"description": "Reflect and formulate an answer using bank identity, world facts, and opinions.\n\nThis endpoint:\n1. Retrieves experience (conversations and events)\n2. Retrieves world facts relevant to the query\n3. Retrieves existing opinions (bank's perspectives)\n4. Uses LLM to formulate a contextual answer\n5. Returns plain text answer and the facts used",
|
||||
"operationId": "reflect",
|
||||
"parameters": [
|
||||
{
|
||||
|
|
@ -4948,7 +4948,7 @@
|
|||
}
|
||||
],
|
||||
"title": "Types",
|
||||
"description": "List of fact types to recall: 'world', 'experience', 'observation'. Defaults to world and experience if not specified. Note: 'opinion' is accepted but ignored (opinions are excluded from recall)."
|
||||
"description": "List of fact types to recall: 'world', 'experience', 'observation'. Defaults to world and experience if not specified."
|
||||
},
|
||||
"budget": {
|
||||
"$ref": "#/components/schemas/Budget",
|
||||
|
|
|
|||
Loading…
Reference in a new issue