fix(engine): classify first-person agent experiences as 'experience' fact type (#775)
* fix(engine): classify first-person agent experiences as 'experience' fact type The extraction prompt defined "assistant" too narrowly as only "interactions with assistant (requests, recommendations)", causing the LLM to classify first-person agent actions (code changes, debugging, discoveries) as "world". Broadened the fact_type definition in the prompt and Pydantic model descriptions to cover all first-person actions, experiences, and observations by the speaker. * style: fix line length in fact_extraction.py
This commit is contained in:
Nicolò Boschi
GitHub
parent
b104bad02c
commit
0096115678
2 changed files with 140 additions and 5 deletions
|
|
@ -159,7 +159,9 @@ class ExtractedFact(BaseModel):
|
|||
fact_kind: str = Field(default="conversation", description="'event' or 'conversation'")
|
||||
occurred_start: str | None = Field(default=None, description="ISO timestamp for events")
|
||||
occurred_end: str | None = Field(default=None, description="ISO timestamp for event end")
|
||||
fact_type: Literal["world", "assistant"] = Field(description="'world' or 'assistant'")
|
||||
fact_type: Literal["world", "assistant"] = Field(
|
||||
description="'world' = objective/external facts. 'assistant' = first-person actions, experiences, or observations by the speaker."
|
||||
)
|
||||
entities: list[Entity] | None = Field(default=None, description="People, places, concepts")
|
||||
causal_relations: list[FactCausalRelation] | None = Field(
|
||||
default=None, description="Links to previous facts (target_index < this fact's index)"
|
||||
|
|
@ -261,7 +263,7 @@ class ExtractedFactVerbose(BaseModel):
|
|||
)
|
||||
|
||||
fact_type: Literal["world", "assistant"] = Field(
|
||||
description="'world' = about the user/others (background, experiences). 'assistant' = experience with the assistant."
|
||||
description="'world' = objective/external facts about other people, events, general knowledge. 'assistant' = first-person actions, experiences, or observations by the speaker (e.g., 'I changed X', 'I discovered Y')."
|
||||
)
|
||||
|
||||
entities: list[Entity] | None = Field(
|
||||
|
|
@ -352,7 +354,9 @@ class VerbatimExtractedFact(BaseModel):
|
|||
fact_kind: str = Field(default="conversation", description="'event' or 'conversation'")
|
||||
occurred_start: str | None = Field(default=None, description="ISO timestamp for events")
|
||||
occurred_end: str | None = Field(default=None, description="ISO timestamp for event end")
|
||||
fact_type: Literal["world", "assistant"] = Field(description="'world' or 'assistant'")
|
||||
fact_type: Literal["world", "assistant"] = Field(
|
||||
description="'world' = objective/external facts. 'assistant' = first-person actions, experiences, or observations by the speaker."
|
||||
)
|
||||
entities: list[Entity] | None = Field(default=None, description="People, places, concepts")
|
||||
|
||||
@field_validator("entities", mode="before")
|
||||
|
|
@ -499,8 +503,8 @@ fact_kind:
|
|||
- "conversation": Ongoing state, preference, trait (no dates)
|
||||
|
||||
fact_type:
|
||||
- "world": About user's life, other people, external events
|
||||
- "assistant": Interactions with assistant (requests, recommendations)
|
||||
- "world": About other people, external events, general knowledge, objective facts
|
||||
- "assistant": First-person actions, experiences, or observations by the speaker/author (e.g., "I changed X", "I discovered Y", "I debugged Z"). Also includes interactions with the user (requests, recommendations). If the narrator describes something they did, tried, learned, or decided — use "assistant".
|
||||
|
||||
══════════════════════════════════════════════════════════════════════════
|
||||
TEMPORAL HANDLING
|
||||
|
|
|
|||
|
|
@ -0,0 +1,131 @@
|
|||
"""
|
||||
Test that first-person agent experiences are classified as 'experience' fact_type,
|
||||
not 'world'. This is critical for AI agent systems that store their own operational
|
||||
experiences (debugging, code changes, user interactions) separately from world knowledge.
|
||||
"""
|
||||
from datetime import datetime
|
||||
|
||||
import pytest
|
||||
|
||||
from hindsight_api import LLMConfig
|
||||
from hindsight_api.config import _get_raw_config
|
||||
from hindsight_api.engine.retain.fact_extraction import extract_facts_from_text
|
||||
|
||||
|
||||
class TestAgentExperienceClassification:
|
||||
"""Tests that first-person coding agent experiences get classified as 'experience'."""
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_code_changes_classified_as_experience(self):
|
||||
"""First-person code change descriptions should be experience, not world."""
|
||||
text = """
|
||||
I changed the return type of the `process_request` function from `dict` to `ResponseModel`.
|
||||
After that, I updated the three callers in `api/handlers.py` to destructure the new model fields.
|
||||
The type checker was happy after the change but I noticed one test was still using the old dict keys.
|
||||
"""
|
||||
llm_config = LLMConfig.for_memory()
|
||||
facts, _, _ = await extract_facts_from_text(
|
||||
text=text,
|
||||
event_date=datetime(2025, 3, 28),
|
||||
llm_config=llm_config,
|
||||
agent_name="coding-agent",
|
||||
context="agent work log",
|
||||
config=_get_raw_config(),
|
||||
)
|
||||
|
||||
assert len(facts) > 0, "Should extract at least one fact"
|
||||
world_facts = [f for f in facts if f.fact_type == "world"]
|
||||
experience_facts = [f for f in facts if f.fact_type == "experience"]
|
||||
assert len(experience_facts) > len(world_facts), (
|
||||
f"First-person code changes should be mostly 'experience', "
|
||||
f"got {len(experience_facts)} experience vs {len(world_facts)} world. "
|
||||
f"Facts: {[(f.fact, f.fact_type) for f in facts]}"
|
||||
)
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_debugging_session_classified_as_experience(self):
|
||||
"""First-person debugging narrative should be experience, not world."""
|
||||
text = """
|
||||
The tests were failing with a ConnectionRefusedError on the Redis integration suite.
|
||||
I traced it to the connection pool not being initialized before the first test ran.
|
||||
I added a setup fixture that ensures the pool is warmed up, and all 47 tests pass now.
|
||||
"""
|
||||
llm_config = LLMConfig.for_memory()
|
||||
facts, _, _ = await extract_facts_from_text(
|
||||
text=text,
|
||||
event_date=datetime(2025, 3, 28),
|
||||
llm_config=llm_config,
|
||||
agent_name="coding-agent",
|
||||
context="agent work log",
|
||||
config=_get_raw_config(),
|
||||
)
|
||||
|
||||
assert len(facts) > 0, "Should extract at least one fact"
|
||||
world_facts = [f for f in facts if f.fact_type == "world"]
|
||||
experience_facts = [f for f in facts if f.fact_type == "experience"]
|
||||
assert len(experience_facts) > len(world_facts), (
|
||||
f"First-person debugging should be mostly 'experience', "
|
||||
f"got {len(experience_facts)} experience vs {len(world_facts)} world. "
|
||||
f"Facts: {[(f.fact, f.fact_type) for f in facts]}"
|
||||
)
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_user_interaction_classified_as_experience(self):
|
||||
"""Agent describing interactions with the user should be experience."""
|
||||
text = """
|
||||
The user asked me to refactor the authentication middleware to support JWT tokens.
|
||||
I proposed splitting it into two modules: token_validation.py and session_management.py.
|
||||
The user approved my approach and I started with the token validation logic.
|
||||
I discovered that the existing tests were mocking the wrong interface, so I had to rewrite them first.
|
||||
"""
|
||||
llm_config = LLMConfig.for_memory()
|
||||
facts, _, _ = await extract_facts_from_text(
|
||||
text=text,
|
||||
event_date=datetime(2025, 3, 28),
|
||||
llm_config=llm_config,
|
||||
agent_name="coding-agent",
|
||||
context="agent work log",
|
||||
config=_get_raw_config(),
|
||||
)
|
||||
|
||||
assert len(facts) > 0, "Should extract at least one fact"
|
||||
world_facts = [f for f in facts if f.fact_type == "world"]
|
||||
experience_facts = [f for f in facts if f.fact_type == "experience"]
|
||||
assert len(experience_facts) > len(world_facts), (
|
||||
f"Agent-user interactions should be mostly 'experience', "
|
||||
f"got {len(experience_facts)} experience vs {len(world_facts)} world. "
|
||||
f"Facts: {[(f.fact, f.fact_type) for f in facts]}"
|
||||
)
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_mixed_agent_and_world_facts(self):
|
||||
"""Mix of agent experiences and world knowledge should be classified correctly."""
|
||||
text = """
|
||||
Python 3.12 introduced a new type parameter syntax for generic classes.
|
||||
I migrated our codebase from the old TypeVar approach to the new syntax.
|
||||
The migration touched 23 files but was mostly mechanical.
|
||||
PEP 695 defines the new type statement that makes generics more readable.
|
||||
"""
|
||||
llm_config = LLMConfig.for_memory()
|
||||
facts, _, _ = await extract_facts_from_text(
|
||||
text=text,
|
||||
event_date=datetime(2025, 3, 28),
|
||||
llm_config=llm_config,
|
||||
agent_name="coding-agent",
|
||||
context="agent work log",
|
||||
config=_get_raw_config(),
|
||||
)
|
||||
|
||||
assert len(facts) > 0, "Should extract at least one fact"
|
||||
world_facts = [f for f in facts if f.fact_type == "world"]
|
||||
experience_facts = [f for f in facts if f.fact_type == "experience"]
|
||||
# Should have both types - world facts about Python 3.12/PEP 695,
|
||||
# experience facts about the migration work
|
||||
assert len(world_facts) >= 1, (
|
||||
f"Should have at least 1 world fact about Python 3.12/PEP 695. "
|
||||
f"Facts: {[(f.fact, f.fact_type) for f in facts]}"
|
||||
)
|
||||
assert len(experience_facts) >= 1, (
|
||||
f"Should have at least 1 experience fact about the migration. "
|
||||
f"Facts: {[(f.fact, f.fact_type) for f in facts]}"
|
||||
)
|
||||
Loading…
Reference in a new issue