diff --git a/hindsight-api-slim/hindsight_api/engine/retain/fact_extraction.py b/hindsight-api-slim/hindsight_api/engine/retain/fact_extraction.py index 5e3609b4..ac46c04b 100644 --- a/hindsight-api-slim/hindsight_api/engine/retain/fact_extraction.py +++ b/hindsight-api-slim/hindsight_api/engine/retain/fact_extraction.py @@ -159,7 +159,9 @@ class ExtractedFact(BaseModel): fact_kind: str = Field(default="conversation", description="'event' or 'conversation'") occurred_start: str | None = Field(default=None, description="ISO timestamp for events") occurred_end: str | None = Field(default=None, description="ISO timestamp for event end") - fact_type: Literal["world", "assistant"] = Field(description="'world' or 'assistant'") + fact_type: Literal["world", "assistant"] = Field( + description="'world' = objective/external facts. 'assistant' = first-person actions, experiences, or observations by the speaker." + ) entities: list[Entity] | None = Field(default=None, description="People, places, concepts") causal_relations: list[FactCausalRelation] | None = Field( default=None, description="Links to previous facts (target_index < this fact's index)" @@ -261,7 +263,7 @@ class ExtractedFactVerbose(BaseModel): ) fact_type: Literal["world", "assistant"] = Field( - description="'world' = about the user/others (background, experiences). 'assistant' = experience with the assistant." + description="'world' = objective/external facts about other people, events, general knowledge. 'assistant' = first-person actions, experiences, or observations by the speaker (e.g., 'I changed X', 'I discovered Y')." ) entities: list[Entity] | None = Field( @@ -352,7 +354,9 @@ class VerbatimExtractedFact(BaseModel): fact_kind: str = Field(default="conversation", description="'event' or 'conversation'") occurred_start: str | None = Field(default=None, description="ISO timestamp for events") occurred_end: str | None = Field(default=None, description="ISO timestamp for event end") - fact_type: Literal["world", "assistant"] = Field(description="'world' or 'assistant'") + fact_type: Literal["world", "assistant"] = Field( + description="'world' = objective/external facts. 'assistant' = first-person actions, experiences, or observations by the speaker." + ) entities: list[Entity] | None = Field(default=None, description="People, places, concepts") @field_validator("entities", mode="before") @@ -499,8 +503,8 @@ fact_kind: - "conversation": Ongoing state, preference, trait (no dates) fact_type: -- "world": About user's life, other people, external events -- "assistant": Interactions with assistant (requests, recommendations) +- "world": About other people, external events, general knowledge, objective facts +- "assistant": First-person actions, experiences, or observations by the speaker/author (e.g., "I changed X", "I discovered Y", "I debugged Z"). Also includes interactions with the user (requests, recommendations). If the narrator describes something they did, tried, learned, or decided — use "assistant". ══════════════════════════════════════════════════════════════════════════ TEMPORAL HANDLING diff --git a/hindsight-api-slim/tests/test_fact_extraction_agent_experience.py b/hindsight-api-slim/tests/test_fact_extraction_agent_experience.py new file mode 100644 index 00000000..c12f6860 --- /dev/null +++ b/hindsight-api-slim/tests/test_fact_extraction_agent_experience.py @@ -0,0 +1,131 @@ +""" +Test that first-person agent experiences are classified as 'experience' fact_type, +not 'world'. This is critical for AI agent systems that store their own operational +experiences (debugging, code changes, user interactions) separately from world knowledge. +""" +from datetime import datetime + +import pytest + +from hindsight_api import LLMConfig +from hindsight_api.config import _get_raw_config +from hindsight_api.engine.retain.fact_extraction import extract_facts_from_text + + +class TestAgentExperienceClassification: + """Tests that first-person coding agent experiences get classified as 'experience'.""" + + @pytest.mark.asyncio + async def test_code_changes_classified_as_experience(self): + """First-person code change descriptions should be experience, not world.""" + text = """ +I changed the return type of the `process_request` function from `dict` to `ResponseModel`. +After that, I updated the three callers in `api/handlers.py` to destructure the new model fields. +The type checker was happy after the change but I noticed one test was still using the old dict keys. +""" + llm_config = LLMConfig.for_memory() + facts, _, _ = await extract_facts_from_text( + text=text, + event_date=datetime(2025, 3, 28), + llm_config=llm_config, + agent_name="coding-agent", + context="agent work log", + config=_get_raw_config(), + ) + + assert len(facts) > 0, "Should extract at least one fact" + world_facts = [f for f in facts if f.fact_type == "world"] + experience_facts = [f for f in facts if f.fact_type == "experience"] + assert len(experience_facts) > len(world_facts), ( + f"First-person code changes should be mostly 'experience', " + f"got {len(experience_facts)} experience vs {len(world_facts)} world. " + f"Facts: {[(f.fact, f.fact_type) for f in facts]}" + ) + + @pytest.mark.asyncio + async def test_debugging_session_classified_as_experience(self): + """First-person debugging narrative should be experience, not world.""" + text = """ +The tests were failing with a ConnectionRefusedError on the Redis integration suite. +I traced it to the connection pool not being initialized before the first test ran. +I added a setup fixture that ensures the pool is warmed up, and all 47 tests pass now. +""" + llm_config = LLMConfig.for_memory() + facts, _, _ = await extract_facts_from_text( + text=text, + event_date=datetime(2025, 3, 28), + llm_config=llm_config, + agent_name="coding-agent", + context="agent work log", + config=_get_raw_config(), + ) + + assert len(facts) > 0, "Should extract at least one fact" + world_facts = [f for f in facts if f.fact_type == "world"] + experience_facts = [f for f in facts if f.fact_type == "experience"] + assert len(experience_facts) > len(world_facts), ( + f"First-person debugging should be mostly 'experience', " + f"got {len(experience_facts)} experience vs {len(world_facts)} world. " + f"Facts: {[(f.fact, f.fact_type) for f in facts]}" + ) + + @pytest.mark.asyncio + async def test_user_interaction_classified_as_experience(self): + """Agent describing interactions with the user should be experience.""" + text = """ +The user asked me to refactor the authentication middleware to support JWT tokens. +I proposed splitting it into two modules: token_validation.py and session_management.py. +The user approved my approach and I started with the token validation logic. +I discovered that the existing tests were mocking the wrong interface, so I had to rewrite them first. +""" + llm_config = LLMConfig.for_memory() + facts, _, _ = await extract_facts_from_text( + text=text, + event_date=datetime(2025, 3, 28), + llm_config=llm_config, + agent_name="coding-agent", + context="agent work log", + config=_get_raw_config(), + ) + + assert len(facts) > 0, "Should extract at least one fact" + world_facts = [f for f in facts if f.fact_type == "world"] + experience_facts = [f for f in facts if f.fact_type == "experience"] + assert len(experience_facts) > len(world_facts), ( + f"Agent-user interactions should be mostly 'experience', " + f"got {len(experience_facts)} experience vs {len(world_facts)} world. " + f"Facts: {[(f.fact, f.fact_type) for f in facts]}" + ) + + @pytest.mark.asyncio + async def test_mixed_agent_and_world_facts(self): + """Mix of agent experiences and world knowledge should be classified correctly.""" + text = """ +Python 3.12 introduced a new type parameter syntax for generic classes. +I migrated our codebase from the old TypeVar approach to the new syntax. +The migration touched 23 files but was mostly mechanical. +PEP 695 defines the new type statement that makes generics more readable. +""" + llm_config = LLMConfig.for_memory() + facts, _, _ = await extract_facts_from_text( + text=text, + event_date=datetime(2025, 3, 28), + llm_config=llm_config, + agent_name="coding-agent", + context="agent work log", + config=_get_raw_config(), + ) + + assert len(facts) > 0, "Should extract at least one fact" + world_facts = [f for f in facts if f.fact_type == "world"] + experience_facts = [f for f in facts if f.fact_type == "experience"] + # Should have both types - world facts about Python 3.12/PEP 695, + # experience facts about the migration work + assert len(world_facts) >= 1, ( + f"Should have at least 1 world fact about Python 3.12/PEP 695. " + f"Facts: {[(f.fact, f.fact_type) for f in facts]}" + ) + assert len(experience_facts) >= 1, ( + f"Should have at least 1 experience fact about the migration. " + f"Facts: {[(f.fact, f.fact_type) for f in facts]}" + )