111 lines
4.3 KiB
Python
111 lines
4.3 KiB
Python
"""
|
|
Test deduplication of identical puts.
|
|
"""
|
|
import pytest
|
|
from datetime import datetime, timezone
|
|
from memory.temporal_semantic_memory import TemporalSemanticMemory
|
|
|
|
|
|
@pytest.fixture
|
|
def memory():
|
|
"""Create a memory instance for testing."""
|
|
mem = TemporalSemanticMemory()
|
|
yield mem
|
|
# Cleanup after test
|
|
cursor = mem.conn.cursor()
|
|
cursor.execute("DELETE FROM memory_units WHERE agent_id LIKE 'test_%'")
|
|
mem.conn.commit()
|
|
cursor.close()
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_duplicate_put_filters_identical_content(memory):
|
|
"""Test that putting the same content twice doesn't create duplicates."""
|
|
|
|
agent_id = "test_dedup_agent"
|
|
content = "Alice works at Google as a software engineer. She joined last year and loves Python."
|
|
event_date = datetime(2024, 1, 15, 10, 0, 0, tzinfo=timezone.utc)
|
|
|
|
# First put - should create units
|
|
print("\n--- FIRST PUT ---")
|
|
units_1 = await memory.put_async(agent_id, content, "Test context", event_date)
|
|
|
|
assert len(units_1) > 0, "First put should create units"
|
|
print(f"First put created {len(units_1)} units")
|
|
|
|
# Second put with identical content and same date - should be filtered as duplicates
|
|
print("\n--- SECOND PUT (identical) ---")
|
|
units_2 = await memory.put_async(agent_id, content, "Test context", event_date)
|
|
|
|
assert len(units_2) == 0, "Second identical put should create no new units (all duplicates)"
|
|
print(f"Second put created {len(units_2)} units (expected 0)")
|
|
|
|
# Verify database has only the first set of units
|
|
cursor = memory.conn.cursor()
|
|
cursor.execute(
|
|
"SELECT COUNT(*) FROM memory_units WHERE agent_id = %s",
|
|
(agent_id,)
|
|
)
|
|
total_units = cursor.fetchone()[0]
|
|
cursor.close()
|
|
|
|
assert total_units == len(units_1), f"Database should have {len(units_1)} units, found {total_units}"
|
|
print(f"✅ Deduplication working: {total_units} total units in database")
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_duplicate_put_with_paraphrased_content(memory):
|
|
"""Test that similar but paraphrased content is also deduplicated."""
|
|
|
|
agent_id = "test_paraphrase_agent"
|
|
event_date = datetime(2024, 1, 15, 10, 0, 0, tzinfo=timezone.utc)
|
|
|
|
# First put
|
|
content_1 = "Bob is a chef in New York. He owns a restaurant."
|
|
print("\n--- FIRST PUT ---")
|
|
units_1 = await memory.put_async(agent_id, content_1, "Test", event_date)
|
|
|
|
assert len(units_1) > 0, "First put should create units"
|
|
print(f"First put created {len(units_1)} units")
|
|
|
|
# Second put with paraphrased content - should be mostly deduplicated
|
|
# The LLM will extract similar facts that should match via embeddings
|
|
content_2 = "Bob works as a chef in New York City. He is the owner of a restaurant."
|
|
print("\n--- SECOND PUT (paraphrased) ---")
|
|
units_2 = await memory.put_async(agent_id, content_2, "Test", event_date)
|
|
|
|
# May create 0 or very few new units (depending on how LLM extracts facts)
|
|
print(f"Second put created {len(units_2)} units")
|
|
print(f"Deduplication ratio: {len(units_2)}/{len(units_1)} new units from paraphrase")
|
|
|
|
# Just verify it doesn't create the same number of units (some deduplication should happen)
|
|
assert len(units_2) < len(units_1), "Paraphrased content should have fewer new units due to deduplication"
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_different_dates_not_deduplicated(memory):
|
|
"""Test that same content with different dates is NOT deduplicated."""
|
|
|
|
agent_id = "test_dates_agent"
|
|
content = "Charlie went hiking in Yosemite."
|
|
|
|
# First put at date 1
|
|
date_1 = datetime(2024, 1, 1, 10, 0, 0, tzinfo=timezone.utc)
|
|
print("\n--- FIRST PUT (Jan 1) ---")
|
|
units_1 = await memory.put_async(agent_id, content, "Test", date_1)
|
|
|
|
assert len(units_1) > 0
|
|
print(f"First put created {len(units_1)} units")
|
|
|
|
# Second put at date 2 (outside 24-hour window)
|
|
date_2 = datetime(2024, 2, 1, 10, 0, 0, tzinfo=timezone.utc)
|
|
print("\n--- SECOND PUT (Feb 1, outside time window) ---")
|
|
units_2 = await memory.put_async(agent_id, content, "Test", date_2)
|
|
|
|
# Should create new units because dates are far apart
|
|
assert len(units_2) > 0, "Same content with different dates (outside window) should create new units"
|
|
print(f"Second put created {len(units_2)} units (not deduplicated due to date difference)")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
pytest.main([__file__, "-v", "-s"])
|