diff --git a/.gitignore b/.gitignore index 44c48644..48cee815 100644 --- a/.gitignore +++ b/.gitignore @@ -21,3 +21,9 @@ wheels/ # NLTK data (will be downloaded automatically) nltk_data/ + +# Large benchmark datasets (will be downloaded automatically) +benchmarks/longmemeval/longmemeval_s_cleaned.json + +# Debug logs +logs/ diff --git a/CLAUDE.md b/CLAUDE.md index 2651728e..a1612c1f 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -2,6 +2,7 @@ Do not write any markdown file, just write the code. # Workflow -After your changes, make sure everything is working fine by running the main script. +- After your changes, make sure everything is working fine by running the tests. - keep the readme.md architecture section up to date when you change the implementation -- when changing an implemetation, do not keep the old one as fallback \ No newline at end of file +- when changing an implemetation, do not keep the old one as fallback +- to run test, use uv run pytest tests \ No newline at end of file diff --git a/README.md b/README.md index 607a014c..f4e3a84d 100644 --- a/README.md +++ b/README.md @@ -90,43 +90,44 @@ The combination of these three networks enables powerful memory retrieval that g The search algorithm explores the memory graph using spreading activation: -1. **Entry Points**: Find top-3 semantically similar memories to the query (vector search) -2. **Activation Spreading**: Start with activation = 1.0 at entry points +1. **Entry Points**: Find top-3 semantically similar memories to the query (vector search, similarity ≥ 0.5) +2. **Activation Spreading**: Start with activation = actual similarity score (0.5 to 1.0) at entry points 3. **Graph Traversal**: Follow links to neighbors, spreading activation with decay (0.8 factor) 4. **Thinking Budget**: Limit exploration to N units (controls computational cost) -5. **Dynamic Weighting**: Combine activation with recency and frequency: +5. **Dynamic Weighting**: Combine activation, semantic similarity, recency, and frequency: ``` - final_weight = activation × recency × frequency + final_weight = 0.30 × activation + 0.30 × semantic_similarity + 0.25 × recency + 0.15 × frequency + semantic_similarity = cosine_similarity(query_embedding, memory_embedding) recency = exp(-0.1 × days_since) - frequency = 1.0 + log(access_count + 1) / log(10) + frequency = normalized to [0, 1] from log(access_count + 1) / log(10) ``` 6. **Return Top-K**: Sort by final weight and return top results This approach ensures: -- Recently accessed memories get boosted (recency bias) -- Frequently accessed memories get boosted (importance signal) -- Graph structure influences results (not just vector similarity) +- Semantic relevance to query is always considered (30% weight) +- Graph structure influences results through activation (30% weight) +- Recently accessed memories get boosted (25% weight - recency bias) +- Frequently accessed memories get boosted (15% weight - importance signal) ### Self-Contained Memory Units -Every memory unit is processed to be self-contained through coreference resolution: +Every memory unit is self-contained through LLM fact extraction: **Problem**: "She joined Google last year" - unclear who "she" is -**Solution**: Fast batch coreference resolution that: -- Replaces personal pronouns (he, she, it, they) with actual referents -- Processes all sentences in one batch (O(n) instead of O(n²)) -- Uses neural coreference model for high accuracy -- Provides fallback to custom spaCy-based resolution if needed +**Solution**: LLM-based fact extraction that: +- Resolves pronouns to actual referents during extraction +- Makes facts readable without original context +- Includes all relevant details (WHO, WHAT, WHERE, WHEN, WHY, HOW) +- Processes facts in parallel for speed **Result**: "Alice joined Google last year" - fully self-contained **Technology**: -- **FastCoref** - Fast, accurate neural coreference resolution -- Processes 2.8K documents in 25 seconds on GPU -- Significant speedup over sequential spaCy approach -- Fallback to custom spaCy implementation if needed +- LLM fact extraction with detailed prompts for pronoun resolution +- Structured output using Pydantic models +- Batch processing for efficiency ### LLM-Based Fact Extraction @@ -165,9 +166,8 @@ Raw content is processed through an LLM to extract meaningful facts before stora - `psycopg2-binary` - PostgreSQL client - `sentence-transformers` - Local embedding model (bge-small-en-v1.5) - `torch` - Deep learning framework (for embeddings) -- `fastcoref` - Fast neural coreference resolution - `spacy` - NLP (NER, dependency parsing, tokenization) -- `nltk` - Sentence tokenization +- `langchain-text-splitters` - Intelligent text chunking - `networkx` - Graph operations - `pyvis` - Interactive HTML graph visualization - `matplotlib` - Static graph visualization @@ -247,7 +247,7 @@ memory-poc/ ├── memory/ # Core memory system package │ ├── temporal_semantic_memory.py # Main memory system class │ ├── entity_resolver.py # Entity extraction and disambiguation -│ ├── coref_resolver.py # Coreference resolution +│ ├── llm_client.py # LLM-based fact extraction │ └── utils.py # Utility functions │ ├── demos/ # Demo scripts diff --git a/benchmarks/locomo/benchmark_results.json b/benchmarks/locomo/benchmark_results.json index ff79651b..408b16f8 100644 --- a/benchmarks/locomo/benchmark_results.json +++ b/benchmarks/locomo/benchmark_results.json @@ -1,18 +1,22 @@ { - "overall_accuracy": 33.33333333333333, - "total_correct": 1, - "total_questions": 3, + "overall_accuracy": 60.0, + "total_correct": 6, + "total_questions": 10, "conversation_results": [ { "sample_id": "conv-26", "metrics": { - "accuracy": 33.33333333333333, - "correct": 1, - "total": 3, + "accuracy": 60.0, + "correct": 6, + "total": 10, "category_stats": { + "1": { + "correct": 2, + "total": 3 + }, "2": { - "correct": 0, - "total": 2 + "correct": 3, + "total": 6 }, "3": { "correct": 1, @@ -20,30 +24,2319 @@ } }, "detailed_results": [ + { + "question": "What is Caroline's relationship status?", + "correct_answer": "Single", + "predicted_answer": "I don't know", + "reasoning": "The context does not provide any information about Caroline's current relationship status. It mentions a tough breakup in the past, but there is no indication of whether she is currently in a relationship or single.", + "category": 1, + "retrieved_memories": [ + { + "id": "0b5694e4-7c0b-422c-bed4-588d92fb067f", + "text": "Caroline has known her current friends for four years, having moved from her home country around 2019-06-09.", + "context": "Conversation session between Caroline and Melanie", + "event_date": "2023-06-09T19:55:00+00:00", + "weight": 0.5928971016147087, + "activation": 0.738161846289745, + "semantic_similarity": 0.7381618257592838, + "recency": 1.1144095709452762e-38, + "frequency": 2.0 + }, + { + "id": "43224c50-a69d-4d79-bda5-98253119a520", + "text": "Caroline expressed appreciation for her friendship with Melanie.", + "context": "Conversation session between Caroline and Melanie", + "event_date": "2023-08-17T13:50:00+00:00", + "weight": 0.5855437934744278, + "activation": 0.725906290831977, + "semantic_similarity": 0.7259063540827823, + "recency": 1.0781237162542037e-35, + "frequency": 2.0 + }, + { + "id": "1e910def-a7cb-4593-be18-99c833176fa3", + "text": "Caroline's gender transition and artistic expression have altered her relationships: some close friends continue to support her, while a few could not handle the changes; overall she feels happier and her relationships now feel more genuine.", + "context": "Conversation session between Caroline and Melanie", + "event_date": "2023-09-13T00:09:00+00:00", + "weight": 0.581743502522861, + "activation": 0.719572463048328, + "semantic_similarity": 0.7195725453612087, + "recency": 1.5153148641642148e-34, + "frequency": 2.0 + }, + { + "id": "7f5db464-eed6-4af6-b3bc-820f94193202", + "text": "Caroline feels lucky to have Melanie as a friend who reminds her of happy moments and supports her during life's struggles.", + "context": "Conversation session between Caroline and Melanie", + "event_date": "2023-08-17T13:50:00+00:00", + "weight": 0.5294705960529247, + "activation": 0.5807250326655816, + "semantic_similarity": 0.6841769541775008, + "recency": 1.078119636786517e-35, + "frequency": 2.0 + }, + { + "id": "de8d33ff-fe15-4f25-80f8-c534bc9ceb05", + "text": "Melanie and Caroline consider that they can always be there for each other, indicating a mutual supportive relationship.", + "context": "Conversation session between Caroline and Melanie", + "event_date": "2023-10-22T09:55:00+00:00", + "weight": 0.5256005666944299, + "activation": 0.5807250326655816, + "semantic_similarity": 0.6712768563158512, + "recency": 7.796889739732441e-33, + "frequency": 2.0 + }, + { + "id": "ad619ec1-c081-4fab-bb8f-707d8eed6fc0", + "text": "Caroline mentioned a tough breakup in the past and expressed thankfulness for her friends, family, and mentors who have supported her.", + "context": "Conversation session between Caroline and Melanie", + "event_date": "2023-06-09T19:55:00+00:00", + "weight": 0.5232549537202421, + "activation": 0.590529477031796, + "semantic_similarity": 0.6536537020356776, + "recency": 1.1144053544427697e-38, + "frequency": 2.0 + }, + { + "id": "aa36595e-58ca-42dc-b50d-8c785526bbf3", + "text": "Melanie's support has been meaningful to Caroline; Caroline feels grateful for Melanie's support throughout her journey and values being able to share and help others.", + "context": "Conversation session between Caroline and Melanie", + "event_date": "2023-10-22T09:55:00+00:00", + "weight": 0.520671892248089, + "activation": 0.5807250326655816, + "semantic_similarity": 0.6548479414947148, + "recency": 7.796889740120575e-33, + "frequency": 2.0 + }, + { + "id": "5e2c78a6-b363-44f7-a74b-23b7b61980f9", + "text": "Caroline has been looking into counseling or mental health work since her last conversation with Melanie, expressing passion for helping people, describing the work as tough but rewarding.", + "context": "Conversation session between Caroline and Melanie", + "event_date": "2023-07-06T20:18:00+00:00", + "weight": 0.5166050086282288, + "activation": 0.5807250326655816, + "semantic_similarity": 0.6412916627618478, + "recency": 1.6608494086599004e-37, + "frequency": 2.0 + }, + { + "id": "98c08619-7bb3-467c-a87e-027ee4b31021", + "text": "Both Caroline and Melanie affirmed their commitment to continue supporting each other, spreading love, acceptance, and hope, and to motivate each other through life's challenges.", + "context": "Conversation session between Caroline and Melanie", + "event_date": "2023-06-09T19:55:00+00:00", + "weight": 0.5154365483453377, + "activation": 0.590529477031796, + "semantic_similarity": 0.6275923507859963, + "recency": 1.1144053548684114e-38, + "frequency": 2.0 + }, + { + "id": "60fa2ddb-d241-4067-910e-ad8b864ac31b", + "text": "Melanie said that Caroline has always been there for her, and she appreciates their friendship.", + "context": "Conversation session between Caroline and Melanie", + "event_date": "2023-08-17T13:50:00+00:00", + "weight": 0.5153598866708379, + "activation": 0.5807250326655816, + "semantic_similarity": 0.6371412562372113, + "recency": 1.0781196367128996e-35, + "frequency": 2.0 + }, + { + "id": "d20a1038-61b6-4baa-9d6a-7d3a729434af", + "text": "Caroline expressed gratitude for the love and support she has received throughout her transition and aims to give a voice to the trans community.", + "context": "Conversation session between Caroline and Melanie", + "event_date": "2023-06-09T19:55:00+00:00", + "weight": 0.5147455739355009, + "activation": 0.590529477031796, + "semantic_similarity": 0.6252891027532071, + "recency": 1.1144053547845722e-38, + "frequency": 2.0 + }, + { + "id": "71e5a874-ce13-49df-a1a2-8f9fa377324c", + "text": "Caroline commented on Melanie's purple shoes, asking whether they are intended for walking or running.", + "context": "Conversation session between Caroline and Melanie", + "event_date": "2023-07-12T16:33:00+00:00", + "weight": 0.5136131344114743, + "activation": 0.5807250326655816, + "semantic_similarity": 0.6313187487059991, + "recency": 2.9793470414229446e-37, + "frequency": 2.0 + }, + { + "id": "73850c77-935b-45c3-9a68-3552e0f85121", + "text": "Caroline emphasizes that mental health is a priority and advises Melanie to take care of herself.", + "context": "Conversation session between Caroline and Melanie", + "event_date": "2023-07-12T16:33:00+00:00", + "weight": 0.5120275116131798, + "activation": 0.5807250326655816, + "semantic_similarity": 0.626033339378351, + "recency": 2.9793470416780796e-37, + "frequency": 2.0 + }, + { + "id": "50dd73e8-e477-4c45-b052-b3eae96a1a6a", + "text": "Melanie expressed pride in Caroline's work and said she is proud to be part of the difference Caroline is making.", + "context": "Conversation session between Caroline and Melanie", + "event_date": "2023-06-09T19:55:00+00:00", + "weight": 0.5110527232495037, + "activation": 0.590529477031796, + "semantic_similarity": 0.6129796004665498, + "recency": 1.1144053547510302e-38, + "frequency": 2.0 + }, + { + "id": "006df526-f850-48a9-a2f3-a13799f0391c", + "text": "Melanie states that the conversation with Caroline has been great for her mental health and she intends to continue the positive practices.", + "context": "Conversation session between Caroline and Melanie", + "event_date": "2023-07-12T16:33:00+00:00", + "weight": 0.5098334678141485, + "activation": 0.5807250326655816, + "semantic_similarity": 0.6187198600482468, + "recency": 2.9793470418677586e-37, + "frequency": 2.0 + }, + { + "id": "d4afd073-6eda-41ec-b138-8fc46c23bc23", + "text": "Caroline has not yet tried pottery but is interested in trying new art forms and may try pottery sometime in the future.", + "context": "Conversation session between Caroline and Melanie", + "event_date": "2023-09-13T00:09:00+00:00", + "weight": 0.5093245191024905, + "activation": 0.5756579704386624, + "semantic_similarity": 0.6220904265696393, + "recency": 1.5153019395586813e-34, + "frequency": 2.0 + }, + { + "id": "c7ffecc3-a614-401d-9f62-a23753c7ca5f", + "text": "Melanie has been reading a book that Caroline recommended a while ago and has been painting to keep busy.", + "context": "Conversation session between Caroline and Melanie", + "event_date": "2023-10-13T10:31:00+00:00", + "weight": 0.5082040688672157, + "activation": 0.5807250326655816, + "semantic_similarity": 0.6132885302251372, + "recency": 3.177913671815118e-33, + "frequency": 2.0 + }, + { + "id": "ff8318fb-cbb5-4f95-a985-da11cfb910df", + "text": "Caroline painted a piece after visiting an LGBTQ center, intending to capture the community's unity and strength; she shared this painting with Melanie on July 17, 2023.", + "context": "Conversation session between Caroline and Melanie", + "event_date": "2023-07-17T14:31:00+00:00", + "weight": 0.5053548330296247, + "activation": 0.5807250326655816, + "semantic_similarity": 0.6037910774331675, + "recency": 4.870691184975256e-37, + "frequency": 2.0 + }, + { + "id": "f62cd380-3b68-4116-b0b3-787ce4c96eac", + "text": "Caroline advised Melanie to research and find an adoption agency or lawyer, gather necessary documents such as references, financial information, and medical checks, and to prepare emotionally because the waiting period can be hard.", + "context": "Conversation session between Caroline and Melanie", + "event_date": "2023-10-13T10:31:00+00:00", + "weight": 0.5012917682997061, + "activation": 0.5807250326655816, + "semantic_similarity": 0.5902475283334387, + "recency": 3.177913672068922e-33, + "frequency": 2.0 + }, + { + "id": "a8b8baac-87de-4f41-b0c5-b32f2870f1d6", + "text": "Caroline is inspired by her work that makes a difference for the LGBTQ+ community, believing that helping create a more loving world is amazing, and this inspiration motivates her to keep making art.", + "context": "Conversation session between Caroline and Melanie", + "event_date": "2023-09-13T00:09:00+00:00", + "weight": 0.501231271249604, + "activation": 0.5756579704386624, + "semantic_similarity": 0.5951129337266843, + "recency": 1.5153019392219583e-34, + "frequency": 2.0 + } + ], + "is_correct": false, + "correctness_reasoning": "The predicted answer 'I don't know' does not state that Caroline is single, so it does not match the correct answer." + }, + { + "question": "When is Melanie planning on going camping?", + "correct_answer": "June 2023", + "predicted_answer": "I don't know", + "reasoning": "The context does not provide specific information about any future camping plans for Melanie. It mentions past camping trips and her family's enjoyment of camping, but there is no indication of when they are planning to go camping next.", + "category": 2, + "retrieved_memories": [ + { + "id": "d6fa6c91-b0b0-48c3-8dc6-a2981abaae82", + "text": "On 2023-07-15, Melanie and her family went on a camping trip in the forest.", + "context": "Conversation session between Caroline and Melanie", + "event_date": "2023-07-15T13:51:00+00:00", + "weight": 0.6385532224646773, + "activation": 0.81425535891285, + "semantic_similarity": 0.8142553826360744, + "recency": 3.976579229693862e-37, + "frequency": 2.0 + }, + { + "id": "1bcbab0e-f708-46f3-9191-4b591598a2df", + "text": "Melanie went camping with her family on the weekend of July 1\u20132, 2023, which was two weekends before the conversation date of July 17, 2023.", + "context": "Conversation session between Caroline and Melanie", + "event_date": "2023-07-01T14:31:00+00:00", + "weight": 0.6341995563735137, + "activation": 0.806999302744782, + "semantic_similarity": 0.8069992185002633, + "recency": 9.833400746436395e-38, + "frequency": 2.0 + }, + { + "id": "757040fe-bd93-4ed8-bbe2-731b6b33fb56", + "text": "Melanie went camping with her children a few weeks before September 13, 2023 (approximately on August 23, 2023), during which they explored the forest, hiked, and had a blast.", + "context": "Conversation session between Caroline and Melanie", + "event_date": "2023-08-23T00:09:00+00:00", + "weight": 0.610266205211061, + "activation": 0.767110319985189, + "semantic_similarity": 0.767110364051681, + "recency": 1.8555264197290307e-35, + "frequency": 2.0 + }, + { + "id": "8a78111e-a82b-478f-93d5-5256f4727402", + "text": "Melanie took her family camping in the mountains during the week prior to June 27, 2023 (last week), providing a nice time together.", + "context": "Conversation session between Caroline and Melanie", + "event_date": "2023-06-20T10:37:00+00:00", + "weight": 0.5694267831878538, + "activation": 0.65140428713028, + "semantic_similarity": 0.746684990162566, + "recency": 3.2204808252609967e-38, + "frequency": 2.0 + }, + { + "id": "fa88ef2c-bf7a-4349-9898-21d90bd4cda0", + "text": "Melanie's family looks forward to an annual summer camping trip, which they consider the highlight of their summer, as of July 20, 2023.", + "context": "Conversation session between Caroline and Melanie", + "event_date": "2023-07-20T20:56:00+00:00", + "weight": 0.5663847277711936, + "activation": 0.65140428713028, + "semantic_similarity": 0.7365448054403654, + "recency": 6.752627520291924e-37, + "frequency": 2.0 + }, + { + "id": "33bfbe46-a1b7-4119-8110-33f78643abe5", + "text": "During a family camping trip in August 2022, Melanie's family observed the Perseid meteor shower, made wishes, and felt awe while lying under a clear, star\u2011filled sky.", + "context": "Conversation session between Caroline and Melanie", + "event_date": "2022-08-12T20:56:00+00:00", + "weight": 0.5624554345902368, + "activation": 0.65140428713028, + "semantic_similarity": 0.7234471615038428, + "recency": 9.475486416300536e-52, + "frequency": 2.0 + }, + { + "id": "60116c77-ad6d-4038-a91e-ab7621a2f94f", + "text": "Melanie suggested doing a family outing this summer, proposing a group activity during the 2023 summer season.", + "context": "Conversation session between Caroline and Melanie", + "event_date": "2023-06-21T13:50:00+00:00", + "weight": 0.5620544578312987, + "activation": 0.65140428713028, + "semantic_similarity": 0.7221105723073825, + "recency": 3.607205789759941e-38, + "frequency": 2.0 + }, + { + "id": "2c65ccf4-14ae-4519-9a27-7af35209c634", + "text": "Melanie had a quiet weekend on July 8\u20139, 2023, the weekend immediately after the family camping trip, to relax and unplug.", + "context": "Conversation session between Caroline and Melanie", + "event_date": "2023-07-08T14:31:00+00:00", + "weight": 0.5614518383638576, + "activation": 0.65140428713028, + "semantic_similarity": 0.7201018407492454, + "recency": 1.980188230602638e-37, + "frequency": 2.0 + }, + { + "id": "dbc74f58-282c-441c-9b8a-9b729c6b0f15", + "text": "Melanie enjoys camping trips with her family because nature provides peace, serenity, and opportunities to bond over stories, campfires, birdsong, and fresh air, which refreshes her soul and helps her reset and recharge.", + "context": "Conversation session between Caroline and Melanie", + "event_date": "2023-10-20T18:55:00+00:00", + "weight": 0.5538515296521284, + "activation": 0.65140428713028, + "semantic_similarity": 0.6947674783768145, + "recency": 6.627240762789156e-33, + "frequency": 2.0 + }, + { + "id": "be729e0b-7a37-4fef-98c1-5da6390b128a", + "text": "Melanie painted a lake sunrise painting last year (2022-05-08), and the artwork is special to her.", + "context": "Conversation session between Caroline and Melanie", + "event_date": "2022-05-08T13:56:00+00:00", + "weight": 0.5442424851154435, + "activation": 0.65140428713028, + "semantic_similarity": 0.6627373299211983, + "recency": 6.233149802321527e-56, + "frequency": 2.0 + }, + { + "id": "1e8079c8-e0bf-4634-a94f-a3cc14de2542", + "text": "Melanie posted a recent picture taken on October 19, 2023, showing her children enjoying the Grand Canyon, which she described as a nice way to relax after the roadtrip.", + "context": "Conversation session between Caroline and Melanie", + "event_date": "2023-10-19T18:55:00+00:00", + "weight": 0.5442232136177595, + "activation": 0.65140428713028, + "semantic_similarity": 0.6626730915955849, + "recency": 5.996575421476561e-33, + "frequency": 2.0 + }, + { + "id": "f9c087ae-d272-4bfb-b111-fc538fd577fd", + "text": "On 2023-07-14, Melanie took her kids to a pottery workshop where they all made their own pots; the activity was fun and therapeutic, and the kids loved it, being excited to get their hands dirty and create something with clay.", + "context": "Conversation session between Caroline and Melanie", + "event_date": "2023-07-14T13:51:00+00:00", + "weight": 0.543724684777126, + "activation": 0.65140428713028, + "semantic_similarity": 0.6610113287934735, + "recency": 3.598129503171291e-37, + "frequency": 2.0 + }, + { + "id": "64621ccc-3569-43ab-bcbc-300752e4cba9", + "text": "On the weekend of 2023-07-08, Melanie and her kids painted a nature-inspired artwork together, which they described as their latest work.", + "context": "Conversation session between Caroline and Melanie", + "event_date": "2023-07-08T13:51:00+00:00", + "weight": 0.5436108257052754, + "activation": 0.65140428713028, + "semantic_similarity": 0.6606317985539714, + "recency": 1.9746953406449577e-37, + "frequency": 2.0 + }, + { + "id": "6cd22599-91e4-4777-8cea-136df2061405", + "text": "Melanie is currently swamped with caring for her kids and managing work responsibilities.", + "context": "Conversation session between Caroline and Melanie", + "event_date": "2023-05-08T13:56:00+00:00", + "weight": 0.541549458385559, + "activation": 0.65140428713028, + "semantic_similarity": 0.6537605741549166, + "recency": 4.430534817106696e-40, + "frequency": 2.0 + }, + { + "id": "e460bf87-7ca8-40cb-bb49-3750d164c90d", + "text": "Melanie spent August 24, 2023, volunteering with her family at a homeless shelter, observing neglected individuals and feeling that they made a positive difference.", + "context": "Conversation session between Caroline and Melanie", + "event_date": "2023-08-24T13:33:00+00:00", + "weight": 0.5405761492521944, + "activation": 0.65140428713028, + "semantic_similarity": 0.6505162103770347, + "recency": 2.1684094851390347e-35, + "frequency": 2.0 + }, + { + "id": "6ce7be9f-ef63-4d9f-9bce-fd223d501ee8", + "text": "Melanie finds peace through creativity and family support.", + "context": "Conversation session between Caroline and Melanie", + "event_date": "2023-07-15T13:51:00+00:00", + "weight": 0.5396226296575926, + "activation": 0.65140428713028, + "semantic_similarity": 0.6473378117283618, + "recency": 3.976548088225122e-37, + "frequency": 2.0 + }, + { + "id": "c1db0587-badf-4875-9b80-a3e2ad5a64ee", + "text": "Melanie shared a photograph of her family camping at the beach, stating that the experience brings her family closer together.", + "context": "Conversation session between Caroline and Melanie", + "event_date": "2023-07-06T20:18:00+00:00", + "weight": 0.5386915029894479, + "activation": 0.65140428713028, + "semantic_similarity": 0.6442340561678795, + "recency": 1.660782919341833e-37, + "frequency": 2.0 + }, + { + "id": "99533d0b-08a4-4bfb-ae6f-82ff14200c83", + "text": "Melanie has been married for five years, meaning she married her husband around 2018-06-09.", + "context": "Conversation session between Caroline and Melanie", + "event_date": "2018-06-09T19:55:00+00:00", + "weight": 0.5374705262130155, + "activation": 0.65140428713028, + "semantic_similarity": 0.6401641335797715, + "recency": 5.5571651502913045e-118, + "frequency": 2.0 + }, + { + "id": "6b0289f7-0dbc-4a51-8628-16d23a5991c1", + "text": "Melanie feels inspired by the autumn season and is planning to create several new paintings in the near future, as expressed on August 25, 2023.", + "context": "Conversation session between Caroline and Melanie", + "event_date": "2023-08-25T13:33:00+00:00", + "weight": 0.5371921049278133, + "activation": 0.65140428713028, + "semantic_similarity": 0.6392360626290977, + "recency": 2.3964631018378227e-35, + "frequency": 2.0 + }, + { + "id": "fb45c7ac-8079-4af6-bcf2-6ae9dd3ea1d4", + "text": "Melanie recently completed a landscape painting, which she shared during the conversation on August 25, 2023.", + "context": "Conversation session between Caroline and Melanie", + "event_date": "2023-08-25T13:33:00+00:00", + "weight": 0.5361437921894194, + "activation": 0.65140428713028, + "semantic_similarity": 0.6357416868344515, + "recency": 2.3964631016491878e-35, + "frequency": 2.0 + } + ], + "is_correct": false, + "correctness_reasoning": "The predicted answer 'I don't know' does not provide the required information 'June 2023' and therefore is not equivalent to the correct answer." + }, + { + "question": "When did Caroline give a speech at a school?", + "correct_answer": "The week before 9 June 2023", + "predicted_answer": "2023-06-02", + "reasoning": "The context explicitly states that Caroline delivered a school event talk about her transgender journey on 2023-06-02.", + "category": 2, + "retrieved_memories": [ + { + "id": "e4502529-fc39-465d-82b7-d56d0e56fde6", + "text": "Caroline delivered a school event talk about her transgender journey and encouraged students to get involved in the LGBTQ community; the event took place last week on 2023-06-02.", + "context": "Conversation session between Caroline and Melanie", + "event_date": "2023-06-02T19:55:00+00:00", + "weight": 0.6057134820306492, + "activation": 0.759522452382897, + "semantic_similarity": 0.759522487719267, + "recency": 5.534217638137463e-39, + "frequency": 2.0 + }, + { + "id": "95f094d7-70ed-4438-8026-da6e3dd6a571", + "text": "During the school event on 2023-06-02, Caroline felt powerful sharing her struggles and personal development since coming out, and observed that the audience related to her story and were inspired to become better allies.", + "context": "Conversation session between Caroline and Melanie", + "event_date": "2023-06-02T19:55:00+00:00", + "weight": 0.5996411386976285, + "activation": 0.749401882326492, + "semantic_similarity": 0.7494019133322697, + "recency": 5.534217636465764e-39, + "frequency": 2.0 + }, + { + "id": "ec88eee3-ba37-427f-9bf4-e0995d5fc61a", + "text": "On Friday, June 23, 2023, Caroline attended an LGBTQ+ counseling workshop where professionals discussed various therapeutic methods for working with transgender people and demonstrated passion for creating safe spaces for individuals like her.", + "context": "Conversation session between Caroline and Melanie", + "event_date": "2023-06-23T10:37:00+00:00", + "weight": 0.5552226440478177, + "activation": 0.6753710299288, + "semantic_similarity": 0.6753711168972591, + "recency": 4.3475615065288293e-38, + "frequency": 2.0 + }, + { + "id": "4dc1b6d9-cb36-4156-91eb-6e633ac75d64", + "text": "Caroline shared a photograph taken when she and Melanie met up last week; the photo was taken on 2023-06-02.", + "context": "Conversation session between Caroline and Melanie", + "event_date": "2023-06-02T19:55:00+00:00", + "weight": 0.5172251325700236, + "activation": 0.6076179619063176, + "semantic_similarity": 0.6164658133270943, + "recency": 5.534211300014424e-39, + "frequency": 2.0 + }, + { + "id": "c5161f91-693c-48a1-a288-ad149f8fc765", + "text": "Caroline attended an LGBTQ conference on 2023-07-10, two days before the reference date, where she met and connected with people who have experienced similar journeys, found the environment welcoming, felt totally accepted, and expressed gratitude for the community.", + "context": "Conversation session between Caroline and Melanie", + "event_date": "2023-07-10T16:33:00+00:00", + "weight": 0.5129628609491097, + "activation": 0.54029682394304, + "semantic_similarity": 0.6695793792206591, + "recency": 2.4393975389592374e-37, + "frequency": 2.0 + }, + { + "id": "22612de6-dda7-4df8-af30-5d488b1bdc10", + "text": "On 2023-07-14, Caroline attended a council meeting for adoption; she found it inspiring and emotional, observed many people wanting to create loving homes for children in need, and felt even more determined to adopt.", + "context": "Conversation session between Caroline and Melanie", + "event_date": "2023-07-14T13:51:00+00:00", + "weight": 0.5005418695471089, + "activation": 0.54029682394304, + "semantic_similarity": 0.628176074547323, + "recency": 3.5984424466313127e-37, + "frequency": 2.0 + }, + { + "id": "006df526-f850-48a9-a2f3-a13799f0391c", + "text": "Melanie states that the conversation with Caroline has been great for her mental health and she intends to continue the positive practices.", + "context": "Conversation session between Caroline and Melanie", + "event_date": "2023-07-12T16:33:00+00:00", + "weight": 0.4926971599924729, + "activation": 0.54029682394304, + "semantic_similarity": 0.6020270426985361, + "recency": 2.9794868813890685e-37, + "frequency": 2.0 + }, + { + "id": "a10ae2d0-45de-4fbb-8c37-9762142b5158", + "text": "As of July 17, 2023, Caroline is mentoring a transgender teen who shares her own gender identity, and together they are working on building confidence and developing positive coping strategies.", + "context": "Conversation session between Caroline and Melanie", + "event_date": "2023-07-17T14:31:00+00:00", + "weight": 0.4920764235358188, + "activation": 0.5054853867820114, + "semantic_similarity": 0.6347693583373847, + "recency": 4.870900738597122e-37, + "frequency": 2.0 + }, + { + "id": "50dd73e8-e477-4c45-b052-b3eae96a1a6a", + "text": "Melanie expressed pride in Caroline's work and said she is proud to be part of the difference Caroline is making.", + "context": "Conversation session between Caroline and Melanie", + "event_date": "2023-06-09T19:55:00+00:00", + "weight": 0.49152321091146134, + "activation": 0.54029682394304, + "semantic_similarity": 0.5981138790951646, + "recency": 1.11445329739746e-38, + "frequency": 2.0 + }, + { + "id": "63dd0bce-9e94-439c-a66b-2348579d05dc", + "text": "Caroline attended an LGBTQ+ pride parade, observed a happy crowd, felt a sense of belonging, and recognized significant growth in the LGBTQ+ community.", + "context": "Conversation session between Caroline and Melanie", + "event_date": "2023-06-26T13:36:00+00:00", + "weight": 0.4902385241685657, + "activation": 0.54029682394304, + "semantic_similarity": 0.5938315899521791, + "recency": 5.941992552736401e-38, + "frequency": 2.0 + }, + { + "id": "5f5ec14c-8715-4770-988a-6aa1fe4176f0", + "text": "Caroline attended an LGBTQ support group on 2023-05-07, finding the experience powerful.", + "context": "Conversation session between Caroline and Melanie", + "event_date": "2023-05-07T13:56:00+00:00", + "weight": 0.48909858660060834, + "activation": 0.49128755869399277, + "semantic_similarity": 0.639041063308035, + "recency": 4.009247628467812e-40, + "frequency": 2.0 + }, + { + "id": "f72607f8-5bf9-482e-ba74-f5ac97c15ac7", + "text": "The LGBTQ community experience showed Caroline how important it is to fight for trans rights and spread awareness.", + "context": "Conversation session between Caroline and Melanie", + "event_date": "2023-07-10T16:33:00+00:00", + "weight": 0.48869952969070274, + "activation": 0.54029682394304, + "semantic_similarity": 0.5887016083593024, + "recency": 2.4393975388943084e-37, + "frequency": 2.0 + }, + { + "id": "a4f20608-0fb6-4235-b47b-b64482d3e648", + "text": "Caroline stated her goal is to give kids a loving home, expressed gratitude for support from friends and mentors, and said she feels hopeful and optimistic about turning her adoption dream into reality.", + "context": "Conversation session between Caroline and Melanie", + "event_date": "2023-05-25T13:14:00+00:00", + "weight": 0.4880850435147672, + "activation": 0.54029682394304, + "semantic_similarity": 0.5866533211061841, + "recency": 2.418389539846299e-39, + "frequency": 2.0 + }, + { + "id": "88e28f0b-51b1-45bb-860b-4b1a734d3462", + "text": "Caroline reflected that volunteering reminded her of her own past struggles and feeling alone, and she was glad to share her story and offer support, feeling she could make a difference.", + "context": "Conversation session between Caroline and Melanie", + "event_date": "2023-08-28T15:19:00+00:00", + "weight": 0.4854217191260243, + "activation": 0.54029682394304, + "semantic_similarity": 0.5777755731437075, + "recency": 3.2590704712838855e-35, + "frequency": 2.0 + }, + { + "id": "05f1dcaa-6f99-46b5-87e9-d2efa64e01cd", + "text": "Caroline joined a mentorship program for LGBTQ youth on the weekend of July 15\u201316, 2023, aiming to support and empower young members of the community.", + "context": "Conversation session between Caroline and Melanie", + "event_date": "2023-07-15T14:31:00+00:00", + "weight": 0.4852769317696861, + "activation": 0.4948453757498368, + "semantic_similarity": 0.6227443968157836, + "recency": 3.987956229570448e-37, + "frequency": 2.0 + }, + { + "id": "c7ffecc3-a614-401d-9f62-a23753c7ca5f", + "text": "Melanie has been reading a book that Caroline recommended a while ago and has been painting to keep busy.", + "context": "Conversation session between Caroline and Melanie", + "event_date": "2023-10-13T10:31:00+00:00", + "weight": 0.4846367597711927, + "activation": 0.54029682394304, + "semantic_similarity": 0.5751590419609356, + "recency": 3.178062826735761e-33, + "frequency": 2.0 + }, + { + "id": "3f797cf0-1c31-4f25-8815-0791293e3bcd", + "text": "During her transition and journey toward self-acceptance, Caroline received invaluable help and encouragement from friends, family, and people she looked up to, which boosted her through tough times and helped her discover her true self.", + "context": "Conversation session between Caroline and Melanie", + "event_date": "2023-10-22T09:55:00+00:00", + "weight": 0.4830354920709111, + "activation": 0.54029682394304, + "semantic_similarity": 0.5698214829599971, + "recency": 7.797255689131378e-33, + "frequency": 2.0 + }, + { + "id": "f3fd8e60-f94c-4dad-9a29-358228690663", + "text": "Caroline plans to attend a transgender conference later this month, expressing excitement to meet community members and learn more about advocacy.", + "context": "Conversation session between Caroline and Melanie", + "event_date": "2023-07-03T13:36:00+00:00", + "weight": 0.48275036326470055, + "activation": 0.4959545664604603, + "semantic_similarity": 0.6132133110885414, + "recency": 1.1965703607295037e-37, + "frequency": 2.0 + }, + { + "id": "1a905e0b-4767-4a9e-b3ad-bc32d0657ab2", + "text": "Caroline says that her own personal journey and the support she received significantly improved her life; observing the benefits of counseling and support groups motivated her to care more about mental health, understand herself better, and create a safe, inviting environment for others to grow.", + "context": "Conversation session between Caroline and Melanie", + "event_date": "2023-06-27T10:37:00+00:00", + "weight": 0.48258318586340954, + "activation": 0.54029682394304, + "semantic_similarity": 0.5683137956016583, + "recency": 6.485792193876516e-38, + "frequency": 2.0 + }, + { + "id": "c15fad08-4a6a-4c55-9d80-696705c085c2", + "text": "Caroline and her mentee attended an LGBT pride event on June 17, 2023, where they experienced a supportive and loving atmosphere.", + "context": "Conversation session between Caroline and Melanie", + "event_date": "2023-06-17T14:31:00+00:00", + "weight": 0.481936534162537, + "activation": 0.47851202244483465, + "semantic_similarity": 0.6279430914302889, + "recency": 2.425078680378972e-38, + "frequency": 2.0 + } + ], + "is_correct": true, + "correctness_reasoning": "The predicted answer '2023-06-02' is exactly one week before 9 June 2023, matching the correct answer 'The week before 9 June 2023'." + }, + { + "question": "When did Melanie run a charity race?", + "correct_answer": "The sunday before 25 May 2023", + "predicted_answer": "May 20, 2023", + "reasoning": "The context explicitly states that Melanie ran a charity race for mental health on Saturday, May 20, 2023. This provides a clear and direct answer to the question.", + "category": 2, + "retrieved_memories": [ + { + "id": "96406bc5-690f-4f4e-ad4b-968a6648bbf1", + "text": "Melanie ran a charity race for mental health on Saturday, May 20, 2023, which she found rewarding and which made her think about the importance of taking care of one's mind.", + "context": "Conversation session between Caroline and Melanie", + "event_date": "2023-05-20T13:14:00+00:00", + "weight": 0.6080577995801428, + "activation": 0.76342961897166, + "semantic_similarity": 0.7634297129621495, + "recency": 1.4667843256884606e-39, + "frequency": 2.0 + }, + { + "id": "e460bf87-7ca8-40cb-bb49-3750d164c90d", + "text": "Melanie spent August 24, 2023, volunteering with her family at a homeless shelter, observing neglected individuals and feeling that they made a positive difference.", + "context": "Conversation session between Caroline and Melanie", + "event_date": "2023-08-24T13:33:00+00:00", + "weight": 0.5461861669634696, + "activation": 0.660310308327966, + "semantic_similarity": 0.6603102482169325, + "recency": 2.168534390799205e-35, + "frequency": 2.0 + }, + { + "id": "487ca766-c480-4748-ac1a-0362311fc948", + "text": "Melanie recently purchased new shoes, which are purple in color and intended for running.", + "context": "Conversation session between Caroline and Melanie", + "event_date": "2023-07-12T16:33:00+00:00", + "weight": 0.5379265600532386, + "activation": 0.646544314405968, + "semantic_similarity": 0.6465442191048271, + "recency": 2.9793993782823916e-37, + "frequency": 2.0 + }, + { + "id": "52c9bd89-9bd2-410a-9dd6-3e1ba3369b5d", + "text": "Caroline thanked Melanie for her encouraging words, said the support means a lot, and pledged to do her best to ensure any adopted children have a safe and loving home.", + "context": "Conversation session between Caroline and Melanie", + "event_date": "2023-05-25T13:14:00+00:00", + "weight": 0.5074708823773078, + "activation": 0.610743695177328, + "semantic_similarity": 0.5808259127470313, + "recency": 2.4183089332937094e-39, + "frequency": 2.0 + }, + { + "id": "312a4401-94ce-4509-94c4-f7f4481fcf83", + "text": "Melanie said she is carving out daily me-time that includes running, reading, or playing her violin, activities that refresh her and help her stay present for her family.", + "context": "Conversation session between Caroline and Melanie", + "event_date": "2023-05-25T13:14:00+00:00", + "weight": 0.5043215443688921, + "activation": 0.610743695177328, + "semantic_similarity": 0.5703281193856459, + "recency": 2.418308933545545e-39, + "frequency": 2.0 + }, + { + "id": "1b99342d-486f-4f1a-b1a1-1b0d94f3ded3", + "text": "Melanie stated that her husband and kids keep her motivated.", + "context": "Conversation session between Caroline and Melanie", + "event_date": "2023-06-09T19:55:00+00:00", + "weight": 0.5010528954816535, + "activation": 0.5282482466623728, + "semantic_similarity": 0.6419280716098056, + "recency": 1.1144119147535568e-38, + "frequency": 2.0 + }, + { + "id": "d6fa6c91-b0b0-48c3-8dc6-a2981abaae82", + "text": "On 2023-07-15, Melanie and her family went on a camping trip in the forest.", + "context": "Conversation session between Caroline and Melanie", + "event_date": "2023-07-15T13:51:00+00:00", + "weight": 0.5002171240450245, + "activation": 0.5282482466623728, + "semantic_similarity": 0.6391421668210422, + "recency": 3.976761380451225e-37, + "frequency": 2.0 + }, + { + "id": "6ce7be9f-ef63-4d9f-9bce-fd223d501ee8", + "text": "Melanie finds peace through creativity and family support.", + "context": "Conversation session between Caroline and Melanie", + "event_date": "2023-07-15T13:51:00+00:00", + "weight": 0.49682061678436584, + "activation": 0.5282482466623728, + "semantic_similarity": 0.6278204759521799, + "recency": 3.976761379816073e-37, + "frequency": 2.0 + }, + { + "id": "35ea25a2-0e9d-4e01-ae30-7655ca3d4c44", + "text": "The concert Melanie attended on 2023-08-13 featured performer Matt Patterson, whose voice and songs she described as amazing and talented.", + "context": "Conversation session between Caroline and Melanie", + "event_date": "2023-08-13T14:24:00+00:00", + "weight": 0.495634289319396, + "activation": 0.5282482466623728, + "semantic_similarity": 0.6238660510689472, + "recency": 7.243978255284424e-36, + "frequency": 2.0 + }, + { + "id": "d8d3b55e-b418-4ea0-886a-d4c0f8feef62", + "text": "Melanie shared a picture taken on 2023-08-13 showing many people having a blast at the concert, reminding her of the importance of cultivating a loving and accepting environment for her children.", + "context": "Conversation session between Caroline and Melanie", + "event_date": "2023-08-13T14:24:00+00:00", + "weight": 0.4955201613237674, + "activation": 0.5282482466623728, + "semantic_similarity": 0.6234856244168518, + "recency": 7.243978255577915e-36, + "frequency": 2.0 + }, + { + "id": "99533d0b-08a4-4bfb-ae6f-82ff14200c83", + "text": "Melanie has been married for five years, meaning she married her husband around 2018-06-09.", + "context": "Conversation session between Caroline and Melanie", + "event_date": "2018-06-09T19:55:00+00:00", + "weight": 0.49343939367036094, + "activation": 0.5282482466623728, + "semantic_similarity": 0.6165497322388304, + "recency": 5.557442803148722e-118, + "frequency": 2.0 + }, + { + "id": "6cd22599-91e4-4777-8cea-136df2061405", + "text": "Melanie is currently swamped with caring for her kids and managing work responsibilities.", + "context": "Conversation session between Caroline and Melanie", + "event_date": "2023-05-08T13:56:00+00:00", + "weight": 0.4929780369843564, + "activation": 0.5282482466623728, + "semantic_similarity": 0.6150118766188151, + "recency": 4.430739346790953e-40, + "frequency": 2.0 + }, + { + "id": "50dd73e8-e477-4c45-b052-b3eae96a1a6a", + "text": "Melanie expressed pride in Caroline's work and said she is proud to be part of the difference Caroline is making.", + "context": "Conversation session between Caroline and Melanie", + "event_date": "2023-06-09T19:55:00+00:00", + "weight": 0.49254514957772355, + "activation": 0.5282482466623728, + "semantic_similarity": 0.6135689185967056, + "recency": 1.1144161503627e-38, + "frequency": 2.0 + }, + { + "id": "757040fe-bd93-4ed8-bbe2-731b6b33fb56", + "text": "Melanie went camping with her children a few weeks before September 13, 2023 (approximately on August 23, 2023), during which they explored the forest, hiked, and had a blast.", + "context": "Conversation session between Caroline and Melanie", + "event_date": "2023-08-23T00:09:00+00:00", + "weight": 0.49111653757489904, + "activation": 0.5282482466623728, + "semantic_similarity": 0.6088068785872908, + "recency": 1.855604365068456e-35, + "frequency": 2.0 + }, + { + "id": "f9c087ae-d272-4bfb-b111-fc538fd577fd", + "text": "On 2023-07-14, Melanie took her kids to a pottery workshop where they all made their own pots; the activity was fun and therapeutic, and the kids loved it, being excited to get their hands dirty and create something with clay.", + "context": "Conversation session between Caroline and Melanie", + "event_date": "2023-07-14T13:51:00+00:00", + "weight": 0.4909032205325806, + "activation": 0.5282482466623728, + "semantic_similarity": 0.6080958217795625, + "recency": 3.598322500348928e-37, + "frequency": 2.0 + }, + { + "id": "e21e9110-0c25-415b-a959-a21eb72d8a4b", + "text": "Melanie once fed a horse a carrot.", + "context": "Conversation session between Caroline and Melanie", + "event_date": "2023-08-23T15:31:00+00:00", + "weight": 0.4908149382345375, + "activation": 0.5282482466623728, + "semantic_similarity": 0.6078015474527522, + "recency": 1.9783006598182982e-35, + "frequency": 2.0 + }, + { + "id": "a18b044a-2586-45b9-93d1-3360db92e821", + "text": "Melanie uses painting as a fun way to express her feelings, get creative, and relax after a long day.", + "context": "Conversation session between Caroline and Melanie", + "event_date": "2023-05-08T13:56:00+00:00", + "weight": 0.49081204805059286, + "activation": 0.610743695177328, + "semantic_similarity": 0.5252964649913149, + "recency": 4.430756193083742e-40, + "frequency": 2.0 + }, + { + "id": "64621ccc-3569-43ab-bcbc-300752e4cba9", + "text": "On the weekend of 2023-07-08, Melanie and her kids painted a nature-inspired artwork together, which they described as their latest work.", + "context": "Conversation session between Caroline and Melanie", + "event_date": "2023-07-08T13:51:00+00:00", + "weight": 0.49078994519644037, + "activation": 0.5282482466623728, + "semantic_similarity": 0.6077182373257619, + "recency": 1.974793753559377e-37, + "frequency": 2.0 + }, + { + "id": "54526f1d-bc1e-4eb1-96e9-d0bbe6c3e56e", + "text": "Melanie celebrated her daughter's birthday with a concert on the evening of 2023-08-13, featuring music, joy, and a warm summer breeze; she observed her kids' smiles and expressed thankfulness for these special moments.", + "context": "Conversation session between Caroline and Melanie", + "event_date": "2023-08-13T14:24:00+00:00", + "weight": 0.4906547525976461, + "activation": 0.5282482466623728, + "semantic_similarity": 0.6072675953297808, + "recency": 7.243978254990933e-36, + "frequency": 2.0 + }, + { + "id": "6fb4ae9a-e9aa-4284-ae34-c5a157b98fcc", + "text": "Melanie stated that her family is her biggest motivation and support, giving her strength to keep going.", + "context": "Conversation session between Caroline and Melanie", + "event_date": "2023-10-14T18:55:00+00:00", + "weight": 0.48920699159952397, + "activation": 0.5282482466623728, + "semantic_similarity": 0.6024417253360405, + "recency": 3.637288558995905e-33, + "frequency": 2.0 + } + ], + "is_correct": false, + "correctness_reasoning": "The correct answer specifies \"the Sunday before 25 May 2023,\" which is 21 May 2023. The predicted answer gives May 20, 2023 (a Saturday), so they are not equivalent." + }, { "question": "When did Caroline go to the LGBTQ support group?", "correct_answer": "7 May 2023", - "predicted_answer": "Caroline attended the LGBTQ support group yesterday.", + "predicted_answer": "2023-05-07", + "reasoning": "The context explicitly states that Caroline attended an LGBTQ support group on 2023-05-07, providing a clear answer to the question.", "category": 2, - "is_correct": false + "retrieved_memories": [ + { + "id": "5f5ec14c-8715-4770-988a-6aa1fe4176f0", + "text": "Caroline attended an LGBTQ support group on 2023-05-07, finding the experience powerful.", + "context": "Conversation session between Caroline and Melanie", + "event_date": "2023-05-07T13:56:00+00:00", + "weight": 0.6796956536236841, + "activation": 0.882826116169278, + "semantic_similarity": 0.8828260625763358, + "recency": 4.009011402269459e-40, + "frequency": 2.0 + }, + { + "id": "3a2f0dfc-4fc1-4986-9474-ce54481550a3", + "text": "Caroline joined a new LGBTQ activist group called Connected LGBTQ Activists on Tuesday, July 18, 2023.", + "context": "Conversation session between Caroline and Melanie", + "event_date": "2023-07-18T20:56:00+00:00", + "weight": 0.6497555047258308, + "activation": 0.832925821331913, + "semantic_similarity": 0.832925861087523, + "recency": 5.528718569948636e-37, + "frequency": 2.0 + }, + { + "id": "72855141-35e9-47cd-92aa-5d1ffb89ae3a", + "text": "The LGBTQ support group made Caroline feel accepted and gave her courage to embrace herself.", + "context": "Conversation session between Caroline and Melanie", + "event_date": "2023-05-08T13:56:00+00:00", + "weight": 0.6274170101632144, + "activation": 0.795695054274503, + "semantic_similarity": 0.7956949796028784, + "recency": 4.43064281101677e-40, + "frequency": 2.0 + }, + { + "id": "c5161f91-693c-48a1-a288-ad149f8fc765", + "text": "Caroline attended an LGBTQ conference on 2023-07-10, two days before the reference date, where she met and connected with people who have experienced similar journeys, found the environment welcoming, felt totally accepted, and expressed gratitude for the community.", + "context": "Conversation session between Caroline and Melanie", + "event_date": "2023-07-10T16:33:00+00:00", + "weight": 0.600047675078097, + "activation": 0.7062608929354224, + "semantic_similarity": 0.7938980239915674, + "recency": 2.4392508031077425e-37, + "frequency": 2.0 + }, + { + "id": "ec88eee3-ba37-427f-9bf4-e0995d5fc61a", + "text": "On Friday, June 23, 2023, Caroline attended an LGBTQ+ counseling workshop where professionals discussed various therapeutic methods for working with transgender people and demonstrated passion for creating safe spaces for individuals like her.", + "context": "Conversation session between Caroline and Melanie", + "event_date": "2023-06-23T10:37:00+00:00", + "weight": 0.598690975861133, + "activation": 0.7062608929354224, + "semantic_similarity": 0.789375693268354, + "recency": 4.3472950109126337e-38, + "frequency": 2.0 + }, + { + "id": "63dd0bce-9e94-439c-a66b-2348579d05dc", + "text": "Caroline attended an LGBTQ+ pride parade, observed a happy crowd, felt a sense of belonging, and recognized significant growth in the LGBTQ+ community.", + "context": "Conversation session between Caroline and Melanie", + "event_date": "2023-06-26T13:36:00+00:00", + "weight": 0.5832675302614079, + "activation": 0.7062608929354224, + "semantic_similarity": 0.7379642079359373, + "recency": 5.941635134168808e-38, + "frequency": 2.0 + }, + { + "id": "f72607f8-5bf9-482e-ba74-f5ac97c15ac7", + "text": "The LGBTQ community experience showed Caroline how important it is to fight for trans rights and spread awareness.", + "context": "Conversation session between Caroline and Melanie", + "event_date": "2023-07-10T16:33:00+00:00", + "weight": 0.5811890862685835, + "activation": 0.7062608929354224, + "semantic_similarity": 0.7310360612931892, + "recency": 2.439250805417151e-37, + "frequency": 2.0 + }, + { + "id": "453f811d-9535-42e7-8784-7a0eb35362ba", + "text": "During the June 17, 2023 LGBT pride event, Caroline recalls that the best moment was seeing her mentee's face light up upon witnessing the community's support.", + "context": "Conversation session between Caroline and Melanie", + "event_date": "2023-06-17T14:31:00+00:00", + "weight": 0.5757488452699021, + "activation": 0.7062608929354224, + "semantic_similarity": 0.7129019246309181, + "recency": 2.4249328050903984e-38, + "frequency": 2.0 + }, + { + "id": "c8ae7b11-e6b1-4360-9c03-7a265411447f", + "text": "The group Connected LGBTQ Activists, which Caroline joined, is composed of diverse members investing in positive changes, holds regular meetings, and plans events and campaigns to support each other, as described on July 20, 2023.", + "context": "Conversation session between Caroline and Melanie", + "event_date": "2023-07-20T20:56:00+00:00", + "weight": 0.5706201879423725, + "activation": 0.6663406570655304, + "semantic_similarity": 0.7357266360757113, + "recency": 6.752783785716208e-37, + "frequency": 2.0 + }, + { + "id": "05f1dcaa-6f99-46b5-87e9-d2efa64e01cd", + "text": "Caroline joined a mentorship program for LGBTQ youth on the weekend of July 15\u201316, 2023, aiming to support and empower young members of the community.", + "context": "Conversation session between Caroline and Melanie", + "event_date": "2023-07-15T14:31:00+00:00", + "weight": 0.5674557696440229, + "activation": 0.6089139880927572, + "semantic_similarity": 0.7826052440539858, + "recency": 3.9877163420957654e-37, + "frequency": 2.0 + }, + { + "id": "c15fad08-4a6a-4c55-9d80-696705c085c2", + "text": "Caroline and her mentee attended an LGBT pride event on June 17, 2023, where they experienced a supportive and loving atmosphere.", + "context": "Conversation session between Caroline and Melanie", + "event_date": "2023-06-17T14:31:00+00:00", + "weight": 0.5674245686658036, + "activation": 0.6096885261040705, + "semantic_similarity": 0.7817267027819415, + "recency": 2.424932805009003e-38, + "frequency": 2.0 + }, + { + "id": "22612de6-dda7-4df8-af30-5d488b1bdc10", + "text": "On 2023-07-14, Caroline attended a council meeting for adoption; she found it inspiring and emotional, observed many people wanting to create loving homes for children in need, and felt even more determined to adopt.", + "context": "Conversation session between Caroline and Melanie", + "event_date": "2023-07-14T13:51:00+00:00", + "weight": 0.5658305569267591, + "activation": 0.7062608929354224, + "semantic_similarity": 0.6798409634871081, + "recency": 3.5982259920890336e-37, + "frequency": 2.0 + }, + { + "id": "aa36595e-58ca-42dc-b50d-8c785526bbf3", + "text": "Melanie's support has been meaningful to Caroline; Caroline feels grateful for Melanie's support throughout her journey and values being able to share and help others.", + "context": "Conversation session between Caroline and Melanie", + "event_date": "2023-10-22T09:55:00+00:00", + "weight": 0.5543802248372177, + "activation": 0.7062608929354224, + "semantic_similarity": 0.6416731898553034, + "recency": 7.796786674122317e-33, + "frequency": 2.0 + }, + { + "id": "3f797cf0-1c31-4f25-8815-0791293e3bcd", + "text": "During her transition and journey toward self-acceptance, Caroline received invaluable help and encouragement from friends, family, and people she looked up to, which boosted her through tough times and helped her discover her true self.", + "context": "Conversation session between Caroline and Melanie", + "event_date": "2023-10-22T09:55:00+00:00", + "weight": 0.5483398419787363, + "activation": 0.7062608929354224, + "semantic_similarity": 0.6215385803270321, + "recency": 7.796786674402086e-33, + "frequency": 2.0 + }, + { + "id": "88e28f0b-51b1-45bb-860b-4b1a734d3462", + "text": "Caroline reflected that volunteering reminded her of her own past struggles and feeling alone, and she was glad to share her story and offer support, feeling she could make a difference.", + "context": "Conversation session between Caroline and Melanie", + "event_date": "2023-08-28T15:19:00+00:00", + "weight": 0.5473932926751475, + "activation": 0.7062608929354224, + "semantic_similarity": 0.6183834159817357, + "recency": 3.258874434261174e-35, + "frequency": 2.0 + }, + { + "id": "1a905e0b-4767-4a9e-b3ad-bc32d0657ab2", + "text": "Caroline says that her own personal journey and the support she received significantly improved her life; observing the benefits of counseling and support groups motivated her to care more about mental health, understand herself better, and create a safe, inviting environment for others to grow.", + "context": "Conversation session between Caroline and Melanie", + "event_date": "2023-06-27T10:37:00+00:00", + "weight": 0.5473810399227539, + "activation": 0.7062608929354224, + "semantic_similarity": 0.6183425734737571, + "recency": 6.485402065039465e-38, + "frequency": 2.0 + }, + { + "id": "a4f20608-0fb6-4235-b47b-b64482d3e648", + "text": "Caroline stated her goal is to give kids a loving home, expressed gratitude for support from friends and mentors, and said she feels hopeful and optimistic about turning her adoption dream into reality.", + "context": "Conversation session between Caroline and Melanie", + "event_date": "2023-05-25T13:14:00+00:00", + "weight": 0.54652741823091, + "activation": 0.7062608929354224, + "semantic_similarity": 0.6154971678342778, + "recency": 2.4182440707207888e-39, + "frequency": 2.0 + }, + { + "id": "50dd73e8-e477-4c45-b052-b3eae96a1a6a", + "text": "Melanie expressed pride in Caroline's work and said she is proud to be part of the difference Caroline is making.", + "context": "Conversation session between Caroline and Melanie", + "event_date": "2023-06-09T19:55:00+00:00", + "weight": 0.5463333678377977, + "activation": 0.7062608929354224, + "semantic_similarity": 0.61485033319057, + "recency": 1.1143862616477157e-38, + "frequency": 2.0 + }, + { + "id": "45907243-354f-45f5-9bd6-61199272b94e", + "text": "Caroline is organizing an LGBTQ art show scheduled for September 2023 (specifically around September 25, 2023) to exhibit her paintings and promote understanding and acceptance within the community.", + "context": "Conversation session between Caroline and Melanie", + "event_date": "2023-09-25T13:33:00+00:00", + "weight": 0.5424624381226976, + "activation": 0.5546027376588833, + "semantic_similarity": 0.7536053894167751, + "recency": 5.3197997659606686e-34, + "frequency": 2.0 + }, + { + "id": "d34361a2-3e61-4679-8516-7de3a31de2a8", + "text": "Caroline volunteered at an LGBTQ+ youth center, found it gratifying to talk to similar young people, and it reminded her of how essential it is to be kind and show support.", + "context": "Conversation session between Caroline and Melanie", + "event_date": "2023-08-28T15:19:00+00:00", + "weight": 0.5378260542978969, + "activation": 0.5650087143483379, + "semantic_similarity": 0.7277447999779849, + "recency": 3.2588642110786975e-35, + "frequency": 2.0 + } + ], + "is_correct": true, + "correctness_reasoning": "The predicted answer 2023-05-07 represents the same date as 7 May 2023, just in ISO format, so it is equivalent to the correct answer." }, { "question": "When did Melanie paint a sunrise?", "correct_answer": 2022, - "predicted_answer": "I don't know.", + "predicted_answer": "2022-05-08", + "reasoning": "The context explicitly states that Melanie painted a lake sunrise painting last year on 2022-05-08. This directly answers the question about when she painted a sunrise.", "category": 2, - "is_correct": false + "retrieved_memories": [ + { + "id": "be729e0b-7a37-4fef-98c1-5da6390b128a", + "text": "Melanie painted a lake sunrise painting last year (2022-05-08), and the artwork is special to her.", + "context": "Conversation session between Caroline and Melanie", + "event_date": "2022-05-08T13:56:00+00:00", + "weight": 0.649579216254692, + "activation": 0.832632040004971, + "semantic_similarity": 0.8326320141773355, + "recency": 6.233113001978623e-56, + "frequency": 2.0 + }, + { + "id": "74c1a0a2-91c5-4724-8b8e-84d3d2a1492f", + "text": "Melanie created a painting on 2023-10-06 that was inspired by sunsets; the colors of the painting make her feel calm.", + "context": "Conversation session between Caroline and Melanie", + "event_date": "2023-10-06T10:31:00+00:00", + "weight": 0.6319213512865487, + "activation": 0.803202224064072, + "semantic_similarity": 0.8032022802244235, + "recency": 1.578038526900449e-33, + "frequency": 2.0 + }, + { + "id": "7d96e021-b14a-4ecd-adb2-3036aeac918a", + "text": "Melanie recently painted a horse, creating a painting that she shared in a photo.", + "context": "Conversation session between Caroline and Melanie", + "event_date": "2023-08-23T15:31:00+00:00", + "weight": 0.5858844971979617, + "activation": 0.726474090008383, + "semantic_similarity": 0.7264742339848227, + "recency": 1.9781976575661012e-35, + "frequency": 2.0 + }, + { + "id": "fb45c7ac-8079-4af6-bcf2-6ae9dd3ea1d4", + "text": "Melanie recently completed a landscape painting, which she shared during the conversation on August 25, 2023.", + "context": "Conversation session between Caroline and Melanie", + "event_date": "2023-08-25T13:33:00+00:00", + "weight": 0.5676932504481701, + "activation": 0.6661056320039769, + "semantic_similarity": 0.7262052028232568, + "recency": 2.3964478024196726e-35, + "frequency": 2.0 + }, + { + "id": "64621ccc-3569-43ab-bcbc-300752e4cba9", + "text": "On the weekend of 2023-07-08, Melanie and her kids painted a nature-inspired artwork together, which they described as their latest work.", + "context": "Conversation session between Caroline and Melanie", + "event_date": "2023-07-08T13:51:00+00:00", + "weight": 0.5674790675852099, + "activation": 0.6661056320039769, + "semantic_similarity": 0.7254912599467229, + "recency": 1.9746827385360728e-37, + "frequency": 2.0 + }, + { + "id": "92a5a070-fa23-4903-ad73-660b7940181b", + "text": "In her sunset\u2011inspired painting, Melanie intentionally used peaceful blue streaks to convey tranquility, noting that the color blue calms her and that she wanted a serene vibe combined with vibrant colors.", + "context": "Conversation session between Caroline and Melanie", + "event_date": "2023-10-06T10:31:00+00:00", + "weight": 0.5589249296518987, + "activation": 0.6661056320039769, + "semantic_similarity": 0.6969774668356853, + "recency": 1.5780265930621085e-33, + "frequency": 2.0 + }, + { + "id": "b1a619f5-f467-428e-a84f-e6a48bee33e5", + "text": "Melanie and her children completed another collaborative painting, similar to a previous work, on July 17, 2023.", + "context": "Conversation session between Caroline and Melanie", + "event_date": "2023-07-17T14:31:00+00:00", + "weight": 0.5537432899118439, + "activation": 0.6661056320039769, + "semantic_similarity": 0.679705334368836, + "recency": 4.870446048270206e-37, + "frequency": 2.0 + }, + { + "id": "eb32a52f-791a-42dc-8476-a542e84d2a71", + "text": "Melanie's favorite artistic activities are painting landscapes and still life, which she appreciates for their depiction of nature, as she mentioned on August 25, 2023.", + "context": "Conversation session between Caroline and Melanie", + "event_date": "2023-08-25T13:33:00+00:00", + "weight": 0.552060742248721, + "activation": 0.6661056320039769, + "semantic_similarity": 0.6740968421584266, + "recency": 2.3964396354115454e-35, + "frequency": 2.0 + }, + { + "id": "ca9d8d15-1560-4823-936d-79be60dfb4f9", + "text": "Melanie has been involved in art for seven years and has recently identified painting and pottery as her primary muses.", + "context": "Conversation session between Caroline and Melanie", + "event_date": "2023-09-13T00:09:00+00:00", + "weight": 0.5478093718537007, + "activation": 0.6661056320039769, + "semantic_similarity": 0.6599256075083587, + "recency": 1.515227697180664e-34, + "frequency": 2.0 + }, + { + "id": "f9c087ae-d272-4bfb-b111-fc538fd577fd", + "text": "On 2023-07-14, Melanie took her kids to a pottery workshop where they all made their own pots; the activity was fun and therapeutic, and the kids loved it, being excited to get their hands dirty and create something with clay.", + "context": "Conversation session between Caroline and Melanie", + "event_date": "2023-07-14T13:51:00+00:00", + "weight": 0.5477892165130516, + "activation": 0.6661056320039769, + "semantic_similarity": 0.6598584230395286, + "recency": 3.598106543018016e-37, + "frequency": 2.0 + }, + { + "id": "c7ffecc3-a614-401d-9f62-a23753c7ca5f", + "text": "Melanie has been reading a book that Caroline recommended a while ago and has been painting to keep busy.", + "context": "Conversation session between Caroline and Melanie", + "event_date": "2023-10-13T10:31:00+00:00", + "weight": 0.5473645916854543, + "activation": 0.6661056320039769, + "semantic_similarity": 0.6584430069475373, + "recency": 3.177755324011146e-33, + "frequency": 2.0 + }, + { + "id": "ff8318fb-cbb5-4f95-a985-da11cfb910df", + "text": "Caroline painted a piece after visiting an LGBTQ center, intending to capture the community's unity and strength; she shared this painting with Melanie on July 17, 2023.", + "context": "Conversation session between Caroline and Melanie", + "event_date": "2023-07-17T14:31:00+00:00", + "weight": 0.5463288420862615, + "activation": 0.6661056320039769, + "semantic_similarity": 0.6549905082835616, + "recency": 4.8704460485521115e-37, + "frequency": 2.0 + }, + { + "id": "6ce7be9f-ef63-4d9f-9bce-fd223d501ee8", + "text": "Melanie finds peace through creativity and family support.", + "context": "Conversation session between Caroline and Melanie", + "event_date": "2023-07-15T13:51:00+00:00", + "weight": 0.5458624279050149, + "activation": 0.6661056320039769, + "semantic_similarity": 0.6534357943460729, + "recency": 3.976522710008435e-37, + "frequency": 2.0 + }, + { + "id": "33bfbe46-a1b7-4119-8110-33f78643abe5", + "text": "During a family camping trip in August 2022, Melanie's family observed the Perseid meteor shower, made wishes, and felt awe while lying under a clear, star\u2011filled sky.", + "context": "Conversation session between Caroline and Melanie", + "event_date": "2022-08-12T20:56:00+00:00", + "weight": 0.5424828580079306, + "activation": 0.6661056320039769, + "semantic_similarity": 0.6421705613557918, + "recency": 9.475391120595462e-52, + "frequency": 2.0 + }, + { + "id": "6b0289f7-0dbc-4a51-8628-16d23a5991c1", + "text": "Melanie feels inspired by the autumn season and is planning to create several new paintings in the near future, as expressed on August 25, 2023.", + "context": "Conversation session between Caroline and Melanie", + "event_date": "2023-08-25T13:33:00+00:00", + "weight": 0.5421374526811097, + "activation": 0.6661056320039769, + "semantic_similarity": 0.6410192102663889, + "recency": 2.3964396346432544e-35, + "frequency": 2.0 + }, + { + "id": "8064be3c-0593-4421-b4b5-b3aadffaebfc", + "text": "Melanie also produced an abstract painting and says she loves how art allows her to release her emotions.", + "context": "Conversation session between Caroline and Melanie", + "event_date": "2023-10-13T10:31:00+00:00", + "weight": 0.5409897524410486, + "activation": 0.6661056320039769, + "semantic_similarity": 0.6371935427995187, + "recency": 3.1777553241325774e-33, + "frequency": 2.0 + }, + { + "id": "d8d3b55e-b418-4ea0-886a-d4c0f8feef62", + "text": "Melanie shared a picture taken on 2023-08-13 showing many people having a blast at the concert, reminding her of the importance of cultivating a loving and accepting environment for her children.", + "context": "Conversation session between Caroline and Melanie", + "event_date": "2023-08-13T14:24:00+00:00", + "weight": 0.5392727580067038, + "activation": 0.6661056320039769, + "semantic_similarity": 0.6314702280183694, + "recency": 7.243571021106428e-36, + "frequency": 2.0 + }, + { + "id": "e3463e52-1b17-4a7a-b548-e5ccde5a4e2a", + "text": "Melanie created a pottery plate in her pottery class on August 24, 2023.", + "context": "Conversation session between Caroline and Melanie", + "event_date": "2023-08-24T13:33:00+00:00", + "weight": 0.539217989521479, + "activation": 0.6661056320039769, + "semantic_similarity": 0.6312876664009535, + "recency": 2.168395641871343e-35, + "frequency": 2.0 + }, + { + "id": "99533d0b-08a4-4bfb-ae6f-82ff14200c83", + "text": "Melanie has been married for five years, meaning she married her husband around 2018-06-09.", + "context": "Conversation session between Caroline and Melanie", + "event_date": "2018-06-09T19:55:00+00:00", + "weight": 0.5364959624660652, + "activation": 0.6661056320039769, + "semantic_similarity": 0.6222142428829073, + "recency": 5.557090313333293e-118, + "frequency": 2.0 + }, + { + "id": "d6fa6c91-b0b0-48c3-8dc6-a2981abaae82", + "text": "On 2023-07-15, Melanie and her family went on a camping trip in the forest.", + "context": "Conversation session between Caroline and Melanie", + "event_date": "2023-07-15T13:51:00+00:00", + "weight": 0.5359886032130999, + "activation": 0.6661056320039769, + "semantic_similarity": 0.6205230453730228, + "recency": 3.976522710500861e-37, + "frequency": 2.0 + } + ], + "is_correct": true, + "correctness_reasoning": "The predicted answer '2022-05-08' includes the year 2022, which matches the correct answer. The extra specificity does not contradict the required information, so the answers are equivalent." }, { "question": "What fields would Caroline be likely to pursue in her educaton?", "correct_answer": "Psychology, counseling certification", - "predicted_answer": "Caroline would be likely to pursue fields in counseling or mental health.", + "predicted_answer": "Counseling and mental health", + "reasoning": "The context provides multiple statements indicating Caroline's interest in pursuing a career in counseling and mental health. She has been actively researching these fields, plans to specialize in working with transgender individuals, and has expressed a desire to help others who have faced similar challenges. Therefore, it is clear that these are the fields she is likely to pursue in her education.", "category": 3, - "is_correct": true + "retrieved_memories": [ + { + "id": "5819027e-8dda-4d6c-aecf-696e998337c2", + "text": "Caroline plans to continue her education and explore career options, expressing excitement about the prospect.", + "context": "Conversation session between Caroline and Melanie", + "event_date": "2023-05-08T13:56:00+00:00", + "weight": 0.6333278909006648, + "activation": 0.805546402931213, + "semantic_similarity": 0.8055465667376696, + "recency": 4.430686961864207e-40, + "frequency": 2.0 + }, + { + "id": "eeb4a464-dc3f-4123-a16e-bfa95dedf0ea", + "text": "Caroline has recently been researching a career in counseling and mental health because she wants to help people who have experienced similar challenges to her own.", + "context": "Conversation session between Caroline and Melanie", + "event_date": "2023-06-27T10:37:00+00:00", + "weight": 0.5878932463286886, + "activation": 0.729822131416549, + "semantic_similarity": 0.7298220230124131, + "recency": 6.485474681344336e-38, + "frequency": 2.0 + }, + { + "id": "c759737b-8fff-4941-8c74-c832339e7595", + "text": "Caroline is actively researching and considering career options in counseling and mental health because she believes it is important for people to have someone to talk to and wants to help make that happen.", + "context": "Conversation session between Caroline and Melanie", + "event_date": "2023-07-12T16:33:00+00:00", + "weight": 0.582146898172474, + "activation": 0.720244824886322, + "semantic_similarity": 0.7202448356885913, + "recency": 2.9793410177962352e-37, + "frequency": 2.0 + }, + { + "id": "280b9bd7-1ed9-4a8a-b169-89af4775ae9b", + "text": "Caroline is keen on becoming a counselor or working in mental health so she can support people who face similar issues.", + "context": "Conversation session between Caroline and Melanie", + "event_date": "2023-05-08T13:56:00+00:00", + "weight": 0.5574724976051302, + "activation": 0.6444371223449705, + "semantic_similarity": 0.7138045363387968, + "recency": 4.43068163950985e-40, + "frequency": 2.0 + }, + { + "id": "bf146ef5-5001-4773-af33-8ab30c2b167f", + "text": "Caroline plans to specialize in working with transgender individuals, aiming to help them accept themselves and support their mental health.", + "context": "Conversation session between Caroline and Melanie", + "event_date": "2023-06-27T10:37:00+00:00", + "weight": 0.5190508116640551, + "activation": 0.5838577051332392, + "semantic_similarity": 0.646311667080278, + "recency": 6.485466888248173e-38, + "frequency": 2.0 + }, + { + "id": "05f1dcaa-6f99-46b5-87e9-d2efa64e01cd", + "text": "Caroline joined a mentorship program for LGBTQ youth on the weekend of July 15\u201316, 2023, aiming to support and empower young members of the community.", + "context": "Conversation session between Caroline and Melanie", + "event_date": "2023-07-15T14:31:00+00:00", + "weight": 0.5110870124043192, + "activation": 0.5761958599090576, + "semantic_similarity": 0.6274275147720062, + "recency": 3.987756193524277e-37, + "frequency": 2.0 + }, + { + "id": "a10ae2d0-45de-4fbb-8c37-9762142b5158", + "text": "As of July 17, 2023, Caroline is mentoring a transgender teen who shares her own gender identity, and together they are working on building confidence and developing positive coping strategies.", + "context": "Conversation session between Caroline and Melanie", + "event_date": "2023-07-17T14:31:00+00:00", + "weight": 0.5098787746974163, + "activation": 0.5761958599090576, + "semantic_similarity": 0.6234000557489967, + "recency": 4.870656422564771e-37, + "frequency": 2.0 + }, + { + "id": "1a905e0b-4767-4a9e-b3ad-bc32d0657ab2", + "text": "Caroline says that her own personal journey and the support she received significantly improved her life; observing the benefits of counseling and support groups motivated her to care more about mental health, understand herself better, and create a safe, inviting environment for others to grow.", + "context": "Conversation session between Caroline and Melanie", + "event_date": "2023-06-27T10:37:00+00:00", + "weight": 0.509216984880943, + "activation": 0.5838577051332392, + "semantic_similarity": 0.6135322444699044, + "recency": 6.485466888090481e-38, + "frequency": 2.0 + }, + { + "id": "72855141-35e9-47cd-92aa-5d1ffb89ae3a", + "text": "The LGBTQ support group made Caroline feel accepted and gave her courage to embrace herself.", + "context": "Conversation session between Caroline and Melanie", + "event_date": "2023-05-08T13:56:00+00:00", + "weight": 0.5076642422722653, + "activation": 0.6444371223449705, + "semantic_similarity": 0.5477770185625805, + "recency": 4.430681638150903e-40, + "frequency": 2.0 + }, + { + "id": "5477179a-380b-4266-8c50-a88d995c574a", + "text": "Caroline found the transgender stories shared at the LGBTQ support group inspiring, and she felt happy and thankful for the support she received.", + "context": "Conversation session between Caroline and Melanie", + "event_date": "2023-05-08T13:56:00+00:00", + "weight": 0.5044990900556001, + "activation": 0.6444371223449705, + "semantic_similarity": 0.5372265111736967, + "recency": 4.430681638263671e-40, + "frequency": 2.0 + }, + { + "id": "83e62a1b-74b9-40e2-b6d8-e4d9dab557a6", + "text": "Caroline received a necklace as a gift from her grandmother in Sweden when she was young; the necklace symbolizes love, faith, and strength and serves as a reminder of her roots and the love and support she receives from her family.", + "context": "Conversation session between Caroline and Melanie", + "event_date": "2023-06-27T10:37:00+00:00", + "weight": 0.4990035365793578, + "activation": 0.5838577051332392, + "semantic_similarity": 0.5794874167979536, + "recency": 6.485466888405682e-38, + "frequency": 2.0 + }, + { + "id": "73850c77-935b-45c3-9a68-3552e0f85121", + "text": "Caroline emphasizes that mental health is a priority and advises Melanie to take care of herself.", + "context": "Conversation session between Caroline and Melanie", + "event_date": "2023-07-12T16:33:00+00:00", + "weight": 0.4945876133452587, + "activation": 0.5761958599090576, + "semantic_similarity": 0.5724295179084713, + "recency": 2.9793374327131804e-37, + "frequency": 2.0 + }, + { + "id": "55adbb21-7685-4f71-8e02-e1b1e8a7c0b9", + "text": "Caroline intends to continue volunteering at the LGBTQ+ youth center, considers it an important part of her life, has made strong connections with people there, believes in community and supporting each other, and wants to keep making a difference.", + "context": "Conversation session between Caroline and Melanie", + "event_date": "2023-08-28T15:19:00+00:00", + "weight": 0.49254106871292913, + "activation": 0.48033283357744794, + "semantic_similarity": 0.6614707287989824, + "recency": 3.2589070053349804e-35, + "frequency": 2.0 + }, + { + "id": "4c301737-73f0-4944-a209-32acae5d4478", + "text": "Caroline says the book taught her self-acceptance, how to find support, that tough times don't last, and that hope and love exist; she also notes that pets bring a lot of joy.", + "context": "Conversation session between Caroline and Melanie", + "event_date": "2023-07-12T16:33:00+00:00", + "weight": 0.4924063984750212, + "activation": 0.5761958599090576, + "semantic_similarity": 0.5651588016743463, + "recency": 2.9793374365373596e-37, + "frequency": 2.0 + }, + { + "id": "53358350-28d3-4192-ba20-b7259eeebee8", + "text": "Caroline previously struggled with mental health, received helpful support, which made her realize the importance of having a support system for others, and consequently started looking into counseling and mental health career options to assist others on their journeys.", + "context": "Conversation session between Caroline and Melanie", + "event_date": "2023-07-12T16:33:00+00:00", + "weight": 0.491981419428552, + "activation": 0.5031505271192448, + "semantic_similarity": 0.6367875376425951, + "recency": 2.9793374376615006e-37, + "frequency": 2.0 + }, + { + "id": "006df526-f850-48a9-a2f3-a13799f0391c", + "text": "Melanie states that the conversation with Caroline has been great for her mental health and she intends to continue the positive practices.", + "context": "Conversation session between Caroline and Melanie", + "event_date": "2023-07-12T16:33:00+00:00", + "weight": 0.4911463040758277, + "activation": 0.5761958599090576, + "semantic_similarity": 0.5609584870103681, + "recency": 2.9793374331028256e-37, + "frequency": 2.0 + }, + { + "id": "ddf46000-7d88-44b4-b9f7-ba012537e710", + "text": "Caroline loves the book \"Becoming Nicole\" by Amy Ellis Nutt, a true story about a trans girl and her family, which made her feel connected, gave her hope for her own path, and she highly recommends it.", + "context": "Conversation session between Caroline and Melanie", + "event_date": "2023-07-12T16:33:00+00:00", + "weight": 0.4907655327709255, + "activation": 0.5761958599090576, + "semantic_similarity": 0.5596892493273606, + "recency": 2.9793374370442836e-37, + "frequency": 2.0 + }, + { + "id": "5c2765c4-57b3-4606-a52c-53a87daf2b2f", + "text": "Caroline expressed that she is ready to become a mother and share her love and family.", + "context": "Conversation session between Caroline and Melanie", + "event_date": "2023-10-13T10:31:00+00:00", + "weight": 0.4874331455256029, + "activation": 0.4813448124600541, + "semantic_similarity": 0.6434323392919554, + "recency": 3.177903424126703e-33, + "frequency": 2.0 + }, + { + "id": "5e2c78a6-b363-44f7-a74b-23b7b61980f9", + "text": "Caroline has been looking into counseling or mental health work since her last conversation with Melanie, expressing passion for helping people, describing the work as tough but rewarding.", + "context": "Conversation session between Caroline and Melanie", + "event_date": "2023-07-06T20:18:00+00:00", + "weight": 0.4868065782901396, + "activation": 0.5008205505932704, + "semantic_similarity": 0.6218680437071952, + "recency": 1.6608440540150602e-37, + "frequency": 2.0 + }, + { + "id": "1c925f41-511f-4e75-89ef-b0412f644b60", + "text": "Caroline said the parade motivated her to use her personal story to help others, and she is considering a career in counseling and mental health to give back to the community.", + "context": "Conversation session between Caroline and Melanie", + "event_date": "2023-07-03T13:36:00+00:00", + "weight": 0.48498031705910283, + "activation": 0.4804768746325535, + "semantic_similarity": 0.6361241822311224, + "recency": 1.1965103429385313e-37, + "frequency": 2.0 + } + ], + "is_correct": true, + "correctness_reasoning": "The predicted answer 'Counseling and mental health' covers the same domains as the correct answer 'Psychology, counseling certification'. 'Counseling' matches the counseling certification, and 'mental health' corresponds to the field of psychology. Thus the content is equivalent." + }, + { + "question": "What is Caroline's identity?", + "correct_answer": "Transgender woman", + "predicted_answer": "Caroline is a trans woman.", + "reasoning": "The context explicitly states that Caroline created a painting intended to represent her journey as a trans woman, indicating her identity.", + "category": 1, + "retrieved_memories": [ + { + "id": "c83ebdb6-822c-4208-8da4-6ffbfdd3fd46", + "text": "Caroline created a painting intended to represent her journey as a trans woman; the red and blue colors symbolize the binary gender system, their mixture signifies breaking that rigidity, and the artwork serves as a reminder to love her authentic self after a long process of self-acceptance.", + "context": "Conversation session between Caroline and Melanie", + "event_date": "2023-09-13T00:09:00+00:00", + "weight": 0.554857861963098, + "activation": 0.674763082010196, + "semantic_similarity": 0.6747631245334641, + "recency": 1.515269859495297e-34, + "frequency": 2.0 + }, + { + "id": "79d809b7-ed0b-48ee-80ed-932bc2b61e87", + "text": "Caroline's artwork focuses on expressing her trans experience, aiming to tell her personal story and help others understand the trans community, as of 2023-08-14.", + "context": "Conversation session between Caroline and Melanie", + "event_date": "2023-08-14T14:24:00+00:00", + "weight": 0.5542572501175975, + "activation": 0.673762062339537, + "semantic_similarity": 0.6737621047191215, + "recency": 8.005579483282672e-36, + "frequency": 2.0 + }, + { + "id": "1e910def-a7cb-4593-be18-99c833176fa3", + "text": "Caroline's gender transition and artistic expression have altered her relationships: some close friends continue to support her, while a few could not handle the changes; overall she feels happier and her relationships now feel more genuine.", + "context": "Conversation session between Caroline and Melanie", + "event_date": "2023-09-13T00:09:00+00:00", + "weight": 0.5508956355256378, + "activation": 0.668159286003198, + "semantic_similarity": 0.6681594990822615, + "recency": 1.515269859121716e-34, + "frequency": 2.0 + }, + { + "id": "a8b8baac-87de-4f41-b0c5-b32f2870f1d6", + "text": "Caroline is inspired by her work that makes a difference for the LGBTQ+ community, believing that helping create a more loving world is amazing, and this inspiration motivates her to keep making art.", + "context": "Conversation session between Caroline and Melanie", + "event_date": "2023-09-13T00:09:00+00:00", + "weight": 0.5072196185015274, + "activation": 0.5398104656081568, + "semantic_similarity": 0.6509215960636011, + "recency": 1.5152683093644787e-34, + "frequency": 2.0 + }, + { + "id": "ed205e95-5f5f-4fa3-a5de-ae15db6e08ca", + "text": "Caroline has been creating art since she was about 17 years old, finding the practice empowering and cathartic.", + "context": "Conversation session between Caroline and Melanie", + "event_date": "2023-09-13T00:09:00+00:00", + "weight": 0.5054569679485338, + "activation": 0.5398104656081568, + "semantic_similarity": 0.6450460942202894, + "recency": 1.5152683094083203e-34, + "frequency": 2.0 + }, + { + "id": "d4afd073-6eda-41ec-b138-8fc46c23bc23", + "text": "Caroline has not yet tried pottery but is interested in trying new art forms and may try pottery sometime in the future.", + "context": "Conversation session between Caroline and Melanie", + "event_date": "2023-09-13T00:09:00+00:00", + "weight": 0.5008132128945666, + "activation": 0.5398104656081568, + "semantic_similarity": 0.629566910707065, + "recency": 1.5152683098906222e-34, + "frequency": 2.0 + }, + { + "id": "2ddf41e8-b729-4ec4-9371-1c1787bad2b8", + "text": "Caroline creates inclusive and diverse artwork, using it as a platform to advocate for the LGBTQ+ community and promote acceptance; she recently produced a painting (unspecified) as of 2023-08-14.", + "context": "Conversation session between Caroline and Melanie", + "event_date": "2023-08-14T14:24:00+00:00", + "weight": 0.5006426751759953, + "activation": 0.5390096498716296, + "semantic_similarity": 0.6297992673816878, + "recency": 8.00557129215223e-36, + "frequency": 2.0 + }, + { + "id": "3ac1218b-6a5e-46e7-b7cf-47af5ec87d96", + "text": "Caroline created a self-portrait painting last week (the week of 2023-08-14 to 2023-08-20), which she posted in the conversation.", + "context": "Conversation session between Caroline and Melanie", + "event_date": "2023-08-16T15:31:00+00:00", + "weight": 0.49407176384397755, + "activation": 0.5390096498716296, + "semantic_similarity": 0.6078962296082956, + "recency": 9.823627847539195e-36, + "frequency": 2.0 + }, + { + "id": "f3826dfa-cd0a-460c-a3d3-b79f67176ce2", + "text": "Caroline is planning to host an LGBTQ art show featuring her paintings in August 2023, specifically scheduled for August 17, 2023.", + "context": "Conversation session between Caroline and Melanie", + "event_date": "2023-08-17T14:31:00+00:00", + "weight": 0.48931827929138116, + "activation": 0.5390096498716296, + "semantic_similarity": 0.5920512810996408, + "recency": 1.0811645302414841e-35, + "frequency": 2.0 + }, + { + "id": "60fa2ddb-d241-4067-910e-ad8b864ac31b", + "text": "Melanie said that Caroline has always been there for her, and she appreciates their friendship.", + "context": "Conversation session between Caroline and Melanie", + "event_date": "2023-08-17T13:50:00+00:00", + "weight": 0.48302552657502695, + "activation": 0.5390096498716296, + "semantic_similarity": 0.5710754387117937, + "recency": 1.0780905926085173e-35, + "frequency": 2.0 + }, + { + "id": "88ee8470-c94d-4821-9614-bff8ee8d6433", + "text": "Caroline attended a pride parade on 2023-08-11, describing it as full of energy and love, which made her feel proud and reinforced her belief in the importance of standing up for equality.", + "context": "Conversation session between Caroline and Melanie", + "event_date": "2023-08-11T14:24:00+00:00", + "weight": 0.4782470974366435, + "activation": 0.5390096498716296, + "semantic_similarity": 0.5551473415838489, + "recency": 5.930673079856595e-36, + "frequency": 2.0 + }, + { + "id": "eab9d6a2-f455-4c10-84e5-fcc6f4c87adc", + "text": "Following her beach visit on August 18, 2023, Caroline painted a sunset scene to capture the calming feeling she experienced.", + "context": "Conversation session between Caroline and Melanie", + "event_date": "2023-08-18T13:33:00+00:00", + "weight": 0.47668413320904524, + "activation": 0.5390096498716296, + "semantic_similarity": 0.5499374608251877, + "recency": 1.1900685984491014e-35, + "frequency": 2.0 + }, + { + "id": "cd1a7636-f1c8-4f36-bd1b-d7b4bd7ea84b", + "text": "Caroline had a not-so-great experience on a hike where she ran into a group of religious conservatives who said something that upset her, which made her think about how much work still needs to be done for LGBTQ rights.", + "context": "Conversation session between Caroline and Melanie", + "event_date": "2023-08-17T13:50:00+00:00", + "weight": 0.47655911482930946, + "activation": 0.5390096498716296, + "semantic_similarity": 0.5495207328927352, + "recency": 1.0780905926734e-35, + "frequency": 2.0 + }, + { + "id": "43224c50-a69d-4d79-bda5-98253119a520", + "text": "Caroline expressed appreciation for her friendship with Melanie.", + "context": "Conversation session between Caroline and Melanie", + "event_date": "2023-08-17T13:50:00+00:00", + "weight": 0.4761944562133029, + "activation": 0.43184837248652547, + "semantic_similarity": 0.6554664815578175, + "recency": 1.0780863881307891e-35, + "frequency": 2.0 + }, + { + "id": "6c731f26-dd5c-4536-b897-8054255fe1b4", + "text": "Caroline produced a painting titled \"Embracing Identity\" that depicts a woman symbolizing the journey of acceptance, with the aim of conveying warmth, love, and self-acceptance; this work was referenced on 2023-08-14.", + "context": "Conversation session between Caroline and Melanie", + "event_date": "2023-08-14T14:24:00+00:00", + "weight": 0.4761134436611424, + "activation": 0.427593002099958, + "semantic_similarity": 0.6594518101038499, + "recency": 8.005571294663159e-36, + "frequency": 2.0 + }, + { + "id": "45907243-354f-45f5-9bd6-61199272b94e", + "text": "Caroline is organizing an LGBTQ art show scheduled for September 2023 (specifically around September 25, 2023) to exhibit her paintings and promote understanding and acceptance within the community.", + "context": "Conversation session between Caroline and Melanie", + "event_date": "2023-09-25T13:33:00+00:00", + "weight": 0.4745712211164256, + "activation": 0.4494030141151468, + "semantic_similarity": 0.6325010562729384, + "recency": 5.319747596682734e-34, + "frequency": 2.0 + }, + { + "id": "ca9d8d15-1560-4823-936d-79be60dfb4f9", + "text": "Melanie has been involved in art for seven years and has recently identified painting and pottery as her primary muses.", + "context": "Conversation session between Caroline and Melanie", + "event_date": "2023-09-13T00:09:00+00:00", + "weight": 0.47390486518457087, + "activation": 0.5398104656081568, + "semantic_similarity": 0.5398724183404129, + "recency": 1.515268309536357e-34, + "frequency": 2.0 + }, + { + "id": "48495b16-a159-4d5e-bd2b-1db64489b0fb", + "text": "Caroline visited a beach on August 18, 2023, watched the sun set below the horizon, and was inspired by the vivid colors.", + "context": "Conversation session between Caroline and Melanie", + "event_date": "2023-08-18T13:33:00+00:00", + "weight": 0.4709709878254469, + "activation": 0.5390096498716296, + "semantic_similarity": 0.5308936428798601, + "recency": 1.190068598484887e-35, + "frequency": 2.0 + }, + { + "id": "72855141-35e9-47cd-92aa-5d1ffb89ae3a", + "text": "The LGBTQ support group made Caroline feel accepted and gave her courage to embrace herself.", + "context": "Conversation session between Caroline and Melanie", + "event_date": "2023-05-08T13:56:00+00:00", + "weight": 0.4706860446327654, + "activation": 0.4206203135012283, + "semantic_similarity": 0.6483331686079898, + "recency": 4.430593898215753e-40, + "frequency": 2.0 + }, + { + "id": "50dd73e8-e477-4c45-b052-b3eae96a1a6a", + "text": "Melanie expressed pride in Caroline's work and said she is proud to be part of the difference Caroline is making.", + "context": "Conversation session between Caroline and Melanie", + "event_date": "2023-06-09T19:55:00+00:00", + "weight": 0.46958491726488694, + "activation": 0.43184837248652547, + "semantic_similarity": 0.6334346850630976, + "recency": 1.1143672569065206e-38, + "frequency": 2.0 + } + ], + "is_correct": true, + "correctness_reasoning": "The predicted answer states that Caroline is a trans woman, which is equivalent to the correct answer 'Transgender woman'." + }, + { + "question": "What did Caroline research?", + "correct_answer": "Adoption agencies", + "predicted_answer": "Caroline researched career options in counseling and mental health, as well as adoption agencies.", + "reasoning": "The context explicitly states that Caroline is researching a career in counseling and mental health (items 0, 1, 6, 12, 16) and also mentions her research into adoption agencies (item 2). Therefore, both areas of research are clearly indicated in the provided context.", + "category": 1, + "retrieved_memories": [ + { + "id": "eeb4a464-dc3f-4123-a16e-bfa95dedf0ea", + "text": "Caroline has recently been researching a career in counseling and mental health because she wants to help people who have experienced similar challenges to her own.", + "context": "Conversation session between Caroline and Melanie", + "event_date": "2023-06-27T10:37:00+00:00", + "weight": 0.5794384677544774, + "activation": 0.715730743662682, + "semantic_similarity": 0.7157308155189092, + "recency": 6.485732412751227e-38, + "frequency": 2.0 + }, + { + "id": "c759737b-8fff-4941-8c74-c832339e7595", + "text": "Caroline is actively researching and considering career options in counseling and mental health because she believes it is important for people to have someone to talk to and wants to help make that happen.", + "context": "Conversation session between Caroline and Melanie", + "event_date": "2023-07-12T16:33:00+00:00", + "weight": 0.5696282642038295, + "activation": 0.699380439482955, + "semantic_similarity": 0.6993804411964767, + "recency": 2.9794594154744854e-37, + "frequency": 2.0 + }, + { + "id": "0188d9d9-2ecb-4339-b41f-071e91baba40", + "text": "Caroline is researching adoption agencies because she has long dreamed of having a family and giving a loving home to children who need one.", + "context": "Conversation session between Caroline and Melanie", + "event_date": "2023-05-25T13:14:00+00:00", + "weight": 0.5632356338407093, + "activation": 0.688726092013202, + "semantic_similarity": 0.6887260207891623, + "recency": 2.4183672474861377e-39, + "frequency": 2.0 + }, + { + "id": "bf146ef5-5001-4773-af33-8ab30c2b167f", + "text": "Caroline plans to specialize in working with transgender individuals, aiming to help them accept themselves and support their mental health.", + "context": "Conversation session between Caroline and Melanie", + "event_date": "2023-06-27T10:37:00+00:00", + "weight": 0.5072137320716724, + "activation": 0.5725845949301456, + "semantic_similarity": 0.6181278453087624, + "recency": 6.485724067976057e-38, + "frequency": 2.0 + }, + { + "id": "1a905e0b-4767-4a9e-b3ad-bc32d0657ab2", + "text": "Caroline says that her own personal journey and the support she received significantly improved her life; observing the benefits of counseling and support groups motivated her to care more about mental health, understand herself better, and create a safe, inviting environment for others to grow.", + "context": "Conversation session between Caroline and Melanie", + "event_date": "2023-06-27T10:37:00+00:00", + "weight": 0.5017927726791095, + "activation": 0.5725845949301456, + "semantic_similarity": 0.6000579806668861, + "recency": 6.485724067765822e-38, + "frequency": 2.0 + }, + { + "id": "83e62a1b-74b9-40e2-b6d8-e4d9dab557a6", + "text": "Caroline received a necklace as a gift from her grandmother in Sweden when she was young; the necklace symbolizes love, faith, and strength and serves as a reminder of her roots and the love and support she receives from her family.", + "context": "Conversation session between Caroline and Melanie", + "event_date": "2023-06-27T10:37:00+00:00", + "weight": 0.5008315705800984, + "activation": 0.5725845949301456, + "semantic_similarity": 0.5968539736701824, + "recency": 6.485724068959394e-38, + "frequency": 2.0 + }, + { + "id": "280b9bd7-1ed9-4a8a-b169-89af4775ae9b", + "text": "Caroline is keen on becoming a counselor or working in mental health so she can support people who face similar issues.", + "context": "Conversation session between Caroline and Melanie", + "event_date": "2023-05-08T13:56:00+00:00", + "weight": 0.4992268594298619, + "activation": 0.5340787715851291, + "semantic_similarity": 0.630010759847744, + "recency": 4.430857334771276e-40, + "frequency": 2.0 + }, + { + "id": "73850c77-935b-45c3-9a68-3552e0f85121", + "text": "Caroline emphasizes that mental health is a priority and advises Melanie to take care of herself.", + "context": "Conversation session between Caroline and Melanie", + "event_date": "2023-07-12T16:33:00+00:00", + "weight": 0.49883968642847987, + "activation": 0.559504351586364, + "semantic_similarity": 0.6032946031752354, + "recency": 2.9794555800195424e-37, + "frequency": 2.0 + }, + { + "id": "a10ae2d0-45de-4fbb-8c37-9762142b5158", + "text": "As of July 17, 2023, Caroline is mentoring a transgender teen who shares her own gender identity, and together they are working on building confidence and developing positive coping strategies.", + "context": "Conversation session between Caroline and Melanie", + "event_date": "2023-07-17T14:31:00+00:00", + "weight": 0.49703533128847976, + "activation": 0.559504351586364, + "semantic_similarity": 0.5972800860419019, + "recency": 4.87084956709961e-37, + "frequency": 2.0 + }, + { + "id": "a4f20608-0fb6-4235-b47b-b64482d3e648", + "text": "Caroline stated her goal is to give kids a loving home, expressed gratitude for support from friends and mentors, and said she feels hopeful and optimistic about turning her adoption dream into reality.", + "context": "Conversation session between Caroline and Melanie", + "event_date": "2023-05-25T13:14:00+00:00", + "weight": 0.49699247986583384, + "activation": 0.5509808736105616, + "semantic_similarity": 0.6056607259422179, + "recency": 2.4183641336192322e-39, + "frequency": 2.0 + }, + { + "id": "05f1dcaa-6f99-46b5-87e9-d2efa64e01cd", + "text": "Caroline joined a mentorship program for LGBTQ youth on the weekend of July 15\u201316, 2023, aiming to support and empower young members of the community.", + "context": "Conversation session between Caroline and Melanie", + "event_date": "2023-07-15T14:31:00+00:00", + "weight": 0.494513164550792, + "activation": 0.559504351586364, + "semantic_similarity": 0.5888728635829426, + "recency": 3.9879143305085155e-37, + "frequency": 2.0 + }, + { + "id": "d8b3fb45-7e02-4aad-8cd9-79e0a7cc1a76", + "text": "Caroline chose a specific adoption agency because it assists LGBTQ+ individuals with adoption, and she was attracted to its inclusivity and support.", + "context": "Conversation session between Caroline and Melanie", + "event_date": "2023-05-25T13:14:00+00:00", + "weight": 0.4941924669756822, + "activation": 0.5509808736105616, + "semantic_similarity": 0.5963273496417124, + "recency": 2.41836413352407e-39, + "frequency": 2.0 + }, + { + "id": "53358350-28d3-4192-ba20-b7259eeebee8", + "text": "Caroline previously struggled with mental health, received helpful support, which made her realize the importance of having a support system for others, and consequently started looking into counseling and mental health career options to assist others on their journeys.", + "context": "Conversation session between Caroline and Melanie", + "event_date": "2023-07-12T16:33:00+00:00", + "weight": 0.4939609162862364, + "activation": 0.4934357091231964, + "semantic_similarity": 0.6531006784975917, + "recency": 2.9794555825023955e-37, + "frequency": 2.0 + }, + { + "id": "60b8d54d-4c1f-477b-904e-38ab8949bf06", + "text": "Caroline expressed excitement about creating a family for children in need, acknowledged that it will be challenging as a single parent, but affirmed she is ready for the challenge.", + "context": "Conversation session between Caroline and Melanie", + "event_date": "2023-05-25T13:14:00+00:00", + "weight": 0.49364870295561747, + "activation": 0.5509808736105616, + "semantic_similarity": 0.5945148029081634, + "recency": 2.41836413346245e-39, + "frequency": 2.0 + }, + { + "id": "006df526-f850-48a9-a2f3-a13799f0391c", + "text": "Melanie states that the conversation with Caroline has been great for her mental health and she intends to continue the positive practices.", + "context": "Conversation session between Caroline and Melanie", + "event_date": "2023-07-12T16:33:00+00:00", + "weight": 0.48826275503183203, + "activation": 0.559504351586364, + "semantic_similarity": 0.5680381651864093, + "recency": 2.979455580557522e-37, + "frequency": 2.0 + }, + { + "id": "ddf46000-7d88-44b4-b9f7-ba012537e710", + "text": "Caroline loves the book \"Becoming Nicole\" by Amy Ellis Nutt, a true story about a trans girl and her family, which made her feel connected, gave her hope for her own path, and she highly recommends it.", + "context": "Conversation session between Caroline and Melanie", + "event_date": "2023-07-12T16:33:00+00:00", + "weight": 0.48645685184642806, + "activation": 0.559504351586364, + "semantic_similarity": 0.5620184879017295, + "recency": 2.9794555818471747e-37, + "frequency": 2.0 + }, + { + "id": "5e2c78a6-b363-44f7-a74b-23b7b61980f9", + "text": "Caroline has been looking into counseling or mental health work since her last conversation with Melanie, expressing passion for helping people, describing the work as tough but rewarding.", + "context": "Conversation session between Caroline and Melanie", + "event_date": "2023-07-06T20:18:00+00:00", + "weight": 0.48609639250226766, + "activation": 0.49115071972664925, + "semantic_similarity": 0.629170588614243, + "recency": 1.6609099143444882e-37, + "frequency": 2.0 + }, + { + "id": "55aa9ea8-0635-4680-9ed1-243b08e8d5fc", + "text": "Caroline's goal is to have a family; she hopes to build her own family by adopting children and providing them a home, viewing adoption as a way of giving back and showing love and acceptance.", + "context": "Conversation session between Caroline and Melanie", + "event_date": "2023-10-22T09:55:00+00:00", + "weight": 0.485850455679414, + "activation": 0.48001741392383207, + "semantic_similarity": 0.6394841050075479, + "recency": 7.797173772342497e-33, + "frequency": 2.0 + }, + { + "id": "5819027e-8dda-4d6c-aecf-696e998337c2", + "text": "Caroline plans to continue her education and explore career options, expressing excitement about the prospect.", + "context": "Conversation session between Caroline and Melanie", + "event_date": "2023-05-08T13:56:00+00:00", + "weight": 0.48272074488201033, + "activation": 0.4272630172681033, + "semantic_similarity": 0.6818061323385977, + "recency": 4.4308439583049156e-40, + "frequency": 2.0 + }, + { + "id": "4c301737-73f0-4944-a209-32acae5d4478", + "text": "Caroline says the book taught her self-acceptance, how to find support, that tough times don't last, and that hope and love exist; she also notes that pets bring a lot of joy.", + "context": "Conversation session between Caroline and Melanie", + "event_date": "2023-07-12T16:33:00+00:00", + "weight": 0.48229796156010274, + "activation": 0.559504351586364, + "semantic_similarity": 0.5481555202806452, + "recency": 2.9794555817644835e-37, + "frequency": 2.0 + } + ], + "is_correct": true, + "correctness_reasoning": "The predicted answer includes \"adoption agencies,\" which matches the correct answer. Although it adds extra information about career options in counseling and mental health, it still contains the required answer, so it is considered equivalent." + }, + { + "question": "When did Caroline meet up with her friends, family, and mentors?", + "correct_answer": "The week before 9 June 2023", + "predicted_answer": "I don't know", + "reasoning": "The context does not provide specific dates or instances of Caroline meeting up with her friends, family, and mentors. It mentions her gratitude for their support and her experiences with them, but it does not specify when these meetings occurred.", + "category": 2, + "retrieved_memories": [ + { + "id": "0b5694e4-7c0b-422c-bed4-588d92fb067f", + "text": "Caroline has known her current friends for four years, having moved from her home country around 2019-06-09.", + "context": "Conversation session between Caroline and Melanie", + "event_date": "2023-06-09T19:55:00+00:00", + "weight": 0.6010526862460175, + "activation": 0.751754485122993, + "semantic_similarity": 0.7517544690303984, + "recency": 1.1144318304372272e-38, + "frequency": 2.0 + }, + { + "id": "d8fcc8ca-5510-49a0-bab6-667aabb04d42", + "text": "Caroline contacted her mentor for adoption advice.", + "context": "Conversation session between Caroline and Melanie", + "event_date": "2023-10-13T10:31:00+00:00", + "weight": 0.5908183021046474, + "activation": 0.734697185000694, + "semantic_similarity": 0.7346971553481304, + "recency": 3.178001608290096e-33, + "frequency": 2.0 + }, + { + "id": "5f5ec14c-8715-4770-988a-6aa1fe4176f0", + "text": "Caroline attended an LGBTQ support group on 2023-05-07, finding the experience powerful.", + "context": "Conversation session between Caroline and Melanie", + "event_date": "2023-05-07T13:56:00+00:00", + "weight": 0.5846830075977348, + "activation": 0.724471688270569, + "semantic_similarity": 0.7244716703885473, + "recency": 4.009170396332213e-40, + "frequency": 2.0 + }, + { + "id": "ad619ec1-c081-4fab-bb8f-707d8eed6fc0", + "text": "Caroline mentioned a tough breakup in the past and expressed thankfulness for her friends, family, and mentors who have supported her.", + "context": "Conversation session between Caroline and Melanie", + "event_date": "2023-06-09T19:55:00+00:00", + "weight": 0.5438181993793406, + "activation": 0.6014035880983944, + "semantic_similarity": 0.7113237431660742, + "recency": 1.1144300572150759e-38, + "frequency": 2.0 + }, + { + "id": "ec88eee3-ba37-427f-9bf4-e0995d5fc61a", + "text": "On Friday, June 23, 2023, Caroline attended an LGBTQ+ counseling workshop where professionals discussed various therapeutic methods for working with transgender people and demonstrated passion for creating safe spaces for individuals like her.", + "context": "Conversation session between Caroline and Melanie", + "event_date": "2023-06-23T10:37:00+00:00", + "weight": 0.5317243625786355, + "activation": 0.5795773506164552, + "semantic_similarity": 0.6928371913123298, + "recency": 4.3474658545707335e-38, + "frequency": 2.0 + }, + { + "id": "c5161f91-693c-48a1-a288-ad149f8fc765", + "text": "Caroline attended an LGBTQ conference on 2023-07-10, two days before the reference date, where she met and connected with people who have experienced similar journeys, found the environment welcoming, felt totally accepted, and expressed gratitude for the community.", + "context": "Conversation session between Caroline and Melanie", + "event_date": "2023-07-10T16:33:00+00:00", + "weight": 0.5304923054101583, + "activation": 0.5795773506164552, + "semantic_similarity": 0.6887303340840725, + "recency": 2.439346662855671e-37, + "frequency": 2.0 + }, + { + "id": "5c2765c4-57b3-4606-a52c-53a87daf2b2f", + "text": "Caroline expressed that she is ready to become a mother and share her love and family.", + "context": "Conversation session between Caroline and Melanie", + "event_date": "2023-10-13T10:31:00+00:00", + "weight": 0.5223467720549109, + "activation": 0.5877577480005552, + "semantic_similarity": 0.6533981588491475, + "recency": 3.1779965507657496e-33, + "frequency": 2.0 + }, + { + "id": "3f797cf0-1c31-4f25-8815-0791293e3bcd", + "text": "During her transition and journey toward self-acceptance, Caroline received invaluable help and encouragement from friends, family, and people she looked up to, which boosted her through tough times and helped her discover her true self.", + "context": "Conversation session between Caroline and Melanie", + "event_date": "2023-10-22T09:55:00+00:00", + "weight": 0.5159861957616187, + "activation": 0.5795773506164552, + "semantic_similarity": 0.6403766352556072, + "recency": 7.797093076866785e-33, + "frequency": 2.0 + }, + { + "id": "05f1dcaa-6f99-46b5-87e9-d2efa64e01cd", + "text": "Caroline joined a mentorship program for LGBTQ youth on the weekend of July 15\u201316, 2023, aiming to support and empower young members of the community.", + "context": "Conversation session between Caroline and Melanie", + "event_date": "2023-07-15T14:31:00+00:00", + "weight": 0.5112932706091949, + "activation": 0.4996917704239485, + "semantic_similarity": 0.7046191316067013, + "recency": 3.987873055010309e-37, + "frequency": 2.0 + }, + { + "id": "aa36595e-58ca-42dc-b50d-8c785526bbf3", + "text": "Melanie's support has been meaningful to Caroline; Caroline feels grateful for Melanie's support throughout her journey and values being able to share and help others.", + "context": "Conversation session between Caroline and Melanie", + "event_date": "2023-10-22T09:55:00+00:00", + "weight": 0.5109544632213276, + "activation": 0.5795773506164552, + "semantic_similarity": 0.6236041934546369, + "recency": 7.797093076623018e-33, + "frequency": 2.0 + }, + { + "id": "88e28f0b-51b1-45bb-860b-4b1a734d3462", + "text": "Caroline reflected that volunteering reminded her of her own past struggles and feeling alone, and she was glad to share her story and offer support, feeling she could make a difference.", + "context": "Conversation session between Caroline and Melanie", + "event_date": "2023-08-28T15:19:00+00:00", + "weight": 0.5108315798564431, + "activation": 0.5795773506164552, + "semantic_similarity": 0.6231945822383552, + "recency": 3.2590025028256203e-35, + "frequency": 2.0 + }, + { + "id": "a4f20608-0fb6-4235-b47b-b64482d3e648", + "text": "Caroline stated her goal is to give kids a loving home, expressed gratitude for support from friends and mentors, and said she feels hopeful and optimistic about turning her adoption dream into reality.", + "context": "Conversation session between Caroline and Melanie", + "event_date": "2023-05-25T13:14:00+00:00", + "weight": 0.5103946960187441, + "activation": 0.5795773506164552, + "semantic_similarity": 0.6217383027793583, + "recency": 2.418339104239752e-39, + "frequency": 2.0 + }, + { + "id": "63dd0bce-9e94-439c-a66b-2348579d05dc", + "text": "Caroline attended an LGBTQ+ pride parade, observed a happy crowd, felt a sense of belonging, and recognized significant growth in the LGBTQ+ community.", + "context": "Conversation session between Caroline and Melanie", + "event_date": "2023-06-26T13:36:00+00:00", + "weight": 0.5084066605467776, + "activation": 0.5795773506164552, + "semantic_similarity": 0.6151115178728037, + "recency": 5.941868633581348e-38, + "frequency": 2.0 + }, + { + "id": "43224c50-a69d-4d79-bda5-98253119a520", + "text": "Caroline expressed appreciation for her friendship with Melanie.", + "context": "Conversation session between Caroline and Melanie", + "event_date": "2023-08-17T13:50:00+00:00", + "weight": 0.5072750954977371, + "activation": 0.4811228704787156, + "semantic_similarity": 0.7097941145137414, + "recency": 1.0781391322707423e-35, + "frequency": 2.0 + }, + { + "id": "c7ffecc3-a614-401d-9f62-a23753c7ca5f", + "text": "Melanie has been reading a book that Caroline recommended a while ago and has been painting to keep busy.", + "context": "Conversation session between Caroline and Melanie", + "event_date": "2023-10-13T10:31:00+00:00", + "weight": 0.5060902901101197, + "activation": 0.5877577480005552, + "semantic_similarity": 0.5992098856998439, + "recency": 3.177996550684864e-33, + "frequency": 2.0 + }, + { + "id": "453f811d-9535-42e7-8784-7a0eb35362ba", + "text": "During the June 17, 2023 LGBT pride event, Caroline recalls that the best moment was seeing her mentee's face light up upon witnessing the community's support.", + "context": "Conversation session between Caroline and Melanie", + "event_date": "2023-06-17T14:31:00+00:00", + "weight": 0.5060437535470546, + "activation": 0.5795773506164552, + "semantic_similarity": 0.6072351612070598, + "recency": 2.4250281037657853e-38, + "frequency": 2.0 + }, + { + "id": "1a905e0b-4767-4a9e-b3ad-bc32d0657ab2", + "text": "Caroline says that her own personal journey and the support she received significantly improved her life; observing the benefits of counseling and support groups motivated her to care more about mental health, understand herself better, and create a safe, inviting environment for others to grow.", + "context": "Conversation session between Caroline and Melanie", + "event_date": "2023-06-27T10:37:00+00:00", + "weight": 0.5058604333261678, + "activation": 0.5795773506164552, + "semantic_similarity": 0.6066240938041042, + "recency": 6.485656933843071e-38, + "frequency": 2.0 + }, + { + "id": "98c08619-7bb3-467c-a87e-027ee4b31021", + "text": "Both Caroline and Melanie affirmed their commitment to continue supporting each other, spreading love, acceptance, and hope, and to motivate each other through life's challenges.", + "context": "Conversation session between Caroline and Melanie", + "event_date": "2023-06-09T19:55:00+00:00", + "weight": 0.50464851264145, + "activation": 0.6014035880983944, + "semantic_similarity": 0.5807581207064392, + "recency": 1.1144300571454247e-38, + "frequency": 2.0 + }, + { + "id": "d20a1038-61b6-4baa-9d6a-7d3a729434af", + "text": "Caroline expressed gratitude for the love and support she has received throughout her transition and aims to give a voice to the trans community.", + "context": "Conversation session between Caroline and Melanie", + "event_date": "2023-06-09T19:55:00+00:00", + "weight": 0.5045089567986043, + "activation": 0.6014035880983944, + "semantic_similarity": 0.5802929345636201, + "recency": 1.1144300571764019e-38, + "frequency": 2.0 + }, + { + "id": "3a2f0dfc-4fc1-4986-9474-ce54481550a3", + "text": "Caroline joined a new LGBTQ activist group called Connected LGBTQ Activists on Tuesday, July 18, 2023.", + "context": "Conversation session between Caroline and Melanie", + "event_date": "2023-07-18T20:56:00+00:00", + "weight": 0.5030478276643547, + "activation": 0.48810711378264965, + "semantic_similarity": 0.6887189784318661, + "recency": 5.528929027864583e-37, + "frequency": 2.0 + } + ], + "is_correct": false, + "correctness_reasoning": "The predicted answer 'I don't know' does not provide the information that Caroline met up with her friends, family, and mentors the week before 9 June 2023, so it is not equivalent to the correct answer." } ] }, - "total_turns": 419 + "total_turns": -1 } ] } \ No newline at end of file diff --git a/benchmarks/locomo/run_benchmark.py b/benchmarks/locomo/run_benchmark.py index 2a4d2f9c..f883b971 100644 --- a/benchmarks/locomo/run_benchmark.py +++ b/benchmarks/locomo/run_benchmark.py @@ -13,10 +13,12 @@ import json from datetime import datetime, timezone, timedelta from memory import TemporalSemanticMemory from typing import List, Dict +from openai import AsyncOpenAI import openai from dotenv import load_dotenv import os import asyncio +import pydantic from rich.console import Console from rich.progress import Progress, SpinnerColumn, TextColumn, BarColumn from rich.table import Table @@ -27,6 +29,24 @@ load_dotenv() console = Console() +def get_groq_client() -> AsyncOpenAI: + """ + Get configured async Groq client for LLM judge. + + Returns: + Configured AsyncOpenAI client pointing to Groq + """ + groq_api_key = os.getenv('GROQ_API_KEY') + if not groq_api_key: + raise ValueError("GROQ_API_KEY environment variable not set") + + base_url = os.getenv('GROQ_BASE_URL', 'https://api.groq.com/openai/v1') + return AsyncOpenAI( + api_key=groq_api_key, + base_url=base_url + ) + + def parse_date(date_string: str) -> datetime: """Parse LoComo date format to datetime.""" # Format: "1:56 pm on 8 May, 2023" @@ -41,7 +61,7 @@ async def ingest_conversation(memory: TemporalSemanticMemory, conversation_data: """ Ingest a LoComo conversation into the memory system (ASYNC version). - Ingests entire conversation as a single large document for maximum efficiency. + Ingests ALL sessions in ONE batch for maximum efficiency. Args: memory: Memory system instance @@ -55,45 +75,55 @@ async def ingest_conversation(memory: TemporalSemanticMemory, conversation_data: # Get all session keys sorted session_keys = sorted([k for k in conv.keys() if k.startswith('session_') and not k.endswith('_date_time')]) + # Collect all sessions as batch items + batch_contents = [] total_turns = 0 - # Build entire conversation as one large text - conversation_parts = [] - for session_key in session_keys: if session_key not in conv or not isinstance(conv[session_key], list): continue session_data = conv[session_key] - # Add all turns from this session + # Build session content from all turns + session_parts = [] for turn in session_data: speaker = turn['speaker'] text = turn['text'] - conversation_parts.append(f"{speaker} said: {text}") + session_parts.append(f"{speaker}: {text}") total_turns += 1 - # Ingest entire conversation in ONE put_async call - # Use the first session date as the event date - first_session_key = session_keys[0] if session_keys else "session_1" - date_key = f"{first_session_key}_date_time" - conversation_date = parse_date(conv.get(date_key, "1:00 pm on 1 January, 2023")) + if not session_parts: + continue - full_conversation = " ".join(conversation_parts) + # Get session date + date_key = f"{session_key}_date_time" + session_date = parse_date(conv.get(date_key, "1:00 pm on 1 January, 2023")) - await memory.put_async( - agent_id=agent_id, - content=full_conversation, - context=f"Full conversation between {speaker_a} and {speaker_b}", - event_date=conversation_date - ) + # Add to batch + session_content = "\n".join(session_parts) + batch_contents.append({ + "content": session_content, + "context": f"Conversation session between {speaker_a} and {speaker_b}", + "event_date": session_date + }) + + # Ingest ALL sessions in ONE batch call (MUCH faster!) + if batch_contents: + await memory.put_batch_async( + agent_id=agent_id, + contents=batch_contents + ) return total_turns +class QuestionAnswer(pydantic.BaseModel): + answer: str + reasoning: str -def answer_question(memory: TemporalSemanticMemory, agent_id: str, question: str, thinking_budget: int = 100) -> str: +async def answer_question(memory: TemporalSemanticMemory, agent_id: str, question: str, thinking_budget: int = 500) -> tuple[str, str, List[Dict]]: """ - Answer a question using the memory system. + Answer a question using the memory system (ASYNC version). Args: memory: Memory system instance @@ -102,36 +132,33 @@ def answer_question(memory: TemporalSemanticMemory, agent_id: str, question: str thinking_budget: How many memory units to explore Returns: - Answer string + Tuple of (answer string, reasoning string, retrieved memories list) """ # Search memory - results = memory.search( + results = await memory.search_async( agent_id=agent_id, query=question, thinking_budget=thinking_budget, top_k=20 # Get more results for better context ) - print("question:", question) - print("Got results:", results) - if not results: - return "I don't have enough information to answer that question." + return "I don't have enough information to answer that question.", "No relevant memories found.", [] - # Build context from top results context_parts = [] - for i, result in enumerate(results[:10], 1): + for i, result in enumerate(results): context_parts.append(f"{i}. {result['text']}") context = "\n".join(context_parts) - # Use OpenAI to generate answer from context + # Use AsyncOpenAI to generate answer from context try: - response = openai.chat.completions.create( + client = AsyncOpenAI() + response = await client.beta.chat.completions.parse( model="gpt-4o-mini", messages=[ { "role": "system", - "content": "You are a helpful assistant. Answer the question based ONLY on the provided context. If the context doesn't contain the answer, say 'I don't know'." + "content": "You are a helpful assistant. Answer the question based ONLY on the provided context. If the context doesn't contain the answer, say 'I don't know'. In the reasoning, explain why you choose or not choose the context items for the answer." }, { "role": "user", @@ -139,14 +166,16 @@ def answer_question(memory: TemporalSemanticMemory, agent_id: str, question: str } ], temperature=0, - max_tokens=150 + max_tokens=8000, + response_format=QuestionAnswer ) - return response.choices[0].message.content.strip() + answer = response.choices[0].message.parsed + return answer.answer, answer.reasoning, results except Exception as e: - return f"Error generating answer: {str(e)}" + return f"Error generating answer: {str(e)}", "Error occurred during answer generation.", results -def evaluate_qa_task( +async def evaluate_qa_task( memory: TemporalSemanticMemory, agent_id: str, qa_pairs: List[Dict], @@ -154,13 +183,11 @@ def evaluate_qa_task( max_questions: int = None ) -> Dict: """ - Evaluate the QA task. + Evaluate the QA task (ASYNC version - processes questions in parallel). Returns: Dict with evaluation metrics """ - results = [] - questions_to_eval = qa_pairs[:max_questions] if max_questions else qa_pairs with Progress( @@ -170,38 +197,97 @@ def evaluate_qa_task( TextColumn("[progress.percentage]{task.percentage:>3.0f}%"), console=console ) as progress: - task = progress.add_task(f"[cyan]Evaluating QA for sample {sample_id}...", total=len(questions_to_eval)) + task = progress.add_task(f"[cyan]Evaluating QA for sample {sample_id} (parallel)...", total=len(questions_to_eval)) - for qa in questions_to_eval: + # Create tasks for all questions + async def process_question(qa): question = qa['question'] correct_answer = qa['answer'] category = qa.get('category', 0) - # Get predicted answer - predicted_answer = answer_question(memory, agent_id, question) + # Get predicted answer, reasoning, and retrieved memories + predicted_answer, reasoning, retrieved_memories = await answer_question(memory, agent_id, question) - results.append({ + return { 'question': question, 'correct_answer': correct_answer, 'predicted_answer': predicted_answer, - 'category': category - }) + 'reasoning': reasoning, + 'category': category, + 'retrieved_memories': retrieved_memories + } + # Process all questions in parallel + question_tasks = [process_question(qa) for qa in questions_to_eval] + + # Use as_completed to update progress as results come in + results = [] + for coro in asyncio.as_completed(question_tasks): + result = await coro + results.append(result) progress.update(task, advance=1) return results +class JudgeResponse(pydantic.BaseModel): + correct: bool + reasoning: str -def calculate_metrics(results: List[Dict]) -> Dict: +async def judge_single_answer(client: AsyncOpenAI, result: Dict, semaphore: asyncio.Semaphore) -> Dict: """ - Calculate evaluation metrics. + Judge a single answer using LLM (with concurrency control). - Uses LLM-as-judge to evaluate answer quality. + Args: + client: Async OpenAI client (Groq) + result: Result dict with question, correct_answer, predicted_answer, category + semaphore: Semaphore to limit concurrent requests + + Returns: + Updated result dict with is_correct field + """ + async with semaphore: + try: + response = await client.beta.chat.completions.parse( + model="openai/gpt-oss-120b", + messages=[ + { + "role": "system", + "content": + "You are an objective judge. Determine if the predicted answer contains the correct answer or they are the same content (with different form is fine)." + }, + { + "role": "user", + "content": f"Question: {result['question']}\nCorrect answer: {result['correct_answer']}\nPredicted answer: {result['predicted_answer']}\n\nAre they equivalent?" + } + ], + temperature=0, + max_tokens=512, + response_format=JudgeResponse + + ) + + judgement = response.choices[0].message.parsed + result['is_correct'] = judgement.correct + result['correctness_reasoning'] = judgement.reasoning + + except Exception as e: + console.print(f"[red]Error judging answer: {e}[/red]") + result['is_correct'] = False + + return result + + +async def calculate_metrics(results: List[Dict]) -> Dict: + """ + Calculate evaluation metrics using parallel LLM-as-judge. + + Processes up to 8 judgments concurrently for speed. """ - correct = 0 total = len(results) + client = get_groq_client() - category_stats = {} + # Semaphore to limit to 8 concurrent requests + semaphore = asyncio.Semaphore(8) with Progress( SpinnerColumn(), @@ -210,49 +296,33 @@ def calculate_metrics(results: List[Dict]) -> Dict: TextColumn("[progress.percentage]{task.percentage:>3.0f}%"), console=console ) as progress: - task = progress.add_task("[yellow]Judging answers with LLM...", total=total) + task = progress.add_task("[yellow]Judging answers with LLM (parallel, max 8)...", total=total) + # Create all judgment tasks + judgment_tasks = [] for result in results: - # Use LLM as judge - try: - response = openai.chat.completions.create( - model="gpt-4o-mini", - messages=[ - { - "role": "system", - "content": "You are an objective judge. Determine if the predicted answer is semantically equivalent to the correct answer. Answer with ONLY 'yes' or 'no'." - }, - { - "role": "user", - "content": f"Question: {result['question']}\nCorrect answer: {result['correct_answer']}\nPredicted answer: {result['predicted_answer']}\n\nAre they equivalent?" - } - ], - temperature=0, - max_tokens=5 - ) - - judgment = response.choices[0].message.content.strip().lower() - is_correct = 'yes' in judgment - - if is_correct: - correct += 1 - - result['is_correct'] = is_correct - - # Track by category - category = result['category'] - if category not in category_stats: - category_stats[category] = {'correct': 0, 'total': 0} - category_stats[category]['total'] += 1 - if is_correct: - category_stats[category]['correct'] += 1 - - except Exception as e: - console.print(f"[red]Error judging answer: {e}[/red]") - result['is_correct'] = False + judgment_task = judge_single_answer(client, result, semaphore) + judgment_tasks.append(judgment_task) + # Process in parallel with progress updates + judged_results = [] + for coro in asyncio.as_completed(judgment_tasks): + judged_result = await coro + judged_results.append(judged_result) progress.update(task, advance=1) + # Calculate stats + correct = sum(1 for r in judged_results if r.get('is_correct', False)) + category_stats = {} + + for result in judged_results: + category = result['category'] + if category not in category_stats: + category_stats[category] = {'correct': 0, 'total': 0} + category_stats[category]['total'] += 1 + if result.get('is_correct', False): + category_stats[category]['correct'] += 1 + accuracy = (correct / total * 100) if total > 0 else 0 return { @@ -260,17 +330,82 @@ def calculate_metrics(results: List[Dict]) -> Dict: 'correct': correct, 'total': total, 'category_stats': category_stats, - 'detailed_results': results + 'detailed_results': judged_results } -def run_benchmark(max_conversations: int = None, max_questions_per_conv: int = None): +async def process_single_conversation( + memory: TemporalSemanticMemory, + conv_data: Dict, + i: int, + total_convs: int, + max_questions_per_conv: int, + skip_ingestion: bool +) -> Dict: + """ + Process a single conversation (ingest + evaluate). + + Args: + memory: Memory system instance + conv_data: Conversation data + i: Conversation index (1-based) + total_convs: Total number of conversations + max_questions_per_conv: Max questions to evaluate per conversation + skip_ingestion: Whether to skip ingestion + + Returns: + Result dict with sample_id, metrics, total_turns + """ + sample_id = conv_data['sample_id'] + agent_id = "locomo" # Single agent for all Locomo benchmark data + + console.print(f"\n[bold blue]Conversation {i}/{total_convs}[/bold blue] (Sample ID: {sample_id})") + + if not skip_ingestion: + # Clear previous locomo agent data only (multi-tenant safe) + if i == 1: # Only cleanup on first conversation + console.print(" [2] Clearing previous 'locomo' agent data...") + memory.delete_agent(agent_id) + console.print(f" [green]✓[/green] Cleared 'locomo' agent data") + + # Ingest conversation (sessions processed in parallel) + console.print(" [3] Ingesting conversation (sessions in parallel)...") + total_turns = await ingest_conversation(memory, conv_data, agent_id) + console.print(f" [green]✓[/green] Ingested {total_turns} turns across multiple sessions") + else: + total_turns = -1 + + # Evaluate QA (async - questions processed in parallel) + console.print(f" [4] Evaluating {len(conv_data['qa'])} QA pairs (parallel)...") + qa_results = await evaluate_qa_task( + memory, + agent_id, + conv_data['qa'], + sample_id, + max_questions=max_questions_per_conv + ) + + # Calculate metrics (async with parallel LLM judging) + console.print(" [5] Calculating metrics...") + metrics = await calculate_metrics(qa_results) + + console.print(f" [green]✓[/green] Accuracy: {metrics['accuracy']:.2f}% ({metrics['correct']}/{metrics['total']})") + + return { + 'sample_id': sample_id, + 'metrics': metrics, + 'total_turns': total_turns + } + + +def run_benchmark(max_conversations: int = None, max_questions_per_conv: int = None, skip_ingestion: bool = False): """ Run the LoComo benchmark. Args: max_conversations: Maximum number of conversations to evaluate (None for all) max_questions_per_conv: Maximum questions per conversation (None for all) + skip_ingestion: Whether to skip ingestion and use existing data """ console.print("\n[bold cyan]LoComo Benchmark - Entity-Aware Memory System[/bold cyan]") console.print("=" * 80) @@ -288,54 +423,17 @@ def run_benchmark(max_conversations: int = None, max_questions_per_conv: int = N memory = TemporalSemanticMemory() console.print(" [green]✓[/green] Memory system initialized") - # Run evaluation for each conversation + # Run evaluation (conversations sequential, sessions within each conversation parallel) all_results = [] for i, conv_data in enumerate(conversations_to_eval, 1): - sample_id = conv_data['sample_id'] - agent_id = f"locomo_{sample_id}" - - console.print(f"\n[bold blue]Conversation {i}/{len(conversations_to_eval)}[/bold blue] (Sample ID: {sample_id})") - - # Clear previous data - import psycopg2 - conn = psycopg2.connect(os.getenv('DATABASE_URL')) - cursor = conn.cursor() - cursor.execute("DELETE FROM memory_units WHERE agent_id = %s", (agent_id,)) - cursor.execute("DELETE FROM memory_links WHERE agent_id = %s", (agent_id,)) - cursor.execute("DELETE FROM entity_cooccurrences WHERE agent_id = %s", (agent_id,)) - cursor.execute("DELETE FROM unit_entities WHERE agent_id = %s", (agent_id,)) - cursor.execute("DELETE FROM entities WHERE agent_id = %s", (agent_id,)) - conn.commit() - cursor.close() - conn.close() - - # Ingest conversation (using async for parallel embedding generation) - console.print(" [3] Ingesting conversation (async with parallel embeddings)...") - total_turns = asyncio.run(ingest_conversation(memory, conv_data, agent_id)) - console.print(f" [green]✓[/green] Ingested {total_turns} conversation turns") - - # Evaluate QA - console.print(f" [4] Evaluating {len(conv_data['qa'])} QA pairs...") - qa_results = evaluate_qa_task( - memory, - agent_id, - conv_data['qa'], - sample_id, - max_questions=max_questions_per_conv + result = asyncio.run( + process_single_conversation( + memory, conv_data, i, len(conversations_to_eval), + max_questions_per_conv, skip_ingestion + ) ) - - # Calculate metrics - console.print(" [5] Calculating metrics...") - metrics = calculate_metrics(qa_results) - - console.print(f" [green]✓[/green] Accuracy: {metrics['accuracy']:.2f}% ({metrics['correct']}/{metrics['total']})") - - all_results.append({ - 'sample_id': sample_id, - 'metrics': metrics, - 'total_turns': total_turns - }) + all_results.append(result) # Overall results console.print("\n[bold green]✓ Benchmark Complete![/bold green]\n") @@ -387,12 +485,14 @@ if __name__ == "__main__": parser = argparse.ArgumentParser(description='Run LoComo benchmark') parser.add_argument('--max-conversations', type=int, default=None, help='Maximum conversations to evaluate') parser.add_argument('--max-questions', type=int, default=None, help='Maximum questions per conversation') + parser.add_argument('--skip-ingestion', action='store_true', help='Skip ingestion and use existing data') args = parser.parse_args() results = run_benchmark( max_conversations=args.max_conversations, - max_questions_per_conv=args.max_questions + max_questions_per_conv=args.max_questions, + skip_ingestion=args.skip_ingestion ) # Save results diff --git a/benchmarks/longmemeval/run_benchmark.py b/benchmarks/longmemeval/run_benchmark.py index e27adef6..72173e16 100644 --- a/benchmarks/longmemeval/run_benchmark.py +++ b/benchmarks/longmemeval/run_benchmark.py @@ -22,6 +22,7 @@ from typing import Dict, List, Any from pathlib import Path import time import asyncio +import subprocess from dotenv import load_dotenv # Load environment variables from .env @@ -74,6 +75,43 @@ def parse_args(): return parser.parse_args() +def download_dataset(dataset_path: Path) -> bool: + """ + Download the LongMemEval dataset if it doesn't exist. + + Returns: + True if successful, False otherwise + """ + url = "https://huggingface.co/datasets/xiaowu0162/longmemeval-cleaned/resolve/main/longmemeval_s_cleaned.json" + + console.print(f"[yellow]Dataset not found. Downloading from HuggingFace...[/yellow]") + console.print(f"[dim]URL: {url}[/dim]") + console.print(f"[dim]Destination: {dataset_path}[/dim]") + + try: + # Use curl to download with progress + result = subprocess.run( + ["curl", "-L", "-o", str(dataset_path), url], + capture_output=True, + text=True, + timeout=300 # 5 minute timeout + ) + + if result.returncode == 0 and dataset_path.exists(): + console.print(f"[green]✓ Dataset downloaded successfully[/green]") + return True + else: + console.print(f"[red]✗ Download failed: {result.stderr}[/red]") + return False + + except subprocess.TimeoutExpired: + console.print(f"[red]✗ Download timed out after 5 minutes[/red]") + return False + except Exception as e: + console.print(f"[red]✗ Download error: {e}[/red]") + return False + + def load_dataset(dataset_path: str) -> List[Dict[str, Any]]: """Load LongMemEval dataset from JSON file.""" with open(dataset_path, 'r') as f: @@ -158,7 +196,7 @@ async def ingest_conversation(memory: TemporalSemanticMemory, agent_id: str, ins console.print(f"[yellow]Warning: Failed to ingest session {session_id}: {e}[/yellow]") -def retrieve_memories( +async def retrieve_memories( memory: TemporalSemanticMemory, agent_id: str, query: str, @@ -179,7 +217,7 @@ def retrieve_memories( List of retrieved memory units """ try: - results = memory.search( + results = await memory.search_async( agent_id=agent_id, query=query, thinking_budget=thinking_budget, @@ -323,12 +361,13 @@ def run_benchmark(args): """Run the LongMemEval benchmark evaluation.""" console.print("\n[bold cyan]LongMemEval Benchmark Evaluation[/bold cyan]\n") - # Load dataset + # Load dataset - download if needed dataset_path = Path(__file__).parent / "longmemeval_s_cleaned.json" if not dataset_path.exists(): - console.print(f"[red]Error: Dataset not found at {dataset_path}[/red]") - console.print("[yellow]Run: curl -L 'https://huggingface.co/datasets/xiaowu0162/longmemeval-cleaned/resolve/main/longmemeval_s_cleaned.json' -o longmemeval_s_cleaned.json[/yellow]") - return + if not download_dataset(dataset_path): + console.print(f"[red]Failed to download dataset. Please download manually:[/red]") + console.print("[yellow]curl -L 'https://huggingface.co/datasets/xiaowu0162/longmemeval-cleaned/resolve/main/longmemeval_s_cleaned.json' -o benchmarks/longmemeval/longmemeval_s_cleaned.json[/yellow]") + return console.print(f"[green]Loading dataset from {dataset_path}[/green]") dataset = load_dataset(dataset_path) @@ -374,8 +413,11 @@ def run_benchmark(args): progress.update(instance_task, description=f"[cyan]Instance {idx+1}/{len(dataset)}: {question_id}") - # Create unique agent ID for this instance - agent_id = f"longmemeval_{question_id}" + # Use single agent for all LongMemEval data (cleared per question for isolation) + agent_id = "longmemeval" + + # Clear agent data for this question (each question needs fresh isolated context) + memory.delete_agent(agent_id) # Ingest conversation history try: @@ -385,13 +427,13 @@ def run_benchmark(args): continue # Retrieve memories - memories = retrieve_memories( + memories = asyncio.run(retrieve_memories( memory, agent_id, question, args.thinking_budget, args.top_k - ) + )) # Generate answer predicted_answer = generate_answer(client, question, memories) diff --git a/memory/coref_resolver.py b/memory/coref_resolver.py deleted file mode 100644 index 50ef3ea0..00000000 --- a/memory/coref_resolver.py +++ /dev/null @@ -1,336 +0,0 @@ -""" -Coreference resolution for memory units. - -Ensures every memory unit is self-contained by replacing pronouns -with their actual referents. -""" -import spacy -from typing import List, Dict, Optional -from fastcoref import FCoref -import threading - - -def get_nlp(): - """Get or load spaCy model.""" - try: - return spacy.load("en_core_web_sm") - except OSError: - raise Exception("spaCy model not found. Run: uv pip install https://github.com/explosion/spacy-models/releases/download/en_core_web_sm-3.7.1/en_core_web_sm-3.7.1-py3-none-any.whl") - - -# Global fastcoref model instance (singleton pattern) -_fastcoref_model = None -_fastcoref_lock = threading.Lock() - - -def get_fastcoref_model(): - """Get or load FastCoref model (singleton pattern).""" - global _fastcoref_model - if _fastcoref_model is None: - with _fastcoref_lock: - if _fastcoref_model is None: - # Use CPU by default, can be configured with device='cuda:0' for GPU - _fastcoref_model = FCoref(device='cpu') - return _fastcoref_model - - -def resolve_pronouns_in_text(text: str, context_sentences: List[str] = None) -> str: - """ - Resolve pronouns to their referents to make text self-contained. - - Strategy: - 1. Identify pronouns in the text - 2. Look for named entities in the same sentence or previous sentences - 3. Replace pronouns with the most likely referent based on: - - Gender agreement - - Number agreement (singular/plural) - - Proximity (closer entities more likely) - - Args: - text: The sentence to resolve - context_sentences: Previous sentences for context (optional) - - Returns: - Text with pronouns resolved - """ - nlp = get_nlp() - - # Parse the target sentence - doc = nlp(text) - - # Collect all sentences for context - all_text = text - if context_sentences: - # Add previous sentences for context - all_text = " ".join(context_sentences) + " " + text - - full_doc = nlp(all_text) - - # Extract entities with their positions - entities = [] - for ent in full_doc.ents: - if ent.label_ in ['PERSON', 'ORG', 'GPE', 'PRODUCT']: - entities.append({ - 'text': ent.text, - 'label': ent.label_, - 'start': ent.start_char, - 'end': ent.end_char, - }) - - # Check if sentence already has a named entity subject - has_named_subject = False - for token in doc: - if token.dep_ in ['nsubj', 'nsubjpass'] and token.pos_ == 'PROPN': - has_named_subject = True - break - - # Find pronouns and anaphoric references that need resolution - pronouns_to_replace = [] - - for token in doc: - # Handle pronouns (he, she, it, they) - if token.pos_ == 'PRON' and token.dep_ in ['nsubj', 'nsubjpass']: - # Subject pronouns that need resolution - pron_lower = token.text.lower() - - # Skip if sentence already has a named subject earlier - if has_named_subject and any( - t.dep_ in ['nsubj', 'nsubjpass'] and t.pos_ == 'PROPN' and t.i < token.i - for t in doc - ): - continue - - # Skip if it's already a proper name or demonstrative - if pron_lower in ['i', 'you', 'we', 'this', 'that', 'these', 'those']: - continue - - # Find the best entity to replace it with - referent = find_best_referent( - pronoun=token, - entities=entities, - doc=full_doc - ) - - if referent: - pronouns_to_replace.append({ - 'pronoun': token, - 'referent': referent, - 'start': token.idx, - 'end': token.idx + len(token.text) - }) - - # Handle definite noun phrases (e.g., "The project") - elif token.text.lower() == 'the' and token.head.pos_ == 'NOUN': - # Check if this "the X" phrase is a subject - if token.head.dep_ in ['nsubj', 'nsubjpass']: - # Try to find what "the X" refers to - noun = token.head.text - # Look for indefinite mentions earlier ("a project", "an organization") - for ent_token in reversed(list(full_doc)): - if ent_token.text.lower() == noun.lower(): - # Found a matching noun - check if it has indefinite article - if any(child.text.lower() in ['a', 'an'] for child in ent_token.children): - # Replace "the project" with "the Python project" or similar - # Get the full noun phrase - descriptors = [] - for child in ent_token.children: - if child.pos_ in ['ADJ', 'PROPN', 'NOUN'] and child.i < ent_token.i: - descriptors.append(child.text) - - if descriptors: - full_phrase = ' '.join(descriptors) + ' ' + noun - # Calculate span to replace - span_start = token.idx - span_end = token.head.idx + len(token.head.text) - - pronouns_to_replace.append({ - 'pronoun': token, - 'referent': 'the ' + full_phrase, - 'start': span_start, - 'end': span_end - }) - break - - # Replace pronouns with referents (in reverse order to maintain indices) - result = text - for item in reversed(pronouns_to_replace): - start = item['start'] - end = item['end'] - result = result[:start] + item['referent'] + result[end:] - - return result - - -def find_best_referent( - pronoun, - entities: List[Dict], - doc -) -> Optional[str]: - """ - Find the best entity referent for a pronoun. - - Uses: - - Gender agreement (he/she -> PERSON) - - Number agreement (singular/plural) - - Entity type (he/she -> PERSON, it -> ORG/PRODUCT) - - Proximity (closer entities preferred) - """ - pron_text = pronoun.text.lower() - - # Determine pronoun properties - is_singular = pron_text in ['he', 'she', 'it', 'him', 'her'] - is_plural = pron_text in ['they', 'them'] - is_person = pron_text in ['he', 'she', 'him', 'her'] - is_thing = pron_text in ['it'] - - # Score each entity - candidates = [] - - for entity in entities: - score = 0.0 - - # Proximity score (entities closer to pronoun are better) - # Since entities come from context, those appearing later (higher start position) are closer - proximity_score = entity['start'] / 1000.0 # Normalize by position - score += proximity_score - - # Type matching - if is_person and entity['label'] == 'PERSON': - score += 2.0 # Strong preference for person entities - elif is_thing and entity['label'] in ['ORG', 'PRODUCT', 'GPE']: - score += 2.0 # Organizations/products for "it" - - # Recency: prefer entities that appear just before the pronoun - if entity['end'] < pronoun.idx: - distance = pronoun.idx - entity['end'] - recency = 1.0 / (1.0 + distance / 100.0) - score += recency - - candidates.append((entity['text'], score)) - - # Return the highest scoring candidate - if candidates: - candidates.sort(key=lambda x: x[1], reverse=True) - return candidates[0][0] - - return None - - -def resolve_sentences_fast(sentences: List[str]) -> List[str]: - """ - Fast batch coreference resolution using FastCoref. - - This is significantly faster than the sequential spaCy-based approach: - - Processes entire document in one pass (O(n) instead of O(n²)) - - Uses efficient batching and neural model - - Can process 2.8K documents in 25 seconds on GPU - - Args: - sentences: List of sentences to resolve - - Returns: - List of resolved sentences (self-contained) - """ - if not sentences: - return [] - - # Join sentences into a single document for batch processing - # Add markers to track sentence boundaries - full_text = " ".join(sentences) - - # Get the fastcoref model - model = get_fastcoref_model() - - # Predict coreferences in batch - preds = model.predict(texts=[full_text]) - - if not preds or len(preds) == 0: - # No coreferences found, return original sentences - return sentences - - # Get the first (and only) result - result = preds[0] - - # Get clusters as text strings - clusters = result.get_clusters(as_strings=True) - - if not clusters: - return sentences - - # Build a replacement map: pronoun -> main referent - replacements = {} - for cluster in clusters: - if len(cluster) < 2: - continue - - # The first mention is typically the most complete referent - main_referent = cluster[0] - - # Map all other mentions (pronouns/short references) to the main referent - for mention in cluster[1:]: - mention_lower = mention.lower() - # Only replace if it's likely a pronoun or short reference - if len(mention.split()) <= 2 and any( - pron in mention_lower - for pron in ['he', 'she', 'it', 'they', 'him', 'her', 'them', 'his', 'her', 'their', 'the'] - ): - replacements[mention] = main_referent - - # Apply replacements to each sentence - resolved = [] - for sentence in sentences: - resolved_sentence = sentence - for mention, referent in replacements.items(): - # Case-insensitive replacement but preserve capitalization context - if mention in resolved_sentence: - resolved_sentence = resolved_sentence.replace(mention, referent) - resolved.append(resolved_sentence) - - return resolved - - -def resolve_sentences(sentences: List[str]) -> List[str]: - """ - Resolve pronouns across a list of sentences. - - Uses FastCoref for efficient batch processing. - Falls back to legacy spaCy method if FastCoref fails. - - Args: - sentences: List of sentences to resolve - - Returns: - List of resolved sentences (self-contained) - """ - try: - return resolve_sentences_fast(sentences) - except Exception as e: - # Fallback to legacy method - print(f"FastCoref failed ({e}), falling back to spaCy method") - return resolve_sentences_legacy(sentences) - - -def resolve_sentences_legacy(sentences: List[str]) -> List[str]: - """ - Legacy sequential pronoun resolution (slower, O(n²) complexity). - - Kept as fallback in case FastCoref is unavailable or fails. - - Args: - sentences: List of sentences to resolve - - Returns: - List of resolved sentences (self-contained) - """ - resolved = [] - - for i, sentence in enumerate(sentences): - # Use all previous sentences as context - context = resolved[:i] if i > 0 else [] - - # Resolve pronouns in this sentence - resolved_sentence = resolve_pronouns_in_text(sentence, context) - - resolved.append(resolved_sentence) - - return resolved diff --git a/memory/entity_resolver.py b/memory/entity_resolver.py index 9743758f..1107318c 100644 --- a/memory/entity_resolver.py +++ b/memory/entity_resolver.py @@ -48,6 +48,43 @@ def extract_entities(text: str) -> List[Dict[str, any]]: return entities +def extract_entities_batch(texts: List[str]) -> List[List[Dict[str, any]]]: + """ + Extract entities from multiple texts in batch (MUCH faster than sequential). + + Uses spaCy's nlp.pipe() for efficient batch processing. + + Args: + texts: List of input texts + + Returns: + List of entity lists, one per input text + """ + if not texts: + return [] + + nlp = get_nlp() + + # Process all texts in batch using nlp.pipe (significantly faster!) + docs = list(nlp.pipe(texts, batch_size=50)) + + all_entities = [] + for doc in docs: + entities = [] + for ent in doc.ents: + # Filter to important entity types + if ent.label_ in ['PERSON', 'ORG', 'GPE', 'LOC', 'PRODUCT', 'EVENT']: + entities.append({ + 'text': ent.text, + 'type': ent.label_, + 'start': ent.start_char, + 'end': ent.end_char, + }) + all_entities.append(entities) + + return all_entities + + class EntityResolver: """ Resolves entities to canonical IDs with disambiguation. @@ -62,6 +99,161 @@ class EntityResolver: """ self.conn = db_conn + def resolve_entities_batch( + self, + agent_id: str, + entities_data: List[Dict], + context: str, + unit_event_date, + ) -> List[str]: + """ + Resolve multiple entities in batch (MUCH faster than sequential). + + Groups entities by type, queries candidates in bulk, and resolves + all entities with minimal DB queries. + + Args: + agent_id: Agent ID + entities_data: List of dicts with 'text', 'type', 'nearby_entities' + context: Context where entities appear + unit_event_date: When this unit was created + + Returns: + List of entity IDs in same order as input + """ + if not entities_data: + return [] + + cursor = self.conn.cursor() + + try: + import time + start = time.time() + + # Group entities by type for efficient querying + entities_by_type = {} + for idx, entity_data in enumerate(entities_data): + entity_type = entity_data['type'] + if entity_type not in entities_by_type: + entities_by_type[entity_type] = [] + entities_by_type[entity_type].append((idx, entity_data)) + + # Query ALL candidates for each type in batch + all_candidates = {} # Maps (entity_type, entity_text) -> list of candidates + for entity_type, entities_list in entities_by_type.items(): + # Extract unique entity texts for this type + entity_texts = list(set(e[1]['text'] for e in entities_list)) + + # Query candidates for all texts at once + from psycopg2.extras import execute_values + cursor.execute( + """ + SELECT canonical_name, id, metadata, last_seen, mention_count + FROM entities + WHERE agent_id = %s AND entity_type = %s + """, + (agent_id, entity_type) + ) + type_candidates = cursor.fetchall() + + # Filter candidates in memory (faster than complex SQL for small datasets) + for entity_text in entity_texts: + matching = [] + entity_text_lower = entity_text.lower() + for canonical_name, ent_id, metadata, last_seen, mention_count in type_candidates: + canonical_lower = canonical_name.lower() + # Same matching logic as before + if (entity_text_lower == canonical_lower or + entity_text_lower in canonical_lower or + canonical_lower in entity_text_lower): + matching.append((ent_id, canonical_name, metadata, last_seen, mention_count)) + all_candidates[(entity_type, entity_text)] = matching + + # Resolve each entity using pre-fetched candidates + entity_ids = [None] * len(entities_data) + entities_to_update = [] # (entity_id, unit_event_date) + entities_to_create = [] # (idx, entity_data) + + for idx, entity_data in enumerate(entities_data): + entity_text = entity_data['text'] + entity_type = entity_data['type'] + nearby_entities = entity_data.get('nearby_entities', []) + + candidates = all_candidates.get((entity_type, entity_text), []) + + if not candidates: + # Will create new entity + entities_to_create.append((idx, entity_data)) + continue + + # Score candidates (same logic as before but with pre-fetched data) + best_candidate = None + best_score = 0.0 + best_name_similarity = 0.0 + + nearby_entity_set = {e['text'].lower() for e in nearby_entities if e['text'] != entity_text} + + for candidate_id, canonical_name, metadata, last_seen, mention_count in candidates: + score = 0.0 + + # Name similarity + name_similarity = SequenceMatcher( + None, + entity_text.lower(), + canonical_name.lower() + ).ratio() + score += name_similarity * 0.5 + + # Temporal proximity + if last_seen: + days_diff = abs((unit_event_date - last_seen).total_seconds() / 86400) + if days_diff < 7: + temporal_score = max(0, 1.0 - (days_diff / 7)) + score += temporal_score * 0.2 + + if score > best_score: + best_score = score + best_candidate = candidate_id + best_name_similarity = name_similarity + + # Apply threshold + threshold = 0.4 if entity_type == 'PERSON' and best_name_similarity >= 0.95 else 0.6 + + if best_score > threshold: + entity_ids[idx] = best_candidate + entities_to_update.append((best_candidate, unit_event_date)) + else: + entities_to_create.append((idx, entity_data)) + + # Batch update existing entities + if entities_to_update: + from psycopg2.extras import execute_values + execute_values( + cursor, + """ + UPDATE entities SET + mention_count = mention_count + 1, + last_seen = data.last_seen + FROM (VALUES %s) AS data(id, last_seen) + WHERE entities.id = data.id::uuid + """, + entities_to_update + ) + + # Batch create new entities + if entities_to_create: + for idx, entity_data in entities_to_create: + entity_id = self._create_entity( + cursor, agent_id, entity_data['text'], + entity_data['type'], unit_event_date + ) + entity_ids[idx] = entity_id + + return entity_ids + + finally: + cursor.close() + def resolve_entity( self, agent_id: str, @@ -297,6 +489,75 @@ class EntityResolver: (entity_id_1, entity_id_2) ) + def link_units_to_entities_batch(self, unit_entity_pairs: List[tuple[str, str]]): + """ + Link multiple memory units to entities in batch (MUCH faster than sequential). + + Also updates co-occurrence cache for entities that appear in the same unit. + + Args: + unit_entity_pairs: List of (unit_id, entity_id) tuples + """ + if not unit_entity_pairs: + return + + cursor = self.conn.cursor() + try: + # Batch insert all unit-entity links + from psycopg2.extras import execute_values + execute_values( + cursor, + """ + INSERT INTO unit_entities (unit_id, entity_id) + VALUES %s + ON CONFLICT DO NOTHING + """, + unit_entity_pairs + ) + + # Build map of unit -> entities for co-occurrence calculation + # Use sets to avoid duplicate entities in the same unit + unit_to_entities = {} + for unit_id, entity_id in unit_entity_pairs: + if unit_id not in unit_to_entities: + unit_to_entities[unit_id] = set() + unit_to_entities[unit_id].add(entity_id) + + # Update co-occurrences for all pairs in each unit + cooccurrence_pairs = set() # Use set to avoid duplicates + for unit_id, entity_ids in unit_to_entities.items(): + entity_list = list(entity_ids) # Convert set to list for iteration + # For each pair of entities in this unit, create co-occurrence + for i, entity_id_1 in enumerate(entity_list): + for entity_id_2 in entity_list[i+1:]: + # Skip if same entity (shouldn't happen with set, but be safe) + if entity_id_1 == entity_id_2: + continue + # Ensure consistent ordering (entity_id_1 < entity_id_2) + if entity_id_1 > entity_id_2: + entity_id_1, entity_id_2 = entity_id_2, entity_id_1 + cooccurrence_pairs.add((entity_id_1, entity_id_2)) + + # Batch update co-occurrences + if cooccurrence_pairs: + from datetime import datetime, timezone + now = datetime.now(timezone.utc) + execute_values( + cursor, + """ + INSERT INTO entity_cooccurrences (entity_id_1, entity_id_2, cooccurrence_count, last_cooccurred) + VALUES %s + ON CONFLICT (entity_id_1, entity_id_2) + DO UPDATE SET + cooccurrence_count = entity_cooccurrences.cooccurrence_count + 1, + last_cooccurred = EXCLUDED.last_cooccurred + """, + [(e1, e2, 1, now) for e1, e2 in cooccurrence_pairs] + ) + + finally: + cursor.close() + def get_units_by_entity(self, entity_id: str, limit: int = 100) -> List[str]: """ Get all units that mention an entity. diff --git a/memory/llm_client.py b/memory/llm_client.py index eb0d28b1..c58c6fcb 100644 --- a/memory/llm_client.py +++ b/memory/llm_client.py @@ -6,6 +6,8 @@ Uses OpenAI-compatible API (works with Groq, OpenAI, etc.) import os import json import re +import asyncio +from datetime import datetime from typing import List, Dict, Optional, Literal from openai import AsyncOpenAI from pydantic import BaseModel, Field @@ -16,16 +18,8 @@ class ExtractedFact(BaseModel): fact: str = Field( description="Self-contained factual statement with subject + action + context" ) - speaker: str = Field( - default="narrator", - description="Who said this (name or 'narrator' if not a conversation)" - ) - type: Literal["biographical", "event", "opinion", "recommendation", "description", "relationship"] = Field( - description="Category of the fact" - ) - confidence: Literal["high", "medium", "low"] = Field( - default="medium", - description="Confidence level of the extraction" + date: str = Field( + description="Absolute date/time when this fact occurred in ISO format (YYYY-MM-DDTHH:MM:SSZ). If text mentions relative time (yesterday, last week, this morning), calculate absolute date from the provided context date." ) @@ -36,75 +30,48 @@ class FactExtractionResponse(BaseModel): ) -def split_into_sentences(text: str) -> List[str]: - """ - Fast sentence splitter using regex. - Splits on periods, exclamation marks, and question marks followed by whitespace or end of string. - - Args: - text: Input text to split - - Returns: - List of sentences - """ - # Split on sentence boundaries: .!? followed by space/newline/end - sentences = re.split(r'(?<=[.!?])\s+', text) - return [s.strip() for s in sentences if s.strip()] - - def chunk_text(text: str, max_chars: int = 120000) -> List[str]: """ - Split text into chunks at sentence boundaries. + Split text into chunks at sentence boundaries using LangChain's text splitter. - Keeps chunks under max_chars (~30k tokens assuming 1 token ≈ 4 chars). - This prevents hitting output token limits on large documents. + Uses RecursiveCharacterTextSplitter which intelligently splits at sentence boundaries + and allows chunks to slightly exceed max_chars to finish sentences naturally. Args: text: Input text to chunk max_chars: Maximum characters per chunk (default 120k ≈ 30k tokens) + Note: chunks may slightly exceed this to complete sentences Returns: - List of text chunks, each under max_chars + List of text chunks, roughly under max_chars """ + from langchain_text_splitters import RecursiveCharacterTextSplitter + # If text is small enough, return as-is if len(text) <= max_chars: return [text] - sentences = split_into_sentences(text) - chunks = [] - current_chunk = [] - current_length = 0 + # Configure splitter to split at sentence boundaries first + # Separators in order of preference: paragraphs, newlines, sentences, words + splitter = RecursiveCharacterTextSplitter( + chunk_size=max_chars, + chunk_overlap=0, + length_function=len, + is_separator_regex=False, + separators=[ + "\n\n", # Paragraph breaks + "\n", # Line breaks + ". ", # Sentence endings + "! ", # Exclamations + "? ", # Questions + "; ", # Semicolons + ", ", # Commas + " ", # Words + "", # Characters (last resort) + ], + ) - for sentence in sentences: - sentence_length = len(sentence) - - # If single sentence exceeds max_chars, split it forcefully - if sentence_length > max_chars: - # Save current chunk if any - if current_chunk: - chunks.append(' '.join(current_chunk)) - current_chunk = [] - current_length = 0 - - # Split long sentence into smaller pieces - for i in range(0, len(sentence), max_chars): - chunks.append(sentence[i:i + max_chars]) - continue - - # If adding this sentence would exceed limit, start new chunk - if current_length + sentence_length + 1 > max_chars: - chunks.append(' '.join(current_chunk)) - current_chunk = [sentence] - current_length = sentence_length - else: - current_chunk.append(sentence) - current_length += sentence_length + 1 # +1 for space - - # Add remaining chunk - if current_chunk: - chunks.append(' '.join(current_chunk)) - - return chunks + return splitter.split_text(text) def get_llm_client() -> AsyncOpenAI: @@ -137,22 +104,28 @@ def get_llm_client() -> AsyncOpenAI: ) -async def extract_facts_from_text( - text: str, - model: str = "openai/gpt-oss-20b", - temperature: float = 0.1, - max_tokens: int = 65000, - chunk_size: int = 60000 +async def _extract_facts_from_chunk( + chunk: str, + chunk_index: int, + total_chunks: int, + event_date: datetime, + context: str, + model: str, + temperature: float, + max_tokens: int, + client: AsyncOpenAI ) -> List[Dict[str, str]]: - client = get_llm_client() + """ + Extract facts from a single chunk (internal helper for parallel processing). + """ + # Format event_date for the prompt + event_date_str = event_date.strftime("%Y-%m-%dT%H:%M:%SZ") - # Chunk text if necessary - chunks = chunk_text(text, max_chars=chunk_size) + prompt = f"""You are extracting facts from text for an AI memory system. Each fact will be stored and retrieved later. - all_facts = [] - - for i, chunk in enumerate(chunks): - prompt = f"""You are extracting facts from text for an AI memory system. Each fact will be stored and retrieved later. +## CONTEXT INFORMATION +- Current reference date/time: {event_date_str} +- Context: {context if context else 'no context provided'} ## CRITICAL: Facts must be DETAILED and COMPREHENSIVE @@ -164,66 +137,158 @@ Each fact should: 5. Include surrounding context that makes the fact meaningful 6. Capture nuances, reasons, causes, and implications -## What to EXTRACT: -- Biographical information (jobs, roles, backgrounds, experiences) -- Events (what happened, when, where, who was involved, why) -- Opinions and beliefs (who believes what and why) -- Recommendations and advice (specific suggestions with reasoning) -- Descriptions (detailed explanations of how things work) -- Relationships (connections between people, organizations, concepts) +## TEMPORAL INFORMATION (VERY IMPORTANT) +For each fact, extract the ABSOLUTE date/time when it occurred: +- If text mentions ABSOLUTE dates ("on March 15, 2024", "last Tuesday"), use that date +- If text mentions RELATIVE times ("yesterday", "last week", "this morning", "3 days ago"), calculate the absolute date using the reference date above. +- if text mentions a vague relative time without a specific day ("last week", "this morning"), transform the date in relative with absolute context ("last week" + " 2 june 2024" -> "week before June 2 2024") in the text and use the absolute date for the 'date' field +- If NO specific time is mentioned, use the reference date +- Always output dates in ISO format: YYYY-MM-DDTHH:MM:SSZ -## What to SKIP: -- Greetings, thank yous, acknowledgments +Examples of date extraction: +- Reference: 2024-03-20T10:00:00Z +- "Yesterday I went hiking" → date: 2024-03-19T10:00:00Z +- "Last week I joined Google" → date: 2024-03-13T10:00:00Z (approximately) +- "This morning I had coffee" → date: 2024-03-20T08:00:00Z +- "I work at Google" (no time mentioned) → date: 2024-03-20T10:00:00Z (use reference) + +## What to EXTRACT (BE EXHAUSTIVE - DO NOT SKIP ANYTHING): +- **Biographical information**: jobs, roles, backgrounds, experiences, skills +- **Events (NEVER MISS THESE)**: + - ANY action that happened (went, did, attended, joined, started, finished, etc.) + - Photos, images, videos shared or taken ("here's a photo", "took a picture", "captured") + - Social activities (meetups, gatherings, meals, conversations) + - Achievements, milestones, accomplishments + - Travels, visits, locations visited + - Purchases, acquisitions, creations +- **Opinions and beliefs**: who believes what and why +- **Recommendations and advice**: specific suggestions with reasoning +- **Descriptions**: detailed explanations of how things work +- **Relationships**: connections between people, organizations, concepts +- **States and conditions**: current status, ongoing situations + +## CRITICAL: Extract EVERY event mentioned, even casual ones +- "here's a photo of X" = someone took/shared a photo of X +- "I was with friends last week" = meetup/gathering with friends last week +- "sent you that link" = action of sending a link +- DO NOT skip events just because they seem minor or casual + +## What to SKIP (ONLY these): +- Greetings, thank yous, acknowledgments (unless they reveal information) - Filler words ("um", "uh", "like") -- Pure reactions without content ("wow", "cool") -- Incomplete thoughts +- Pure reactions without content ("wow", "cool", "nice") +- Incomplete thoughts or sentence fragments with no meaning ## EXAMPLES of GOOD facts (detailed, comprehensive): -Input: "Alice mentioned she works at Google in Mountain View. She joined the AI team last year and loves working on large language models." -GOOD: "Alice works at Google in Mountain View on the AI team, which she joined last year, and she loves working on large language models" -BAD: "Alice works at Google" (too short, missing context) +Input: "Alice mentioned she works at Google in Mountain View. She joined the AI team last year." +GOOD fact: "Alice works at Google in Mountain View on the AI team, which she joined last year" +GOOD date: Calculate based on reference date (if reference is 2024-03-20, "last year" = 2023-03-20) -Input: "Bob said he's been hiking every weekend in Yosemite because it helps him clear his mind after stressful work weeks." -GOOD: "Bob has been hiking every weekend in Yosemite because it helps him clear his mind after stressful work weeks" -BAD: "Bob hikes in Yosemite" (missing frequency, reason, and context) +Input: "Yesterday Bob went hiking in Yosemite because it helps him clear his mind." +GOOD fact: "Bob went hiking in Yosemite because it helps him clear his mind" +GOOD date: Reference date minus 1 day -Input: "The new algorithm reduced latency by 40% compared to the baseline by using a novel caching strategy." -GOOD: "The new algorithm reduced latency by 40% compared to the baseline by using a novel caching strategy" -BAD: "The algorithm is faster" (missing numbers, comparison, and method) +Input: "Here's a photo of me with my friends taken last week at the beach." +GOOD fact: "Someone shared/took a photo with their friends at the beach" +GOOD date: Reference date minus 7 days (last week) +NOTE: Extract the event (photo taken/shared with friends at beach), NOT just that a photo exists + +Input: "I sent you that article about AI last Tuesday." +GOOD fact: "Someone sent an article about AI" +GOOD date: Calculate last Tuesday from reference date ## TEXT TO EXTRACT FROM: {chunk} -Remember: Include ALL details, names, numbers, reasons, and context. Facts should be rich and informative, not summaries.""" +Remember: +1. BE EXHAUSTIVE - Extract EVERY event, action, and fact mentioned +2. DO NOT skip casual mentions like "here's a photo", "I was with X", "sent you Y" +3. Include ALL details, names, numbers, reasons, and context in the fact text +4. Extract the absolute date for EACH fact by calculating relative times from the reference date +5. When in doubt, EXTRACT IT - better to have too many facts than miss important events""" - # Use parse() for structured outputs with Pydantic models - response = await client.beta.chat.completions.parse( + response = await client.beta.chat.completions.parse( + model=model, + messages=[ + { + "role": "system", + "content": "You are an EXHAUSTIVE fact extractor. Extract EVERY event, action, and fact mentioned - never skip anything. This includes casual mentions like photos shared, things sent, meetups, gatherings, or any action. Preserve all context, details, and nuances. Calculate absolute dates from relative time expressions. When in doubt, extract it - missing facts is worse than extracting too many." + }, + { + "role": "user", + "content": prompt + } + ], + temperature=temperature, + max_tokens=max_tokens, + response_format=FactExtractionResponse, + extra_body={"service_tier": "auto"}, + ) + + # Extract the parsed response + extraction_response = response.choices[0].message.parsed + + # Convert to dict format + chunk_facts = [fact.model_dump() for fact in extraction_response.facts] + + return chunk_facts + + +async def extract_facts_from_text( + text: str, + event_date: datetime, + context: str = "", + model: str = "openai/gpt-oss-120b", + temperature: float = 0.1, + max_tokens: int = 65000, + chunk_size: int = 5000 +) -> List[Dict[str, str]]: + """ + Extract semantic facts from conversational or narrative text using LLM. + + For large texts (>chunk_size chars), automatically chunks at sentence boundaries + to avoid hitting output token limits. Processes ALL chunks in PARALLEL for speed. + + Args: + text: Input text (conversation, article, etc.) + event_date: Reference date for resolving relative times + context: Context about the conversation/document + model: LLM model to use + temperature: Sampling temperature (lower = more focused) + max_tokens: Maximum tokens in response + chunk_size: Maximum characters per chunk + + Returns: + List of fact dictionaries with 'fact' and 'date' keys + """ + client = get_llm_client() + + # Chunk text if necessary + chunks = chunk_text(text, max_chars=chunk_size) + + # Process all chunks in parallel using asyncio.gather + tasks = [ + _extract_facts_from_chunk( + chunk=chunk, + chunk_index=i, + total_chunks=len(chunks), + event_date=event_date, + context=context, model=model, - messages=[ - { - "role": "system", - "content": "You extract detailed, comprehensive facts from text. Preserve all context, details, and nuances. Never summarize or shorten - include everything relevant." - }, - { - "role": "user", - "content": prompt - } - ], temperature=temperature, max_tokens=max_tokens, - response_format=FactExtractionResponse + client=client ) + for i, chunk in enumerate(chunks) + ] - # Extract the parsed response - extraction_response = response.choices[0].message.parsed + # Wait for all chunks to complete in parallel + chunk_results = await asyncio.gather(*tasks) - # Convert to dict format and add to aggregate - chunk_facts = [fact.model_dump() for fact in extraction_response.facts] + # Flatten results from all chunks + all_facts = [] + for chunk_facts in chunk_results: all_facts.extend(chunk_facts) - # Log progress for large documents - if len(chunks) > 1: - print(f"Processed chunk {i + 1}/{len(chunks)}: extracted {len(chunk_facts)} facts") - return all_facts diff --git a/memory/temporal_semantic_memory.py b/memory/temporal_semantic_memory.py index f5ee7dba..9e012a21 100644 --- a/memory/temporal_semantic_memory.py +++ b/memory/temporal_semantic_memory.py @@ -18,14 +18,15 @@ from sentence_transformers import SentenceTransformer from dotenv import load_dotenv import asyncio import time +from concurrent.futures import ProcessPoolExecutor +import numpy as np from .utils import ( extract_facts, calculate_recency_weight, calculate_frequency_weight, ) -from .entity_resolver import EntityResolver, extract_entities -from .coref_resolver import resolve_sentences +from .entity_resolver import EntityResolver def utcnow(): @@ -33,6 +34,45 @@ def utcnow(): return datetime.now(timezone.utc) +# Global process pool for parallel embedding generation +# Each process loads its own copy of the embedding model +# This provides TRUE parallelism for CPU-bound embedding operations +_PROCESS_POOL = None +_EMBEDDING_MODEL_NAME = "BAAI/bge-small-en-v1.5" + +# Process-local model cache (one per worker process) +_worker_model = None + + +def _get_worker_model(): + """Get or load the embedding model in worker process.""" + global _worker_model + if _worker_model is None: + _worker_model = SentenceTransformer(_EMBEDDING_MODEL_NAME) + return _worker_model + + +def _encode_batch_worker(texts: List[str]) -> List[List[float]]: + """ + Worker function for process pool - encodes texts to embeddings. + + This function runs in a separate process and loads its own model. + """ + model = _get_worker_model() + embeddings = model.encode(texts, convert_to_numpy=True, show_progress_bar=False) + return [emb.tolist() for emb in embeddings] + + +def _get_process_pool(): + """Get or create the global process pool.""" + global _PROCESS_POOL + if _PROCESS_POOL is None: + # Use 4 worker processes for true parallelism + # Adjust based on your CPU cores (each process loads ~500MB model) + _PROCESS_POOL = ProcessPoolExecutor(max_workers=4) + return _PROCESS_POOL + + class TemporalSemanticMemory: """ Advanced memory system using temporal and semantic linking with PostgreSQL. @@ -94,11 +134,13 @@ class TemporalSemanticMemory: async def _generate_embeddings_batch(self, texts: List[str]) -> List[List[float]]: """ - Generate embeddings for multiple texts using local model (batch processing). + Generate embeddings for multiple texts using local model in parallel. - Local models are fast and process batches efficiently without needing - parallel API calls. We run this in asyncio to avoid blocking, but the - actual embedding generation is synchronous. + Uses a ProcessPoolExecutor to achieve TRUE parallelism for CPU-bound + embedding generation. Each worker process loads its own model copy. + + When multiple put_async calls run in parallel, each can generate + embeddings concurrently in separate processes (no GIL contention). Args: texts: List of texts to embed @@ -107,13 +149,15 @@ class TemporalSemanticMemory: List of 384-dimensional embeddings in same order as input texts """ try: - # Run in thread pool to avoid blocking event loop + # Run in process pool for true parallelism loop = asyncio.get_event_loop() + pool = _get_process_pool() embeddings = await loop.run_in_executor( - None, - lambda: self.embedding_model.encode(texts, convert_to_numpy=True, show_progress_bar=False) + pool, + _encode_batch_worker, + texts ) - return [emb.tolist() for emb in embeddings] + return embeddings except Exception as e: raise Exception(f"Failed to generate batch embeddings: {str(e)}") @@ -208,14 +252,7 @@ class TemporalSemanticMemory: """ Store content as memory units with temporal and semantic links (ASYNC version). - This async version generates ALL embeddings in parallel for maximum speed, - then uses batch inserts for database operations. - - Steps: - 1. Split content into sentence units - 2. Resolve coreferences - 3. **Generate ALL embeddings in parallel** (FAST!) - 4. **Batch insert all units and links** (FAST!) + This is a convenience wrapper around put_batch_async for a single content item. Args: agent_id: Unique identifier for the agent @@ -226,131 +263,212 @@ class TemporalSemanticMemory: Returns: List of created unit IDs """ + # Use put_batch_async with a single item (avoids code duplication) + result = await self.put_batch_async( + agent_id=agent_id, + contents=[{ + "content": content, + "context": context, + "event_date": event_date + }] + ) + + # Return the first (and only) list of unit IDs + return result[0] if result else [] + + async def put_batch_async( + self, + agent_id: str, + contents: List[Dict[str, Any]], + ) -> List[List[str]]: + """ + Store multiple content items as memory units in ONE batch operation. + + This is MUCH more efficient than calling put_async multiple times: + - Extracts facts from all contents in parallel + - Generates ALL embeddings in ONE batch + - Does ALL database operations in ONE transaction + + Args: + agent_id: Unique identifier for the agent + contents: List of dicts with keys: + - "content" (required): Text content to store + - "context" (optional): Context about the memory + - "event_date" (optional): When the event occurred + + Returns: + List of lists of unit IDs (one list per content item) + + Example: + unit_ids = await memory.put_batch_async( + agent_id="user123", + contents=[ + {"content": "Alice works at Google", "context": "conversation"}, + {"content": "Bob loves Python", "context": "conversation"}, + ] + ) + # Returns: [["unit-id-1"], ["unit-id-2"]] + """ start_time = time.time() print(f"\n{'='*60}") - print(f"PUT_ASYNC START: {agent_id}") - print(f"Content length: {len(content)} chars") + print(f"PUT_BATCH_ASYNC START: {agent_id}") + print(f"Batch size: {len(contents)} content items") print(f"{'='*60}") - if event_date is None: - event_date = utcnow() - - # Step 1: Extract semantic facts using LLM (async) - step_start = time.time() - try: - facts = await extract_facts(content) - print(f"[1] Extract facts: {len(facts)} facts in {time.time() - step_start:.3f}s") - except Exception as e: - print(f"\n{'='*60}") - print(f"PUT_ASYNC FAILED: Fact extraction error") - print(f"Error: {e}") - print(f"{'='*60}\n") - raise Exception(f"Failed to extract facts from content: {e}") - - # Step 2: Resolve pronouns to make facts even more self-contained - step_start = time.time() - sentences = resolve_sentences(facts) - print(f"[2] Resolve coreferences: {time.time() - step_start:.3f}s") - - # Step 3: Generate ALL embeddings in parallel - step_start = time.time() - embeddings = await self._generate_embeddings_batch(sentences) - print(f"[3] Generate embeddings (parallel): {len(embeddings)} embeddings in {time.time() - step_start:.3f}s") - - # Step 4: Check for duplicates using similarity + temporal window - cursor = self.conn.cursor() - step_start = time.time() - duplicate_flags = self._find_duplicate_facts_batch( - cursor, agent_id, sentences, embeddings, event_date - ) - num_duplicates = sum(duplicate_flags) - - # Filter out duplicates - filtered_data = [ - (sentence, embedding) - for sentence, embedding, is_dup in zip(sentences, embeddings, duplicate_flags) - if not is_dup - ] - - if filtered_data: - sentences, embeddings = zip(*filtered_data) - sentences = list(sentences) - embeddings = list(embeddings) - else: - sentences = [] - embeddings = [] - - print(f"[4] Deduplication check: {num_duplicates} duplicates filtered, {len(sentences)} new facts in {time.time() - step_start:.3f}s") - - # If all facts were duplicates, return empty list - if not sentences: - cursor.close() - print(f"\n{'='*60}") - print(f"PUT_ASYNC COMPLETE: All facts were duplicates, nothing stored") - print(f"{'='*60}\n") + if not contents: return [] - # Step 5: Batch insert everything + # Step 1: Extract facts from ALL contents in parallel + step_start = time.time() + + # Create tasks for parallel fact extraction + fact_extraction_tasks = [] + for item in contents: + content = item["content"] + context = item.get("context", "") + event_date = item.get("event_date") or utcnow() + + task = extract_facts(content, event_date, context) + fact_extraction_tasks.append((task, event_date, context)) + + # Wait for all fact extractions to complete + all_fact_results = await asyncio.gather(*[task for task, _, _ in fact_extraction_tasks]) + + # Flatten and track which facts belong to which content + all_fact_texts = [] + all_fact_dates = [] + all_contexts = [] + content_boundaries = [] # [(start_idx, end_idx), ...] + + current_idx = 0 + for i, ((_, event_date, context), fact_dicts) in enumerate(zip(fact_extraction_tasks, all_fact_results)): + start_idx = current_idx + + for fact_dict in fact_dicts: + all_fact_texts.append(fact_dict['fact']) + try: + from dateutil import parser as date_parser + fact_date = date_parser.isoparse(fact_dict['date']) + all_fact_dates.append(fact_date) + except Exception: + all_fact_dates.append(event_date) + all_contexts.append(context) + + end_idx = current_idx + len(fact_dicts) + content_boundaries.append((start_idx, end_idx)) + current_idx = end_idx + + total_facts = len(all_fact_texts) + + if total_facts == 0: + return [[] for _ in contents] + + # Step 2: Generate ALL embeddings in ONE batch (HUGE speedup!) + step_start = time.time() + all_embeddings = await self._generate_embeddings_batch(all_fact_texts) + print(f"[2] Generate embeddings (parallel): {len(all_embeddings)} embeddings in {time.time() - step_start:.3f}s") + + # Step 3: Process everything in ONE database transaction + cursor = self.conn.cursor() try: - # Batch INSERT all memory units + # Deduplication check for all facts + step_start = time.time() + all_is_duplicate = [] + for sentence, embedding, fact_date in zip(all_fact_texts, all_embeddings, all_fact_dates): + dup_flags = self._find_duplicate_facts_batch( + cursor, agent_id, [sentence], [embedding], fact_date + ) + all_is_duplicate.extend(dup_flags) + + duplicates_filtered = sum(all_is_duplicate) + new_facts = total_facts - duplicates_filtered + print(f"[3] Deduplication check: {duplicates_filtered} duplicates filtered, {new_facts} new facts in {time.time() - step_start:.3f}s") + + # Filter out duplicates + filtered_sentences = [s for s, is_dup in zip(all_fact_texts, all_is_duplicate) if not is_dup] + filtered_embeddings = [e for e, is_dup in zip(all_embeddings, all_is_duplicate) if not is_dup] + filtered_dates = [d for d, is_dup in zip(all_fact_dates, all_is_duplicate) if not is_dup] + filtered_contexts = [c for c, is_dup in zip(all_contexts, all_is_duplicate) if not is_dup] + + if not filtered_sentences: + print(f"[PUT_BATCH_ASYNC] All facts were duplicates, returning empty") + return [[] for _ in contents] + + # Batch insert ALL units step_start = time.time() from psycopg2.extras import execute_values unit_data = [ - (agent_id, sentence, embedding, context, event_date, 0) - for sentence, embedding in zip(sentences, embeddings) + (agent_id, sentence, context, embedding, date, 0) # access_count starts at 0 + for sentence, context, embedding, date in zip( + filtered_sentences, filtered_contexts, filtered_embeddings, filtered_dates + ) ] - unit_ids = execute_values( + results = execute_values( cursor, """ - INSERT INTO memory_units (agent_id, text, embedding, context, event_date, access_count) + INSERT INTO memory_units (agent_id, text, context, embedding, event_date, access_count) VALUES %s RETURNING id """, unit_data, fetch=True ) - created_unit_ids = [str(row[0]) for row in unit_ids] - print(f"[5] Batch insert units: {time.time() - step_start:.3f}s") - # Process entities for all units - step_start = time.time() - all_entity_links = [] - for unit_id, sentence in zip(created_unit_ids, sentences): - entity_links = self._extract_entities_for_batch(cursor, agent_id, unit_id, sentence, context, event_date, sentences) - all_entity_links.extend(entity_links) - print(f"[6] Extract entities: {time.time() - step_start:.3f}s") + created_unit_ids = [str(row[0]) for row in results] + print(f"[5] Batch insert units: {len(created_unit_ids)} units in {time.time() - step_start:.3f}s") - # Create ALL temporal links in batch + # Process entities for ALL units step_start = time.time() - self._create_temporal_links_batch(cursor, agent_id, created_unit_ids, event_date) + all_entity_links = self._extract_entities_batch_optimized( + cursor, agent_id, created_unit_ids, filtered_sentences, "", filtered_dates + ) + print(f"[6] Extract entities (batched): {time.time() - step_start:.3f}s") + + # Create temporal links + step_start = time.time() + self._create_temporal_links_batch_per_fact(cursor, agent_id, created_unit_ids) print(f"[7] Batch create temporal links: {time.time() - step_start:.3f}s") - # Create ALL semantic links in batch + # Create semantic links step_start = time.time() - self._create_semantic_links_batch(cursor, agent_id, created_unit_ids, embeddings) + self._create_semantic_links_batch(cursor, agent_id, created_unit_ids, filtered_embeddings) print(f"[8] Batch create semantic links: {time.time() - step_start:.3f}s") - # Insert all entity links in batch + # Insert entity links step_start = time.time() if all_entity_links: self._insert_entity_links_batch(cursor, all_entity_links) print(f"[9] Batch insert entity links: {time.time() - step_start:.3f}s") + # Commit everything commit_start = time.time() self.conn.commit() print(f"[10] Commit: {time.time() - commit_start:.3f}s") + # Map created unit IDs back to original content items + # Account for duplicates when mapping back + result_unit_ids = [] + filtered_idx = 0 + + for start_idx, end_idx in content_boundaries: + content_unit_ids = [] + for i in range(start_idx, end_idx): + if not all_is_duplicate[i]: + content_unit_ids.append(created_unit_ids[filtered_idx]) + filtered_idx += 1 + result_unit_ids.append(content_unit_ids) + total_time = time.time() - start_time print(f"\n{'='*60}") - print(f"PUT_ASYNC COMPLETE: {len(created_unit_ids)} units stored in {total_time:.3f}s") + print(f"PUT_BATCH_ASYNC COMPLETE: {len(created_unit_ids)} units from {len(contents)} contents in {total_time:.3f}s") print(f"{'='*60}\n") - return created_unit_ids + return result_unit_ids except Exception as e: self.conn.rollback() - raise Exception(f"Failed to store memory: {str(e)}") + raise Exception(f"Failed to store batch memory: {str(e)}") finally: cursor.close() @@ -412,7 +530,11 @@ class TemporalSemanticMemory: ) except Exception as e: - print(f"Warning: Failed to create temporal links: {str(e)}") + print(f"ERROR: Failed to create temporal links: {str(e)}") + import traceback + traceback.print_exc() + # Re-raise to trigger rollback at put_async level + raise def _create_semantic_links( self, @@ -471,77 +593,11 @@ class TemporalSemanticMemory: ) except Exception as e: - print(f"Warning: Failed to create semantic links: {str(e)}") - - def _extract_and_link_entities( - self, - cursor, - agent_id: str, - unit_id: str, - text: str, - context: str, - event_date, - all_sentences: List[str], - ): - """ - Extract entities from text, resolve them, and create entity links. - - Args: - cursor: Database cursor - agent_id: Agent ID - unit_id: Current unit ID - text: Unit text - context: Context - event_date: When created - all_sentences: All sentences from the same PUT (for context) - """ - try: - # Extract entities from this unit - entities = extract_entities(text) - - if not entities: - return - - # Resolve each entity and link - entity_ids = [] - for entity in entities: - entity_id = self.entity_resolver.resolve_entity( - agent_id=agent_id, - entity_text=entity['text'], - entity_type=entity['type'], - context=context, - nearby_entities=entities, - unit_event_date=event_date - ) - entity_ids.append(entity_id) - - # Link unit to entity - self.entity_resolver.link_unit_to_entity(unit_id, entity_id) - - # Create entity links to other units that mention the same entities - for entity_id in set(entity_ids): - # Get other units that mention this entity - related_units = self.entity_resolver.get_units_by_entity(entity_id, limit=50) - - # Create entity links - links = [] - for related_unit_id in related_units: - if str(related_unit_id) != str(unit_id): - links.append((unit_id, related_unit_id, 'entity', 1.0, entity_id)) - - if links: - execute_values( - cursor, - """ - INSERT INTO memory_links (from_unit_id, to_unit_id, link_type, weight, entity_id) - VALUES %s - ON CONFLICT DO NOTHING - """, - links - ) - - except Exception as e: - print(f"Warning: Failed to extract/link entities: {str(e)}") + print(f"ERROR: Failed to create semantic links: {str(e)}") + import traceback + traceback.print_exc() + # Re-raise to trigger rollback at put_async level + raise def search( self, @@ -552,7 +608,34 @@ class TemporalSemanticMemory: live_tracer=None, ) -> List[Dict[str, Any]]: """ - Search memories using spreading activation. + Search memories using spreading activation (synchronous wrapper). + + This is a synchronous wrapper around search_async() for convenience. + For best performance, use search_async() directly. + + Args: + agent_id: Agent ID to search for + query: Search query + thinking_budget: How many units to explore (computational budget) + top_k: Number of results to return + live_tracer: Optional LiveSearchTracer for visualization + + Returns: + List of memory units with their weights, sorted by relevance + """ + # Run async version synchronously + return asyncio.run(self.search_async(agent_id, query, thinking_budget, top_k, live_tracer)) + + async def search_async( + self, + agent_id: str, + query: str, + thinking_budget: int = 50, + top_k: int = 10, + live_tracer=None, + ) -> List[Dict[str, Any]]: + """ + Search memories using spreading activation (ASYNC version). This implements the core SEARCH operation: 1. Find entry points (most relevant units via vector search) @@ -572,14 +655,20 @@ class TemporalSemanticMemory: """ cursor = self.conn.cursor(cursor_factory=RealDictCursor) + search_start = time.time() + print(f"\n[SEARCH] Starting search for query: '{query[:50]}...' (thinking_budget={thinking_budget}, top_k={top_k})") + try: # Step 1: Generate query embedding + step_start = time.time() query_embedding = self._generate_embedding(query) + print(f" [1] Generate query embedding: {time.time() - step_start:.3f}s") # Step 2: Find entry points + step_start = time.time() cursor.execute( """ - SELECT id, text, context, event_date, access_count, + SELECT id, text, context, event_date, access_count, embedding, 1 - (embedding <=> %s::vector) AS similarity FROM memory_units WHERE agent_id = %s @@ -592,104 +681,230 @@ class TemporalSemanticMemory: ) entry_points = cursor.fetchall() + print(f" [2] Find entry points: {len(entry_points)} found in {time.time() - step_start:.3f}s") + if not entry_points: + print(f"[SEARCH] Complete: 0 results in {time.time() - search_start:.3f}s") return [] # Step 3: Spreading activation with budget + step_start = time.time() visited = set() results = [] budget_remaining = thinking_budget - queue = [(dict(unit), 1.0, True) for unit in entry_points] # (unit, activation, is_entry) + # Initialize entry points with their actual similarity scores instead of 1.0 + queue = [(dict(unit), unit["similarity"], True) for unit in entry_points] # (unit, activation, is_entry) + + # Track substep timings + update_access_time = 0 + calculate_weight_time = 0 + query_neighbors_time = 0 + process_neighbors_time = 0 + + # Process nodes in batches for efficient neighbor querying + BATCH_SIZE = 50 + nodes_to_process = [] # (unit, activation, is_entry_point) while queue and budget_remaining > 0: - current_unit, activation, is_entry_point = queue.pop(0) - unit_id = str(current_unit["id"]) + # Collect a batch of nodes to process + while queue and len(nodes_to_process) < BATCH_SIZE and budget_remaining > 0: + current_unit, activation, is_entry_point = queue.pop(0) + unit_id = str(current_unit["id"]) - if unit_id in visited: - continue + if unit_id not in visited: + visited.add(unit_id) + budget_remaining -= 1 + nodes_to_process.append((current_unit, activation, is_entry_point)) - visited.add(unit_id) - budget_remaining -= 1 + if not nodes_to_process: + break - # Increment access count + # Update access counts for batch + substep_start = time.time() + node_ids = [str(node[0]["id"]) for node in nodes_to_process] cursor.execute( - "UPDATE memory_units SET access_count = access_count + 1 WHERE id = %s", - (unit_id,) + "UPDATE memory_units SET access_count = access_count + 1 WHERE id::text = ANY(%s)", + (node_ids,) ) + update_access_time += time.time() - substep_start - # Calculate combined weight - event_date = current_unit["event_date"] - days_since = (utcnow() - event_date).total_seconds() / 86400 - - recency_weight = calculate_recency_weight(days_since) - frequency_weight = calculate_frequency_weight(current_unit.get("access_count", 0)) - - # Combined weight: activation * recency * frequency - final_weight = activation * recency_weight * frequency_weight - - # Notify tracer - if live_tracer: - live_tracer.visit_node( - node_id=unit_id, - text=current_unit["text"], - activation=activation, - recency=recency_weight, - frequency=frequency_weight, - weight=final_weight, - is_entry_point=is_entry_point, - ) - import time - time.sleep(0.15) # Slow down for visualization - - results.append({ - "id": unit_id, - "text": current_unit["text"], - "context": current_unit.get("context", ""), - "event_date": event_date.isoformat(), - "weight": final_weight, - "activation": activation, - "recency": recency_weight, - "frequency": frequency_weight, - }) - - # Spread to neighbors + # Query neighbors for ALL nodes in batch at once + substep_start = time.time() cursor.execute( """ - SELECT ml.to_unit_id, ml.weight, mu.text, mu.context, mu.event_date, mu.access_count + SELECT ml.from_unit_id, ml.to_unit_id, ml.weight, + mu.text, mu.context, mu.event_date, mu.access_count, mu.embedding FROM memory_links ml JOIN memory_units mu ON ml.to_unit_id = mu.id - WHERE ml.from_unit_id = %s + WHERE ml.from_unit_id::text = ANY(%s) AND ml.weight >= 0.1 - ORDER BY ml.weight DESC + ORDER BY ml.from_unit_id, ml.weight DESC """, - (unit_id,) + (node_ids,) ) + all_neighbors = cursor.fetchall() + query_neighbors_time += time.time() - substep_start - neighbors = cursor.fetchall() - for neighbor in neighbors: - neighbor_id = str(neighbor["to_unit_id"]) - if neighbor_id not in visited: - link_weight = neighbor["weight"] - new_activation = activation * link_weight * 0.8 # 0.8 = decay factor + # Group neighbors by from_unit_id + substep_start = time.time() + neighbors_by_node = {} + for neighbor in all_neighbors: + from_id = str(neighbor["from_unit_id"]) + if from_id not in neighbors_by_node: + neighbors_by_node[from_id] = [] + neighbors_by_node[from_id].append(neighbor) - if new_activation > 0.1: - queue.append(({ - "id": neighbor["to_unit_id"], - "text": neighbor["text"], - "context": neighbor.get("context", ""), - "event_date": neighbor["event_date"], - "access_count": neighbor["access_count"], - }, new_activation, False)) # Not an entry point + # Process each node in the batch + for current_unit, activation, is_entry_point in nodes_to_process: + unit_id = str(current_unit["id"]) + + # Calculate combined weight + event_date = current_unit["event_date"] + days_since = (utcnow() - event_date).total_seconds() / 86400 + + recency_weight = calculate_recency_weight(days_since) + frequency_weight = calculate_frequency_weight(current_unit.get("access_count", 0)) + + # Normalize frequency to [0, 1] range + frequency_normalized = (frequency_weight - 1.0) / 1.0 + + # Calculate semantic similarity between query and this memory + memory_embedding = current_unit.get("embedding") + if memory_embedding is not None: + # Cosine similarity = 1 - cosine distance + query_vec = np.array(query_embedding) + memory_vec = np.array(memory_embedding) + # Cosine similarity + dot_product = np.dot(query_vec, memory_vec) + norm_query = np.linalg.norm(query_vec) + norm_memory = np.linalg.norm(memory_vec) + semantic_similarity = dot_product / (norm_query * norm_memory) if norm_query > 0 and norm_memory > 0 else 0.0 + else: + semantic_similarity = 0.0 + + # Combined weight: 30% activation, 30% semantic similarity, 25% recency, 15% frequency + final_weight = 0.3 * activation + 0.3 * semantic_similarity + 0.25 * recency_weight + 0.15 * frequency_normalized + + # Notify tracer + if live_tracer: + live_tracer.visit_node( + node_id=unit_id, + text=current_unit["text"], + activation=activation, + recency=recency_weight, + frequency=frequency_weight, + weight=final_weight, + is_entry_point=is_entry_point, + ) + + results.append({ + "id": unit_id, + "text": current_unit["text"], + "context": current_unit.get("context", ""), + "event_date": event_date.isoformat(), + "weight": final_weight, + "activation": activation, + "semantic_similarity": semantic_similarity, + "recency": recency_weight, + "frequency": frequency_weight, + }) + + # Spread to neighbors (from batch query results) + neighbors = neighbors_by_node.get(unit_id, []) + for neighbor in neighbors: + neighbor_id = str(neighbor["to_unit_id"]) + if neighbor_id not in visited: + link_weight = neighbor["weight"] + new_activation = activation * link_weight * 0.8 # 0.8 = decay factor + + if new_activation > 0.1: + queue.append(({ + "id": neighbor["to_unit_id"], + "text": neighbor["text"], + "context": neighbor.get("context", ""), + "event_date": neighbor["event_date"], + "access_count": neighbor["access_count"], + "embedding": neighbor.get("embedding"), + }, new_activation, False)) # Not an entry point + + calculate_weight_time += time.time() - substep_start + process_neighbors_time += time.time() - substep_start + + # Clear batch for next iteration + nodes_to_process = [] + + spreading_activation_time = time.time() - step_start + num_batches = (len(visited) + BATCH_SIZE - 1) // BATCH_SIZE # Ceiling division + print(f" [3] Spreading activation: {len(visited)} nodes visited in {spreading_activation_time:.3f}s") + print(f" [3.1] Update access counts: {update_access_time:.3f}s") + print(f" [3.2] Calculate weights: {calculate_weight_time:.3f}s") + print(f" [3.3] Query neighbors: {query_neighbors_time:.3f}s ({num_batches} batched queries)") + print(f" [3.4] Process neighbors: {process_neighbors_time:.3f}s") + + step_start = time.time() + self.conn.commit() + print(f" [4] Commit: {time.time() - step_start:.3f}s") + + # Step 4: Sort by final weight and return top results + step_start = time.time() + results.sort(key=lambda x: x["weight"], reverse=True) + top_results = results[:top_k] + print(f" [5] Sort and return top {top_k}: {time.time() - step_start:.3f}s") + + print(f"[SEARCH] Complete: {len(top_results)} results in {time.time() - search_start:.3f}s\n") + return top_results + + except Exception as e: + print(f"[SEARCH] ERROR after {time.time() - search_start:.3f}s: {str(e)}") + self.conn.rollback() + raise Exception(f"Failed to search memories: {str(e)}") + finally: + cursor.close() + + def delete_agent(self, agent_id: str) -> Dict[str, int]: + """ + Delete all data for a specific agent (multi-tenant cleanup). + + This is much more efficient than dropping all tables and allows + multiple agents to coexist in the same database. + + Deletes (with CASCADE): + - All memory units for this agent + - All entities for this agent + - All associated links, unit-entity associations, and co-occurrences + + Args: + agent_id: Agent ID to delete + + Returns: + Dictionary with counts of deleted items + """ + cursor = self.conn.cursor() + + try: + # Count before deletion for reporting + cursor.execute("SELECT COUNT(*) FROM memory_units WHERE agent_id = %s", (agent_id,)) + units_count = cursor.fetchone()[0] + + cursor.execute("SELECT COUNT(*) FROM entities WHERE agent_id = %s", (agent_id,)) + entities_count = cursor.fetchone()[0] + + # Delete memory units (cascades to unit_entities, memory_links) + cursor.execute("DELETE FROM memory_units WHERE agent_id = %s", (agent_id,)) + + # Delete entities (cascades to unit_entities, entity_cooccurrences, memory_links with entity_id) + cursor.execute("DELETE FROM entities WHERE agent_id = %s", (agent_id,)) self.conn.commit() - # Step 4: Sort by final weight and return top results - results.sort(key=lambda x: x["weight"], reverse=True) - return results[:top_k] + return { + "memory_units_deleted": units_count, + "entities_deleted": entities_count + } except Exception as e: self.conn.rollback() - raise Exception(f"Failed to search memories: {str(e)}") + raise Exception(f"Failed to delete agent data: {str(e)}") finally: cursor.close() @@ -743,76 +958,145 @@ class TemporalSemanticMemory: finally: cursor.close() - def _extract_entities_for_batch( + def _extract_entities_batch_optimized( self, cursor, agent_id: str, - unit_id: str, - text: str, + unit_ids: List[str], + sentences: List[str], context: str, - event_date, - all_sentences: List[str], + fact_dates: List, ) -> List[tuple]: """ - Extract entities and return entity links (doesn't insert yet). + Extract entities from ALL sentences in one batch (MUCH faster than sequential). + + Uses spaCy's batch processing to extract entities from all texts at once, + then resolves and links them in bulk. Returns list of tuples for batch insertion: (from_unit_id, to_unit_id, link_type, weight, entity_id) """ - from .entity_resolver import extract_entities + from .entity_resolver import extract_entities_batch try: - # Extract entities from this unit - entities = extract_entities(text) + # Step 1: Extract entities from ALL sentences in one batch (fast!) + substep_start = time.time() + all_entities = extract_entities_batch(sentences) + total_entities = sum(len(ents) for ents in all_entities) + print(f" [6.1] spaCy NER (batch): {total_entities} entities from {len(sentences)} sentences in {time.time() - substep_start:.3f}s") - if not entities: - return [] + # Step 2: Resolve entities in BATCH (much faster!) + substep_start = time.time() + step_6_2_start = time.time() - # Resolve each entity - entity_ids = [] - for entity in entities: - entity_id = self.entity_resolver.resolve_entity( - agent_id=agent_id, - entity_text=entity['text'], - entity_type=entity['type'], - context=context, - nearby_entities=entities, - unit_event_date=event_date - ) - entity_ids.append(entity_id) + # [6.2.1] Prepare all entities for batch resolution + substep_6_2_1_start = time.time() + all_entities_flat = [] + entity_to_unit = [] # Maps flat index to (unit_id, local_index) - # Link unit to entity (this inserts into entity_units) - self.entity_resolver.link_unit_to_entity(unit_id, entity_id) + for unit_id, entities, fact_date in zip(unit_ids, all_entities, fact_dates): + if not entities: + continue - # Now collect entity links for batch insertion - # After link_unit_to_entity has been called, entity_units should exist - links = [] - for entity_id in set(entity_ids): - # Find all other units with this entity (cursor must be fresh) - try: - cursor.execute( - """ - SELECT unit_id - FROM unit_entities - WHERE entity_id = %s AND unit_id != %s - """, - (entity_id, unit_id) + for local_idx, entity in enumerate(entities): + all_entities_flat.append({ + 'text': entity['text'], + 'type': entity['type'], + 'nearby_entities': entities, + }) + entity_to_unit.append((unit_id, local_idx, fact_date)) + print(f" [6.2.1] Prepare entities: {len(all_entities_flat)} entities in {time.time() - substep_6_2_1_start:.3f}s") + + # Resolve ALL entities in one batch call + if all_entities_flat: + # [6.2.2] Batch resolve entities + substep_6_2_2_start = time.time() + # Group by date for batch resolution (most will have same date) + entities_by_date = {} + for idx, (unit_id, local_idx, fact_date) in enumerate(entity_to_unit): + date_key = fact_date + if date_key not in entities_by_date: + entities_by_date[date_key] = [] + entities_by_date[date_key].append((idx, all_entities_flat[idx])) + + # Resolve each date group in batch + resolved_entity_ids = [None] * len(all_entities_flat) + for fact_date, entities_group in entities_by_date.items(): + indices = [idx for idx, _ in entities_group] + entities_data = [entity_data for _, entity_data in entities_group] + + batch_resolved = self.entity_resolver.resolve_entities_batch( + agent_id=agent_id, + entities_data=entities_data, + context=context, + unit_event_date=fact_date ) - related_units = cursor.fetchall() - for (related_unit_id,) in related_units: + for idx, entity_id in zip(indices, batch_resolved): + resolved_entity_ids[idx] = entity_id + print(f" [6.2.2] Resolve entities: {len(all_entities_flat)} entities in {time.time() - substep_6_2_2_start:.3f}s") + + # [6.2.3] Create unit-entity links in BATCH + substep_6_2_3_start = time.time() + # Map resolved entities back to units and collect all (unit, entity) pairs + unit_to_entity_ids = {} + unit_entity_pairs = [] + for idx, (unit_id, local_idx, fact_date) in enumerate(entity_to_unit): + if unit_id not in unit_to_entity_ids: + unit_to_entity_ids[unit_id] = [] + + entity_id = resolved_entity_ids[idx] + unit_to_entity_ids[unit_id].append(entity_id) + unit_entity_pairs.append((unit_id, entity_id)) + + # Batch insert all unit-entity links (MUCH faster!) + self.entity_resolver.link_units_to_entities_batch(unit_entity_pairs) + print(f" [6.2.3] Create unit-entity links (batched): {len(unit_entity_pairs)} links in {time.time() - substep_6_2_3_start:.3f}s") + + print(f" [6.2] Entity resolution (batched): {len(all_entities_flat)} entities resolved in {time.time() - step_6_2_start:.3f}s") + else: + unit_to_entity_ids = {} + print(f" [6.2] Entity resolution (batched): 0 entities in {time.time() - step_6_2_start:.3f}s") + + # Step 3: Create entity links between units that share entities + substep_start = time.time() + # Collect all unique entity IDs + all_entity_ids = set() + for entity_ids in unit_to_entity_ids.values(): + all_entity_ids.update(entity_ids) + + # For each entity, find all units that reference it (one query per entity) + entity_to_units = {} + for entity_id in all_entity_ids: + cursor.execute( + """ + SELECT unit_id + FROM unit_entities + WHERE entity_id = %s + """, + (entity_id,) + ) + entity_to_units[entity_id] = [row[0] for row in cursor.fetchall()] + + # Create bidirectional links between units that share entities + links = [] + for entity_id, units_with_entity in entity_to_units.items(): + # For each pair of units with this entity, create bidirectional links + for i, unit_id_1 in enumerate(units_with_entity): + for unit_id_2 in units_with_entity[i+1:]: # Bidirectional links - links.append((unit_id, related_unit_id, 'entity', 1.0, entity_id)) - links.append((related_unit_id, unit_id, 'entity', 1.0, entity_id)) - except Exception as query_error: - # If there's an error querying, just skip this entity - print(f"Warning: Failed to query entity_units for {entity_id}: {str(query_error)}") - continue + links.append((unit_id_1, unit_id_2, 'entity', 1.0, entity_id)) + links.append((unit_id_2, unit_id_1, 'entity', 1.0, entity_id)) + + print(f" [6.3] Entity link creation: {len(links)} links for {len(all_entity_ids)} unique entities in {time.time() - substep_start:.3f}s") return links except Exception as e: - print(f"Warning: Failed to extract entities: {str(e)}") - return [] + print(f"ERROR: Failed to extract entities in batch: {str(e)}") + import traceback + traceback.print_exc() + # Re-raise to trigger rollback at put_async level + raise def _create_temporal_links_batch( self, @@ -870,7 +1154,89 @@ class TemporalSemanticMemory: ) except Exception as e: - print(f"Warning: Failed to create temporal links: {str(e)}") + print(f"ERROR: Failed to create temporal links: {str(e)}") + import traceback + traceback.print_exc() + # Re-raise to trigger rollback at put_async level + raise + + def _create_temporal_links_batch_per_fact( + self, + cursor, + agent_id: str, + unit_ids: List[str], + time_window_hours: int = 24, + ): + """ + Create temporal links for multiple units, each with their own event_date. + + Queries the event_date for each unit from the database and creates temporal + links based on individual dates (supports per-fact dating). + """ + if not unit_ids: + return + + try: + from psycopg2.extras import execute_values + + # Get the event_date for each new unit + cursor.execute( + """ + SELECT id, event_date + FROM memory_units + WHERE id::text = ANY(%s) + """, + (unit_ids,) + ) + new_units = {str(row[0]): row[1] for row in cursor.fetchall()} + + # Create links based on each unit's individual event_date + links = [] + for unit_id, unit_event_date in new_units.items(): + # Find units within the time window of THIS specific unit + cursor.execute( + """ + SELECT id, event_date + FROM memory_units + WHERE agent_id = %s + AND id != %s + AND event_date BETWEEN %s AND %s + ORDER BY event_date DESC + LIMIT 10 + """, + ( + agent_id, + unit_id, + unit_event_date - timedelta(hours=time_window_hours), + unit_event_date + timedelta(hours=time_window_hours) + ) + ) + + recent_units = cursor.fetchall() + + for recent_id, recent_event_date in recent_units: + # Calculate temporal proximity weight + time_diff_hours = abs((unit_event_date - recent_event_date).total_seconds() / 3600) + weight = max(0.3, 1.0 - (time_diff_hours / time_window_hours)) + links.append((unit_id, recent_id, 'temporal', weight, None)) + + if links: + execute_values( + cursor, + """ + INSERT INTO memory_links (from_unit_id, to_unit_id, link_type, weight, entity_id) + VALUES %s + ON CONFLICT (from_unit_id, to_unit_id, link_type, COALESCE(entity_id, '00000000-0000-0000-0000-000000000000'::uuid)) DO NOTHING + """, + links + ) + + except Exception as e: + print(f"ERROR: Failed to create temporal links: {str(e)}") + import traceback + traceback.print_exc() + # Re-raise to trigger rollback at put_async level + raise def _create_semantic_links_batch( self, @@ -927,7 +1293,11 @@ class TemporalSemanticMemory: ) except Exception as e: - print(f"Warning: Failed to create semantic links: {str(e)}") + print(f"ERROR: Failed to create semantic links: {str(e)}") + import traceback + traceback.print_exc() + # Re-raise to trigger rollback at put_async level + raise def _insert_entity_links_batch(self, cursor, links: List[tuple]): """Insert all entity links in a single batch.""" diff --git a/memory/utils.py b/memory/utils.py index 0593cec9..d7bb6126 100644 --- a/memory/utils.py +++ b/memory/utils.py @@ -1,24 +1,28 @@ """ Utility functions for memory system. """ -from typing import List +from datetime import datetime +from typing import List, Dict from .llm_client import extract_facts_from_text -async def extract_facts(text: str) -> List[str]: +async def extract_facts(text: str, event_date: datetime, context: str = "") -> List[Dict[str, str]]: """ Extract semantic facts from text using LLM. Uses LLM for intelligent fact extraction that: - Filters out social pleasantries and filler words - - Creates self-contained statements + - Creates self-contained statements with absolute dates - Handles conversational text well + - Resolves relative time expressions to absolute dates Args: text: Input text (conversation, article, etc.) + event_date: Reference date for resolving relative times + context: Context about the conversation/document Returns: - List of factual statements + List of fact dictionaries with keys: 'fact' (text) and 'date' (ISO string) Raises: Exception: If LLM fact extraction fails @@ -26,14 +30,12 @@ async def extract_facts(text: str) -> List[str]: if not text or not text.strip(): return [] - fact_dicts = await extract_facts_from_text(text) - # Extract just the fact text - facts = [f['fact'] for f in fact_dicts if f.get('fact')] + fact_dicts = await extract_facts_from_text(text, event_date, context) - if not facts: + if not fact_dicts: raise Exception(f"LLM extracted 0 facts from text of length {len(text)}. This may indicate the text contains no meaningful information, or the LLM failed to extract facts.") - return facts + return fact_dicts def cosine_similarity(vec1: List[float], vec2: List[float]) -> float: diff --git a/memory_graph_interactive.html b/memory_graph_interactive.html index ee7948df..94031fd3 100644 --- a/memory_graph_interactive.html +++ b/memory_graph_interactive.html @@ -1,181 +1,423 @@ + + -
- - - - - - - -| ID | +Text | +Context | +Date | +Entities | +
|---|