control plane and api issues

docker/standalone/Dockerfile

@@ -64,14 +64,21 @@ FROM python:3.11-slim
 
 WORKDIR /app
 
-# Install Node.js, curl, and uv
+# Install Node.js, curl, uv, and pg0 dependencies
 RUN apt-get update && apt-get install -y \
     curl \
+    libxml2 \
+    libssl3 \
+    libgssapi-krb5-2 \
+    && apt-get install -y libicu72 || apt-get install -y libicu74 || apt-get install -y libicu* \
     && curl -fsSL https://deb.nodesource.com/setup_20.x | bash - \
     && apt-get install -y nodejs \
     && rm -rf /var/lib/apt/lists/* \
     && pip install --no-cache-dir uv
 
+# Create non-root user (PostgreSQL cannot run as root)
+RUN useradd -m -s /bin/bash hindsight
+
 # Copy API with virtual environment from builder
 COPY --from=api-base /app/api /app/api
 
@@ -96,13 +103,19 @@ WORKDIR /app
 COPY docker/standalone/start-all.sh /app/start-all.sh
 RUN chmod +x /app/start-all.sh
 
-# Create data directory for pg0
-RUN mkdir -p /app/data
+# Create data directory for pg0 and set ownership
+RUN mkdir -p /app/data && chown -R hindsight:hindsight /app
 
-# Install pg0 to /root/.hindsight/bin/pg0
-RUN mkdir -p /root/.hindsight/bin /root/.local/bin && \
-    export PATH="/root/.hindsight/bin:/root/.local/bin:$PATH" && \
-    curl -fsSL https://raw.githubusercontent.com/vectorize-io/pg0/main/install.sh | bash
+# Switch to non-root user
+USER hindsight
+
+# Install pg0
+RUN curl -fsSL https://raw.githubusercontent.com/vectorize-io/pg0/main/install.sh | bash
+
+# Start pg0 once to verify it works and pre-download PostgreSQL libraries
+RUN pg0 --help && \
+    pg0 start --wait && \
+    pg0 stop
 
 # Expose ports
 EXPOSE 8888 3000
@@ -113,7 +126,7 @@ ENV HINDSIGHT_API_PORT=8888
 ENV HINDSIGHT_API_LOG_LEVEL=info
 ENV NODE_ENV=production
 ENV HINDSIGHT_CP_DATAPLANE_API_URL=http://localhost:8888
-ENV PATH="/app/api/.venv/bin:$PATH"
+ENV PATH="/home/hindsight/.local/bin:/app/api/.venv/bin:${PATH}"
 
 # Run startup script
 CMD ["/app/start-all.sh"]

hindsight-api/alembic/versions/c8e5f2a3b4d1_add_retain_params_to_documents.py

@@ -0,0 +1,39 @@
+"""add_retain_params_to_documents
+
+Revision ID: c8e5f2a3b4d1
+Revises: b7c4d8e9f1a2
+Create Date: 2025-12-02 00:00:00.000000
+
+"""
+from typing import Sequence, Union
+
+from alembic import op
+import sqlalchemy as sa
+from sqlalchemy.dialects import postgresql
+
+
+# revision identifiers, used by Alembic.
+revision: str = 'c8e5f2a3b4d1'
+down_revision: Union[str, Sequence[str], None] = 'b7c4d8e9f1a2'
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    """Add retain_params JSONB column to documents table."""
+
+    # Add retain_params column to store parameters passed during retain
+    op.add_column('documents', sa.Column('retain_params', postgresql.JSONB(), nullable=True))
+
+    # Add index for efficient queries on retain_params
+    op.create_index('idx_documents_retain_params', 'documents', ['retain_params'], postgresql_using='gin')
+
+
+def downgrade() -> None:
+    """Remove retain_params column from documents table."""
+
+    # Drop index
+    op.drop_index('idx_documents_retain_params', table_name='documents')
+
+    # Drop column
+    op.drop_column('documents', 'retain_params')

hindsight-api/hindsight_api/api/http.py

@@ -84,7 +84,7 @@ class RecallRequest(BaseModel):
     model_config = ConfigDict(json_schema_extra={
         "example": {
             "query": "What did Alice say about machine learning?",
-            "types": ["world", "agent"],
+            "types": ["world", "bank"],
             "budget": "mid",
             "max_tokens": 4096,
             "trace": True,
@@ -279,7 +279,8 @@ class MemoryItem(BaseModel):
             "content": "Alice mentioned she's working on a new ML model",
             "timestamp": "2024-01-15T10:30:00Z",
             "context": "team meeting",
-            "metadata": {"source": "slack", "channel": "engineering"}
+            "metadata": {"source": "slack", "channel": "engineering"},
+            "document_id": "meeting_notes_2024_01_15"
         }
     })
 
@@ -287,6 +288,10 @@ class MemoryItem(BaseModel):
     timestamp: Optional[datetime] = None
     context: Optional[str] = None
     metadata: Optional[Dict[str, str]] = None
+    document_id: Optional[str] = Field(
+        default=None,
+        description="Optional document ID for this memory item."
+    )
 
 
 class RetainRequest(BaseModel):
@@ -296,20 +301,20 @@ class RetainRequest(BaseModel):
             "items": [
                 {
                     "content": "Alice works at Google",
-                    "context": "work"
+                    "context": "work",
+                    "document_id": "conversation_123"
                 },
                 {
                     "content": "Bob went hiking yesterday",
-                    "timestamp": "2024-01-15T10:00:00Z"
+                    "timestamp": "2024-01-15T10:00:00Z",
+                    "document_id": "conversation_123"
                 }
             ],
-            "document_id": "conversation_123",
             "async": False
         }
     })
 
     items: List[MemoryItem]
-    document_id: Optional[str] = None
     async_: bool = Field(
         default=False,
         alias="async",
@@ -325,7 +330,6 @@ class RetainResponse(BaseModel):
             "example": {
                 "success": True,
                 "bank_id": "user123",
-                "document_id": "conversation_123",
                 "items_count": 2,
                 "async": False
             }
@@ -334,7 +338,6 @@ class RetainResponse(BaseModel):
 
     success: bool
     bank_id: str
-    document_id: Optional[str] = None
     items_count: int
     async_: bool = Field(alias="async", serialization_alias="async", description="Whether the operation was processed asynchronously")
 
@@ -414,7 +417,7 @@ class ReflectResponse(BaseModel):
                 {
                     "id": "456",
                     "text": "I discussed AI applications last week",
-                    "type": "agent"
+                    "type": "bank"
                 }
             ]
         }
@@ -680,6 +683,27 @@ class DocumentResponse(BaseModel):
     memory_unit_count: int
 
 
+class ChunkResponse(BaseModel):
+    """Response model for get chunk endpoint."""
+    model_config = ConfigDict(json_schema_extra={
+        "example": {
+            "chunk_id": "user123_session_1_0",
+            "document_id": "session_1",
+            "bank_id": "user123",
+            "chunk_index": 0,
+            "chunk_text": "This is the first chunk of the document...",
+            "created_at": "2024-01-15T10:30:00Z"
+        }
+    })
+
+    chunk_id: str
+    document_id: str
+    bank_id: str
+    chunk_index: int
+    chunk_text: str
+    created_at: str
+
+
 class DeleteResponse(BaseModel):
     """Response model for delete operations."""
     model_config = ConfigDict(json_schema_extra={
@@ -859,9 +883,8 @@ def _register_routes(app: FastAPI):
 
     The type parameter is optional and must be one of:
     - 'world': General knowledge about people, places, events, and things that happen
-    - 'agent': Memories about what the AI agent did, actions taken, and tasks performed
+    - 'bank': Memories about what the AI agent did, actions taken, and tasks performed
     - 'opinion': The bank's formed beliefs, perspectives, and viewpoints
-    - 'observation': Synthesized observations about entities (generated automatically)
 
     Set include_entities=true to get entity observations alongside recall results.
         """,
@@ -873,10 +896,10 @@ def _register_routes(app: FastAPI):
 
         try:
             # Validate types
-            valid_fact_types = ["world", "agent", "opinion", "observation"]
+            valid_fact_types = ["world", "bank", "opinion"]
 
             # Default to world, agent, opinion if not specified (exclude observation by default)
-            fact_types = request.types if request.types else ["world", "agent", "opinion"]
+            fact_types = request.types if request.types else ["world", "bank", "opinion"]
             for ft in fact_types:
                 if ft not in valid_fact_types:
                     raise HTTPException(
@@ -1367,6 +1390,34 @@ def _register_routes(app: FastAPI):
             raise HTTPException(status_code=500, detail=str(e))
 
 
+    @app.get(
+        "/v1/default/chunks/{chunk_id}",
+        response_model=ChunkResponse,
+        summary="Get chunk details",
+        description="Get a specific chunk by its ID",
+        operation_id="get_chunk"
+    )
+    async def api_get_chunk(chunk_id: str):
+        """
+        Get a specific chunk with its text.
+
+        Args:
+            chunk_id: Chunk ID (from path, format: bank_id_document_id_chunk_index)
+        """
+        try:
+            chunk = await app.state.memory.get_chunk(chunk_id)
+            if not chunk:
+                raise HTTPException(status_code=404, detail="Chunk not found")
+            return chunk
+        except HTTPException:
+            raise
+        except Exception as e:
+            import traceback
+            error_detail = f"{str(e)}\n\nTraceback:\n{traceback.format_exc()}"
+            logger.error(f"Error in /v1/default/chunks/{chunk_id}: {error_detail}")
+            raise HTTPException(status_code=500, detail=str(e))
+
+
     @app.delete(
         "/v1/default/banks/{bank_id}/documents/{document_id}",
         summary="Delete a document",
@@ -1517,10 +1568,12 @@ This operation cannot be undone.
         """Get memory bank profile (personality + background)."""
         try:
             profile = await app.state.memory.get_bank_profile(bank_id)
+            # Convert PersonalityTraits object to dict for Pydantic
+            personality_dict = profile["personality"].model_dump() if hasattr(profile["personality"], 'model_dump') else dict(profile["personality"])
             return BankProfileResponse(
                 bank_id=bank_id,
                 name=profile["name"],
-                personality=profile["personality"],  # Already a PersonalityTraits object
+                personality=PersonalityTraits(**personality_dict),
                 background=profile["background"]
             )
         except Exception as e:
@@ -1550,10 +1603,11 @@ This operation cannot be undone.
 
             # Get updated profile
             profile = await app.state.memory.get_bank_profile(bank_id)
+            personality_dict = profile["personality"].model_dump() if hasattr(profile["personality"], 'model_dump') else dict(profile["personality"])
             return BankProfileResponse(
                 bank_id=bank_id,
                 name=profile["name"],
-                personality=profile["personality"],  # Already a PersonalityTraits object
+                personality=PersonalityTraits(**personality_dict),
                 background=profile["background"]
             )
         except Exception as e:
@@ -1638,7 +1692,7 @@ This operation cannot be undone.
                 async with acquire_with_retry(pool) as conn:
                     await conn.execute(
                         """
-                        UPDATE agents
+                        UPDATE banks
                         SET background = $2,
                             updated_at = NOW()
                         WHERE bank_id = $1
@@ -1650,10 +1704,11 @@ This operation cannot be undone.
 
             # Get final profile
             final_profile = await app.state.memory.get_bank_profile(bank_id)
+            personality_dict = final_profile["personality"].model_dump() if hasattr(final_profile["personality"], 'model_dump') else dict(final_profile["personality"])
             return BankProfileResponse(
                 bank_id=bank_id,
                 name=final_profile["name"],
-                personality=final_profile["personality"],  # Already a PersonalityTraits object
+                personality=PersonalityTraits(**personality_dict),
                 background=final_profile["background"]
             )
         except Exception as e:
@@ -1677,7 +1732,7 @@ This operation cannot be undone.
     - Efficient batch processing
     - Automatic fact extraction from natural language
     - Entity recognition and linking
-    - Document tracking with automatic upsert (when document_id is provided)
+    - Document tracking with automatic upsert (when document_id is provided on items)
     - Temporal and semantic linking
     - Optional asynchronous processing
 
@@ -1697,7 +1752,7 @@ This operation cannot be undone.
     - Waits for processing to complete
     - Returns after all memories are stored
 
-    Note: If document_id is provided and already exists, the old document and its memory units will be deleted before creating new ones (upsert behavior).
+    Note: If a memory item has a document_id that already exists, the old document and its memory units will be deleted before creating new ones (upsert behavior). Items with the same document_id are grouped together for efficient processing.
         """,
         operation_id="retain_memories"
     )
@@ -1716,6 +1771,8 @@ This operation cannot be undone.
                     content_dict["context"] = item.context
                 if item.metadata:
                     content_dict["metadata"] = item.metadata
+                if item.document_id:
+                    content_dict["document_id"] = item.document_id
                 contents.append(content_dict)
 
             if request.async_:
@@ -1727,14 +1784,13 @@ This operation cannot be undone.
                 async with acquire_with_retry(pool) as conn:
                     await conn.execute(
                         """
-                        INSERT INTO async_operations (id, bank_id, task_type, items_count, document_id)
-                        VALUES ($1, $2, $3, $4, $5)
+                        INSERT INTO async_operations (id, bank_id, task_type, items_count)
+                        VALUES ($1, $2, $3, $4)
                         """,
                         operation_id,
                         bank_id,
                         'retain',
-                        len(contents),
-                        request.document_id
+                        len(contents)
                     )
 
                 # Submit task to background queue
@@ -1742,8 +1798,7 @@ This operation cannot be undone.
                     'type': 'batch_put',
                     'operation_id': str(operation_id),
                     'bank_id': bank_id,
-                    'contents': contents,
-                    'document_id': request.document_id
+                    'contents': contents
                 })
 
                 logging.info(f"Retain task queued for bank_id={bank_id}, {len(contents)} items, operation_id={operation_id}")
@@ -1751,7 +1806,6 @@ This operation cannot be undone.
                 return RetainResponse(
                     success=True,
                     bank_id=bank_id,
-                    document_id=request.document_id,
                     items_count=len(contents),
                     async_=True
                 )
@@ -1760,14 +1814,12 @@ This operation cannot be undone.
                 with metrics.record_operation("retain", bank_id=bank_id):
                     result = await app.state.memory.retain_batch_async(
                         bank_id=bank_id,
-                        contents=contents,
-                        document_id=request.document_id
+                        contents=contents
                     )
 
                 return RetainResponse(
                     success=True,
                     bank_id=bank_id,
-                    document_id=request.document_id,
                     items_count=len(contents),
                     async_=False
                 )

hindsight-api/hindsight_api/engine/llm_wrapper.py

@@ -13,6 +13,9 @@ logger = logging.getLogger(__name__)
 # Disable httpx logging
 logging.getLogger("httpx").setLevel(logging.WARNING)
 
+# Global semaphore to limit concurrent LLM requests across all instances
+_global_llm_semaphore = asyncio.Semaphore(32)
+
 
 class OutputTooLongError(Exception):
     """
@@ -69,12 +72,13 @@ class LLMConfig:
             )
 
         # Create client (private - use .call() method instead)
+        # Disable automatic retries - we handle retries in the call() method
         if self.provider == "ollama":
-            self._client = AsyncOpenAI(api_key="ollama", base_url=self.base_url)
+            self._client = AsyncOpenAI(api_key="ollama", base_url=self.base_url, max_retries=0)
         elif self.base_url:
-            self._client = AsyncOpenAI(api_key=self.api_key, base_url=self.base_url)
+            self._client = AsyncOpenAI(api_key=self.api_key, base_url=self.base_url, max_retries=0)
         else:
-            self._client = AsyncOpenAI(api_key=self.api_key)
+            self._client = AsyncOpenAI(api_key=self.api_key, max_retries=0)
 
         logger.info(
             f"Initialized LLM: provider={self.provider}, model={self.model}, base_url={self.base_url}"
@@ -109,106 +113,107 @@ class LLMConfig:
         Raises:
             Exception: Re-raises any API errors after all retries are exhausted
         """
-        start_time = time.time()
+        # Use global semaphore to limit concurrent requests
+        async with _global_llm_semaphore:
+            start_time = time.time()
 
-        call_params = {
-            "model": self.model,
-            "messages": messages,
-            **kwargs
-        }
-        if self.provider == "groq":
-            call_params["extra_body"] = {
-                "service_tier": "auto",
-                "reasoning_effort": "low",  # Reduce reasoning overhead
-                "include_reasoning": False,  # Disable hidden reasoning tokens
+            call_params = {
+                "model": self.model,
+                "messages": messages,
+                **kwargs
             }
+            if self.provider == "groq":
+                call_params["extra_body"] = {
+                    "service_tier": "auto",
+                    "reasoning_effort": "low",  # Reduce reasoning overhead
+                    "include_reasoning": False,  # Disable hidden reasoning tokens
+                }
 
-        last_exception = None
+            last_exception = None
 
-        for attempt in range(max_retries + 1):
-            try:
-                # Use the appropriate response format
-                if response_format is not None:
-                    # Use JSON mode instead of strict parse for flexibility with optional fields
-                    # This allows the LLM to omit optional fields without validation errors
-                    import json
+            for attempt in range(max_retries + 1):
+                try:
+                    # Use the appropriate response format
+                    if response_format is not None:
+                        # Use JSON mode instead of strict parse for flexibility with optional fields
+                        # This allows the LLM to omit optional fields without validation errors
+                        import json
 
-                    # Add schema to the system message
-                    if hasattr(response_format, 'model_json_schema'):
-                        schema = response_format.model_json_schema()
-                        schema_msg = f"\n\nYou must respond with valid JSON matching this schema:\n{json.dumps(schema, indent=2)}"
+                        # Add schema to the system message
+                        if hasattr(response_format, 'model_json_schema'):
+                            schema = response_format.model_json_schema()
+                            schema_msg = f"\n\nYou must respond with valid JSON matching this schema:\n{json.dumps(schema, indent=2)}"
 
-                        # Add schema to the system message if present, otherwise prepend as user message
-                        if call_params['messages'] and call_params['messages'][0].get('role') == 'system':
-                            call_params['messages'][0]['content'] += schema_msg
+                            # Add schema to the system message if present, otherwise prepend as user message
+                            if call_params['messages'] and call_params['messages'][0].get('role') == 'system':
+                                call_params['messages'][0]['content'] += schema_msg
+                            else:
+                                # No system message, add schema instruction to first user message
+                                if call_params['messages']:
+                                    call_params['messages'][0]['content'] = schema_msg + "\n\n" + call_params['messages'][0]['content']
+
+                        call_params['response_format'] = {"type": "json_object"}
+                        response = await self._client.chat.completions.create(**call_params)
+
+                        # Parse the JSON response
+                        content = response.choices[0].message.content
+                        json_data = json.loads(content)
+
+                        # Return raw JSON if skip_validation is True, otherwise validate with Pydantic
+                        if skip_validation:
+                            result = json_data
                         else:
-                            # No system message, add schema instruction to first user message
-                            if call_params['messages']:
-                                call_params['messages'][0]['content'] = schema_msg + "\n\n" + call_params['messages'][0]['content']
-
-                    call_params['response_format'] = {"type": "json_object"}
-                    response = await self._client.chat.completions.create(**call_params)
-
-                    # Parse the JSON response
-                    content = response.choices[0].message.content
-                    json_data = json.loads(content)
-
-                    # Return raw JSON if skip_validation is True, otherwise validate with Pydantic
-                    if skip_validation:
-                        result = json_data
+                            result = response_format.model_validate(json_data)
                     else:
-                        result = response_format.model_validate(json_data)
-                else:
-                    # Standard completion and return text content
-                    response = await self._client.chat.completions.create(**call_params)
-                    result = response.choices[0].message.content
+                        # Standard completion and return text content
+                        response = await self._client.chat.completions.create(**call_params)
+                        result = response.choices[0].message.content
 
-                # Log call details only if it takes more than 5 seconds
-                duration = time.time() - start_time
-                usage = response.usage
-                if duration > 10.0:
-                    ratio = max(1, usage.completion_tokens) / usage.prompt_tokens
-                    logger.info(
-                        f"slow llm call: model={self.provider}/{self.model}, "
-                        f"input_tokens={usage.prompt_tokens}, output_tokens={usage.completion_tokens}, "
-                        f"total_tokens={usage.total_tokens}, time={duration:.3f}s, ratio out/in={ratio:.2f}"
-                    )
+                    # Log call details only if it takes more than 5 seconds
+                    duration = time.time() - start_time
+                    usage = response.usage
+                    if duration > 10.0:
+                        ratio = max(1, usage.completion_tokens) / usage.prompt_tokens
+                        logger.info(
+                            f"slow llm call: model={self.provider}/{self.model}, "
+                            f"input_tokens={usage.prompt_tokens}, output_tokens={usage.completion_tokens}, "
+                            f"total_tokens={usage.total_tokens}, time={duration:.3f}s, ratio out/in={ratio:.2f}"
+                        )
 
-                return result
+                    return result
 
-            except LengthFinishReasonError as e:
-                # Output exceeded token limits - raise bridge exception for caller to handle
-                logger.warning(f"LLM output exceeded token limits: {str(e)}")
-                raise OutputTooLongError(
-                    f"LLM output exceeded token limits. Input may need to be split into smaller chunks."
-                ) from e
+                except LengthFinishReasonError as e:
+                    # Output exceeded token limits - raise bridge exception for caller to handle
+                    logger.warning(f"LLM output exceeded token limits: {str(e)}")
+                    raise OutputTooLongError(
+                        f"LLM output exceeded token limits. Input may need to be split into smaller chunks."
+                    ) from e
 
-            except APIStatusError as e:
-                last_exception = e
-                if attempt < max_retries:
-                    # Calculate exponential backoff with jitter
-                    backoff = min(initial_backoff * (2 ** attempt), max_backoff)
-                    # Add jitter (±20%)
-                    jitter = backoff * 0.2 * (2 * (time.time() % 1) - 1)
-                    sleep_time = backoff + jitter
+                except APIStatusError as e:
+                    last_exception = e
+                    if attempt < max_retries:
+                        # Calculate exponential backoff with jitter
+                        backoff = min(initial_backoff * (2 ** attempt), max_backoff)
+                        # Add jitter (±20%)
+                        jitter = backoff * 0.2 * (2 * (time.time() % 1) - 1)
+                        sleep_time = backoff + jitter
 
-                    logger.warning(
-                        f"LLM error on attempt {attempt + 1}/{max_retries + 1}. "
-                        f"Retrying in {sleep_time:.2f}s... Error: {str(e)}"
-                    )
-                    await asyncio.sleep(sleep_time)
-                else:
-                    logger.error(f"Non-retryable API error after {max_retries + 1} attempts: {str(e)}")
+                        # Only log if it's a non-retryable error or final attempt
+                        # Silent retry for common transient errors like capacity exceeded
+                        await asyncio.sleep(sleep_time)
+                    else:
+                        # Log only on final failed attempt
+                        logger.error(f"API error after {max_retries + 1} attempts: {str(e)}")
+                        raise
+
+                except Exception as e:
+                    logger.error(f"Unexpected error during LLM call: {type(e).__name__}: {str(e)}")
                     raise
 
-            except Exception as e:
-                logger.error(f"Unexpected error during LLM call: {type(e).__name__}: {str(e)}")
-                raise
-
-        # This should never be reached, but just in case
-        if last_exception:
-            raise last_exception
-        raise RuntimeError(f"LLM call failed after all retries with no exception captured")
+            # This should never be reached, but just in case
+            if last_exception:
+                raise last_exception
+            raise RuntimeError(f"LLM call failed after all retries with no exception captured")
 
     @classmethod
     def for_memory(cls) -> "LLMConfig":

hindsight-api/hindsight_api/engine/memory_engine.py

@@ -11,7 +11,7 @@ This implements a sophisticated memory architecture that combines:
 import json
 import os
 from datetime import datetime, timedelta, timezone
-from typing import Any, Dict, List, Optional, Tuple, Union
+from typing import Any, Dict, List, Optional, Tuple, Union, TypedDict
 import asyncpg
 import asyncio
 from .embeddings import Embeddings, SentenceTransformersEmbeddings
@@ -22,6 +22,23 @@ import uuid
 import logging
 from pydantic import BaseModel, Field
 
+
+class RetainContentDict(TypedDict, total=False):
+    """Type definition for content items in retain_batch_async.
+
+    Fields:
+        content: Text content to store (required)
+        context: Context about the content (optional)
+        event_date: When the content occurred (optional, defaults to now)
+        metadata: Custom key-value metadata (optional)
+        document_id: Document ID for this content item (optional)
+    """
+    content: str  # Required
+    context: str
+    event_date: datetime
+    metadata: Dict[str, str]
+    document_id: str
+
 from .query_analyzer import QueryAnalyzer
 from .search.scoring import (
     calculate_recency_weight,
@@ -218,19 +235,17 @@ class MemoryEngine:
         Handler for batch retain tasks.
 
         Args:
-            task_dict: Dict with 'bank_id', 'contents', 'document_id'
+            task_dict: Dict with 'bank_id', 'contents'
         """
         try:
             bank_id = task_dict.get('bank_id')
             contents = task_dict.get('contents', [])
-            document_id = task_dict.get('document_id')
 
             logger.info(f"[BATCH_RETAIN_TASK] Starting background batch retain for bank_id={bank_id}, {len(contents)} items")
 
             await self.retain_batch_async(
                 bank_id=bank_id,
-                contents=contents,
-                document_id=document_id
+                contents=contents
             )
 
             logger.info(f"[BATCH_RETAIN_TASK] Completed background batch retain for bank_id={bank_id}")
@@ -604,15 +619,19 @@ class MemoryEngine:
         Returns:
             List of created unit IDs
         """
+        # Build content dict
+        content_dict: RetainContentDict = {
+            "content": content,
+            "context": context,
+            "event_date": event_date
+        }
+        if document_id:
+            content_dict["document_id"] = document_id
+
         # Use retain_batch_async with a single item (avoids code duplication)
         result = await self.retain_batch_async(
             bank_id=bank_id,
-            contents=[{
-                "content": content,
-                "context": context,
-                "event_date": event_date
-            }],
-            document_id=document_id,
+            contents=[content_dict],
             fact_type_override=fact_type_override,
             confidence_score=confidence_score
         )
@@ -623,7 +642,7 @@ class MemoryEngine:
     async def retain_batch_async(
         self,
         bank_id: str,
-        contents: List[Dict[str, Any]],
+        contents: List[RetainContentDict],
         document_id: Optional[str] = None,
         fact_type_override: Optional[str] = None,
         confidence_score: Optional[float] = None,
@@ -643,19 +662,32 @@ class MemoryEngine:
                 - "content" (required): Text content to store
                 - "context" (optional): Context about the memory
                 - "event_date" (optional): When the event occurred
-            document_id: Optional document ID for tracking (always upserts if document already exists)
+                - "document_id" (optional): Document ID for this specific content item
+            document_id: **DEPRECATED** - Use "document_id" key in each content dict instead.
+                        Applies the same document_id to ALL content items that don't specify their own.
             fact_type_override: Override fact type for all facts ('world', 'bank', 'opinion')
             confidence_score: Confidence score for opinions (0.0 to 1.0)
 
         Returns:
             List of lists of unit IDs (one list per content item)
 
-        Example:
+        Example (new style - per-content document_id):
             unit_ids = await memory.retain_batch_async(
                 bank_id="user123",
                 contents=[
-                    {"content": "Alice works at Google", "context": "conversation"},
-                    {"content": "Bob loves Python", "context": "conversation"},
+                    {"content": "Alice works at Google", "document_id": "doc1"},
+                    {"content": "Bob loves Python", "document_id": "doc2"},
+                    {"content": "More about Alice", "document_id": "doc1"},
+                ]
+            )
+            # Returns: [["unit-id-1"], ["unit-id-2"], ["unit-id-3"]]
+
+        Example (deprecated style - batch-level document_id):
+            unit_ids = await memory.retain_batch_async(
+                bank_id="user123",
+                contents=[
+                    {"content": "Alice works at Google"},
+                    {"content": "Bob loves Python"},
                 ],
                 document_id="meeting-2024-01-15"
             )
@@ -666,11 +698,17 @@ class MemoryEngine:
         if not contents:
             return []
 
+        # Apply batch-level document_id to contents that don't have their own (backwards compatibility)
+        if document_id:
+            for item in contents:
+                if "document_id" not in item:
+                    item["document_id"] = document_id
+
         # Auto-chunk large batches by character count to avoid timeouts and memory issues
         # Calculate total character count
         total_chars = sum(len(item.get("content", "")) for item in contents)
 
-        CHARS_PER_BATCH = 500_000
+        CHARS_PER_BATCH = 600_000
 
         if total_chars > CHARS_PER_BATCH:
             # Split into smaller batches based on character count
@@ -732,7 +770,7 @@ class MemoryEngine:
     async def _retain_batch_async_internal(
         self,
         bank_id: str,
-        contents: List[Dict[str, Any]],
+        contents: List[RetainContentDict],
         document_id: Optional[str] = None,
         is_first_batch: bool = True,
         fact_type_override: Optional[str] = None,
@@ -768,6 +806,7 @@ class MemoryEngine:
                 task_backend=self._task_backend,
                 format_date_fn=self._format_readable_date,
                 duplicate_checker_fn=self._find_duplicate_facts_batch,
+                regenerate_observations_fn=self._regenerate_observations_sync,
                 bank_id=bank_id,
                 contents_dicts=contents,
                 document_id=document_id,
@@ -982,7 +1021,11 @@ class MemoryEngine:
             temporal_results = []
             aggregated_timings = {"semantic": 0.0, "bm25": 0.0, "graph": 0.0, "temporal": 0.0}
 
-            for ft_semantic, ft_bm25, ft_graph, ft_temporal, ft_timings in all_retrievals:
+            for idx, (ft_semantic, ft_bm25, ft_graph, ft_temporal, ft_timings) in enumerate(all_retrievals):
+                # Log fact types in this retrieval batch
+                ft_name = fact_type[idx] if idx < len(fact_type) else "unknown"
+                logger.debug(f"[SEARCH {search_id}] Fact type '{ft_name}': semantic={len(ft_semantic)}, bm25={len(ft_bm25)}, graph={len(ft_graph)}, temporal={len(ft_temporal) if ft_temporal else 0}")
+
                 semantic_results.extend(ft_semantic)
                 bm25_results.extend(ft_bm25)
                 graph_results.extend(ft_graph)
@@ -996,6 +1039,14 @@ class MemoryEngine:
             if not temporal_results:
                 temporal_results = None
 
+            # Sort combined results by score (descending) so higher-scored results
+            # get better ranks in the trace, regardless of fact type
+            semantic_results.sort(key=lambda r: r.similarity if hasattr(r, 'similarity') else 0, reverse=True)
+            bm25_results.sort(key=lambda r: r.bm25_score if hasattr(r, 'bm25_score') else 0, reverse=True)
+            graph_results.sort(key=lambda r: r.activation if hasattr(r, 'activation') else 0, reverse=True)
+            if temporal_results:
+                temporal_results.sort(key=lambda r: r.combined_score if hasattr(r, 'combined_score') else 0, reverse=True)
+
             retrieval_duration = time.time() - retrieval_start
 
             step_duration = time.time() - step_start
@@ -1206,8 +1257,15 @@ class MemoryEngine:
                 })
                 log_buffer.append(f"  [7] Queued access count updates for {len(visited_ids)} nodes")
 
+            # Log fact_type distribution in results
+            fact_type_counts = {}
+            for sr in top_scored:
+                ft = sr.retrieval.fact_type
+                fact_type_counts[ft] = fact_type_counts.get(ft, 0) + 1
+
             total_time = time.time() - search_start
-            log_buffer.append(f"[SEARCH {search_id}] Complete: {len(top_scored)} results ({total_tokens} tokens) in {total_time:.3f}s")
+            fact_type_summary = ", ".join([f"{ft}={count}" for ft, count in sorted(fact_type_counts.items())])
+            log_buffer.append(f"[SEARCH {search_id}] Complete: {len(top_scored)} results ({fact_type_summary}) ({total_tokens} tokens) in {total_time:.3f}s")
 
             # Log all buffered logs at once
             logger.info("\n" + "\n".join(log_buffer))
@@ -1634,7 +1692,7 @@ class MemoryEngine:
             where_clause = "WHERE " + " AND ".join(query_conditions) if query_conditions else ""
 
             units = await conn.fetch(f"""
-                SELECT id, text, event_date, context, occurred_start, occurred_end, mentioned_at, document_id
+                SELECT id, text, event_date, context, occurred_start, occurred_end, mentioned_at, document_id, chunk_id, fact_type
                 FROM memory_units
                 {where_clause}
                 ORDER BY mentioned_at DESC NULLS LAST, event_date DESC
@@ -1758,7 +1816,9 @@ class MemoryEngine:
                 "mentioned_at": row['mentioned_at'].isoformat() if row['mentioned_at'] else None,
                 "date": row['event_date'].strftime("%Y-%m-%d %H:%M") if row['event_date'] else "N/A",  # Deprecated, kept for backwards compatibility
                 "entities": ", ".join(entities) if entities else "None",
-                "document_id": row['document_id']
+                "document_id": row['document_id'],
+                "chunk_id": row['chunk_id'] if row['chunk_id'] else None,
+                "fact_type": row['fact_type']
             })
 
         return {
@@ -1833,7 +1893,7 @@ class MemoryEngine:
             query_params.append(offset)
 
             units = await conn.fetch(f"""
-                SELECT id, text, event_date, context, fact_type, mentioned_at, occurred_start, occurred_end
+                SELECT id, text, event_date, context, fact_type, mentioned_at, occurred_start, occurred_end, chunk_id
                 FROM memory_units
                 {where_clause}
                 ORDER BY mentioned_at DESC NULLS LAST, created_at DESC
@@ -1877,7 +1937,8 @@ class MemoryEngine:
                     "mentioned_at": row['mentioned_at'].isoformat() if row['mentioned_at'] else None,
                     "occurred_start": row['occurred_start'].isoformat() if row['occurred_start'] else None,
                     "occurred_end": row['occurred_end'].isoformat() if row['occurred_end'] else None,
-                    "entities": ", ".join(entities) if entities else ""
+                    "entities": ", ".join(entities) if entities else "",
+                    "chunk_id": row['chunk_id'] if row['chunk_id'] else None
                 })
 
             return {
@@ -1950,7 +2011,8 @@ class MemoryEngine:
                     content_hash,
                     created_at,
                     updated_at,
-                    LENGTH(original_text) as text_length
+                    LENGTH(original_text) as text_length,
+                    retain_params
                 FROM documents
                 {where_clause}
                 ORDER BY created_at DESC
@@ -1998,7 +2060,8 @@ class MemoryEngine:
                     "created_at": row['created_at'].isoformat() if row['created_at'] else "",
                     "updated_at": row['updated_at'].isoformat() if row['updated_at'] else "",
                     "text_length": row['text_length'] or 0,
-                    "memory_unit_count": unit_count
+                    "memory_unit_count": unit_count,
+                    "retain_params": row['retain_params'] if row['retain_params'] else None
                 })
 
             return {
@@ -2032,7 +2095,8 @@ class MemoryEngine:
                     original_text,
                     content_hash,
                     created_at,
-                    updated_at
+                    updated_at,
+                    retain_params
                 FROM documents
                 WHERE id = $1 AND bank_id = $2
             """, document_id, bank_id)
@@ -2054,7 +2118,47 @@ class MemoryEngine:
                 "content_hash": doc['content_hash'],
                 "created_at": doc['created_at'].isoformat() if doc['created_at'] else "",
                 "updated_at": doc['updated_at'].isoformat() if doc['updated_at'] else "",
-                "memory_unit_count": unit_count_row['unit_count'] if unit_count_row else 0
+                "memory_unit_count": unit_count_row['unit_count'] if unit_count_row else 0,
+                "retain_params": doc['retain_params'] if doc['retain_params'] else None
+            }
+
+    async def get_chunk(
+        self,
+        chunk_id: str
+    ):
+        """
+        Get a specific chunk by its ID.
+
+        Args:
+            chunk_id: Chunk ID (format: bank_id_document_id_chunk_index)
+
+        Returns:
+            Dict with chunk details including chunk_text, or None if not found
+        """
+        pool = await self._get_pool()
+        async with acquire_with_retry(pool) as conn:
+            chunk = await conn.fetchrow("""
+                SELECT
+                    chunk_id,
+                    document_id,
+                    bank_id,
+                    chunk_index,
+                    chunk_text,
+                    created_at
+                FROM chunks
+                WHERE chunk_id = $1
+            """, chunk_id)
+
+            if not chunk:
+                return None
+
+            return {
+                "chunk_id": chunk['chunk_id'],
+                "document_id": chunk['document_id'],
+                "bank_id": chunk['bank_id'],
+                "chunk_index": chunk['chunk_index'],
+                "chunk_text": chunk['chunk_text'],
+                "created_at": chunk['created_at'].isoformat() if chunk['created_at'] else ""
             }
 
     async def _evaluate_opinion_update_async(
@@ -2792,24 +2896,127 @@ Guidelines:
         logger.info(f"[OBSERVATIONS] {entity_name}: {len(facts)} facts -> {len(created_ids)} observations")
         return created_ids
 
+    async def _regenerate_observations_sync(
+        self,
+        bank_id: str,
+        entity_ids: List[str],
+        min_facts: int = 5
+    ) -> None:
+        """
+        Regenerate observations for entities synchronously (called during retain).
+
+        Args:
+            bank_id: Bank identifier
+            entity_ids: List of entity IDs to process
+            min_facts: Minimum facts required to regenerate observations
+        """
+        if not bank_id or not entity_ids:
+            return
+
+        pool = await self._get_pool()
+        async with pool.acquire() as conn:
+            for entity_id in entity_ids:
+                try:
+                    entity_uuid = uuid.UUID(entity_id) if isinstance(entity_id, str) else entity_id
+
+                    # Check if entity exists
+                    entity_exists = await conn.fetchrow(
+                        "SELECT canonical_name FROM entities WHERE id = $1 AND bank_id = $2",
+                        entity_uuid, bank_id
+                    )
+
+                    if not entity_exists:
+                        logger.debug(f"[OBSERVATIONS] Entity {entity_id} not yet in bank {bank_id}, skipping")
+                        continue
+
+                    entity_name = entity_exists['canonical_name']
+
+                    # Count facts linked to this entity
+                    fact_count = await conn.fetchval(
+                        "SELECT COUNT(*) FROM unit_entities WHERE entity_id = $1",
+                        entity_uuid
+                    ) or 0
+
+                    # Only regenerate if entity has enough facts
+                    if fact_count >= min_facts:
+                        await self.regenerate_entity_observations(bank_id, entity_id, entity_name, version=None)
+                    else:
+                        logger.debug(f"[OBSERVATIONS] Skipping {entity_name} ({fact_count} facts < {min_facts} threshold)")
+
+                except Exception as e:
+                    logger.error(f"[OBSERVATIONS] Error processing entity {entity_id}: {e}")
+                    continue
+
     async def _handle_regenerate_observations(self, task_dict: Dict[str, Any]):
         """
         Handler for regenerate_observations tasks.
 
         Args:
-            task_dict: Dict with 'bank_id', 'entity_id', 'entity_name', 'version'
+            task_dict: Dict with 'bank_id' and either:
+                       - 'entity_ids' (list): Process multiple entities
+                       - 'entity_id', 'entity_name': Process single entity (legacy)
         """
         try:
             bank_id = task_dict.get('bank_id')
-            entity_id = task_dict.get('entity_id')
-            entity_name = task_dict.get('entity_name')
-            version = task_dict.get('version')  # last_seen timestamp for deduplication
 
-            if not all([bank_id, entity_id, entity_name]):
-                logger.error(f"[OBSERVATIONS] Missing required fields in task: {task_dict}")
-                return
+            # New format: multiple entity_ids
+            if 'entity_ids' in task_dict:
+                entity_ids = task_dict.get('entity_ids', [])
+                min_facts = task_dict.get('min_facts', 5)
+
+                if not bank_id or not entity_ids:
+                    logger.error(f"[OBSERVATIONS] Missing required fields in task: {task_dict}")
+                    return
+
+                # Process each entity
+                pool = await self._get_pool()
+                async with pool.acquire() as conn:
+                    for entity_id in entity_ids:
+                        try:
+                            # Fetch entity name and check fact count
+                            import uuid as uuid_module
+                            entity_uuid = uuid_module.UUID(entity_id) if isinstance(entity_id, str) else entity_id
+
+                            # First check if entity exists
+                            entity_exists = await conn.fetchrow(
+                                "SELECT canonical_name FROM entities WHERE id = $1 AND bank_id = $2",
+                                entity_uuid, bank_id
+                            )
+
+                            if not entity_exists:
+                                logger.debug(f"[OBSERVATIONS] Entity {entity_id} not yet in bank {bank_id}, skipping")
+                                continue
+
+                            entity_name = entity_exists['canonical_name']
+
+                            # Count facts linked to this entity
+                            fact_count = await conn.fetchval(
+                                "SELECT COUNT(*) FROM unit_entities WHERE entity_id = $1",
+                                entity_uuid
+                            ) or 0
+
+                            # Only regenerate if entity has enough facts
+                            if fact_count >= min_facts:
+                                await self.regenerate_entity_observations(bank_id, entity_id, entity_name, version=None)
+                            else:
+                                logger.debug(f"[OBSERVATIONS] Skipping {entity_name} ({fact_count} facts < {min_facts} threshold)")
+
+                        except Exception as e:
+                            logger.error(f"[OBSERVATIONS] Error processing entity {entity_id}: {e}")
+                            continue
+
+            # Legacy format: single entity
+            else:
+                entity_id = task_dict.get('entity_id')
+                entity_name = task_dict.get('entity_name')
+                version = task_dict.get('version')
+
+                if not all([bank_id, entity_id, entity_name]):
+                    logger.error(f"[OBSERVATIONS] Missing required fields in task: {task_dict}")
+                    return
+
+                await self.regenerate_entity_observations(bank_id, entity_id, entity_name, version)
 
-            await self.regenerate_entity_observations(bank_id, entity_id, entity_name, version)
         except Exception as e:
             logger.error(f"[OBSERVATIONS] Error regenerating observations: {e}")
             import traceback

hindsight-api/hindsight_api/engine/retain/deduplication.py

@@ -44,6 +44,12 @@ async def check_duplicates_batch(
         # Use occurred_start if available, otherwise use mentioned_at
         # For deduplication purposes, we need a time reference
         fact_date = fact.occurred_start if fact.occurred_start is not None else fact.mentioned_at
+
+        # Defensive: if both are None (shouldn't happen), use now()
+        if fact_date is None:
+            from datetime import datetime, timezone
+            fact_date = datetime.now(timezone.utc)
+
         # Round to 12-hour bucket to group similar times
         bucket_key = fact_date.replace(
             hour=(fact_date.hour // 12) * 12,

hindsight-api/hindsight_api/engine/retain/entity_processing.py

@@ -18,7 +18,8 @@ async def process_entities_batch(
     conn,
     bank_id: str,
     unit_ids: List[str],
-    facts: List[ProcessedFact]
+    facts: List[ProcessedFact],
+    log_buffer: List[str] = None
 ) -> List[Tuple[str, str, float]]:
     """
     Process entities for all facts and create entity links.
@@ -35,6 +36,7 @@ async def process_entities_batch(
         bank_id: Bank identifier
         unit_ids: List of unit IDs (same length as facts)
         facts: List of ProcessedFact objects
+        log_buffer: Optional buffer for detailed logging
 
     Returns:
         List of entity link tuples: (unit_id, entity_id, confidence)
@@ -65,7 +67,7 @@ async def process_entities_batch(
         "",  # context (not used in current implementation)
         fact_dates,
         entities_per_fact,
-        []  # log_buffer (optional)
+        log_buffer  # Pass log_buffer for detailed logging
     )
 
     return entity_links

hindsight-api/hindsight_api/engine/retain/fact_extraction.py

@@ -28,24 +28,20 @@ class Fact(BaseModel):
     Final fact model for storage - built from lenient parsing of LLM response.
 
     This is what fact_extraction returns and what the rest of the pipeline expects.
-    Only includes fields with meaningful values - nulls/empties are omitted.
+    Combined fact text format: "what | when | where | who | why"
     """
     # Required fields
-    fact: str = Field(description="Combined fact text from all dimensions")
+    fact: str = Field(description="Combined fact text: what | when | where | who | why")
     fact_type: Literal["world", "bank", "opinion"] = Field(description="Perspective: world/bank/opinion")
 
-    # Optional dimension fields
-    emotional_significance: Optional[str] = None
-    reasoning_motivation: Optional[str] = None
-    preferences_opinions: Optional[str] = None
-    sensory_details: Optional[str] = None
-    observations: Optional[str] = None
-
     # Optional temporal fields
     occurred_start: Optional[str] = None
     occurred_end: Optional[str] = None
     mentioned_at: Optional[str] = None
 
+    # Optional location field
+    where: Optional[str] = Field(None, description="WHERE the fact occurred or is about (specific location, place, or area)")
+
     # Optional structured data
     entities: Optional[List[Entity]] = None
     causal_relations: Optional[List['CausalRelation']] = None
@@ -74,75 +70,93 @@ class CausalRelation(BaseModel):
 
 
 class ExtractedFact(BaseModel):
-    """A single extracted fact with structured dimensions for comprehensive capture."""
+    """A single extracted fact with 5 required dimensions for comprehensive capture."""
 
     model_config = ConfigDict(
         json_schema_mode="validation",
-        # Only require truly critical fields - be lenient with everything else
         json_schema_extra={
-            "required": ["factual_core", "fact_type"]
+            "required": ["what", "when", "where", "who", "why", "fact_type"]
         }
     )
 
-    # Core factual dimension (CRITICAL - required)
-    factual_core: str = Field(
-        description="ACTUAL FACTS - what literally happened/was said. MUST be a complete, grammatically correct sentence with subject and verb. Capture WHAT was said, not just THAT something was said! 'Gina said Jon is the perfect mentor with positivity and determination' NOT 'Jon received encouragement'. Preserve: compliments, assessments, descriptions, key phrases. Be specific!"
+    # ==========================================================================
+    # FIVE REQUIRED DIMENSIONS - LLM must think about each one
+    # ==========================================================================
+
+    what: str = Field(
+        description="WHAT happened - COMPLETE, DETAILED description with ALL specifics. "
+                   "NEVER summarize or omit details. Include: exact actions, objects, quantities, specifics. "
+                   "BE VERBOSE - capture every detail that was mentioned. "
+                   "Example: 'Emily got married to Sarah at a rooftop garden ceremony with 50 guests attending and a live jazz band playing' "
+                   "NOT: 'A wedding happened' or 'Emily got married'"
     )
 
-    # Optional dimensions - only include if present in the text
-    # CRITICAL: Each dimension MUST be a complete, standalone sentence that reads naturally
-    emotional_significance: Optional[str] = Field(
-        default=None,
-        description="Emotions, feelings, personal meaning as a COMPLETE SENTENCE. Include subject + emotion/feeling. Examples: 'Sarah felt thrilled about the promotion', 'This was her favorite memory from childhood', 'The experience was magical for everyone involved', 'John found the loss devastating', 'She considers this her proudest moment'"
-    )
-    reasoning_motivation: Optional[str] = Field(
-        default=None,
-        description="WHY it happened as a COMPLETE SENTENCE. Include subject + motivation/reason. Examples: 'She did this because she wanted to celebrate', 'He wrote the book to cope with grief', 'She was motivated by curiosity about the topic'"
-    )
-    preferences_opinions: Optional[str] = Field(
-        default=None,
-        description="Likes, dislikes, beliefs, values as a COMPLETE SENTENCE. Include subject + preference/opinion. Examples: 'Sarah loves coffee and drinks it daily', 'He thinks AI is transformative technology', 'She prefers working remotely over office work'"
-    )
-    sensory_details: Optional[str] = Field(
-        default=None,
-        description="Visual, auditory, physical descriptions as a COMPLETE SENTENCE. Include subject + descriptive details. USE EXACT WORDS from text! Examples: 'She has bright orange hair', 'The dancer moved so gracefully on stage', 'The beach was awesome', 'The movie had epic visuals', 'The water was freezing cold'"
-    )
-    observations: Optional[str] = Field(
-        default=None,
-        description="Observations, inferences, and specific details/metrics as a COMPLETE SENTENCE. Include subject + observed fact. Use this to capture: background facts, achievements, metrics, personal records, skills. Examples: 'Calvin traveled to Miami for the shoot', 'Gina won dance trophies in competitions', 'She knows programming from previous projects', 'User's personal best 5K time is 25:50', 'Sarah has completed 15 marathons', 'He speaks three languages fluently'"
+    when: str = Field(
+        description="WHEN it happened - ALWAYS include temporal information if mentioned. "
+                   "Include: specific dates, times, durations, relative time references. "
+                   "Examples: 'on June 15th, 2024 at 3pm', 'last weekend', 'for the past 3 years', 'every morning at 6am'. "
+                   "Write 'N/A' ONLY if absolutely no temporal context exists. Prefer converting to absolute dates when possible."
     )
 
-    # Fact kind - optional hint for LLM thinking, not critical for extraction
-    # We don't strictly validate this since it's just guidance for temporal handling
-    fact_kind: Optional[str] = Field(
+    where: str = Field(
+        description="WHERE it happened or is about - SPECIFIC locations, places, areas, regions if applicable. "
+                   "Include: cities, neighborhoods, venues, buildings, countries, specific addresses when mentioned. "
+                   "Examples: 'downtown San Francisco at a rooftop garden venue', 'at the user's home in Brooklyn', 'online via Zoom', 'Paris, France'. "
+                   "Write 'N/A' ONLY if absolutely no location context exists or if the fact is completely location-agnostic."
+    )
+
+    who: str = Field(
+        description="WHO is involved - ALL people/entities with FULL context and relationships. "
+                   "Include: names, roles, relationships to user, background details. "
+                   "Resolve coreferences (if 'my roommate' is later named 'Emily', write 'Emily, the user's college roommate'). "
+                   "BE DETAILED about relationships and roles. "
+                   "Example: 'Emily (user's college roommate from Stanford, now works at Google), Sarah (Emily's partner of 5 years, software engineer)' "
+                   "NOT: 'my friend' or 'Emily and Sarah'"
+    )
+
+    why: str = Field(
+        description="WHY it matters - ALL emotional, contextual, and motivational details. "
+                   "Include EVERYTHING: feelings, preferences, motivations, observations, context, background, significance. "
+                   "BE VERBOSE - capture all the nuance and meaning. "
+                   "FOR ASSISTANT FACTS: MUST include what the user asked/requested that led to this interaction! "
+                   "Example (world): 'The user felt thrilled and inspired, has always dreamed of an outdoor ceremony, mentioned wanting a similar garden venue, was particularly moved by the intimate atmosphere and personal vows' "
+                   "Example (assistant): 'User asked how to fix slow API performance with 1000+ concurrent users, expected 70-80% reduction in database load' "
+                   "NOT: 'User liked it' or 'To help user'"
+    )
+
+    # ==========================================================================
+    # CLASSIFICATION
+    # ==========================================================================
+
+    fact_kind: str = Field(
         default="conversation",
-        description="Optional hint: 'conversation' = general info, 'event' = specific datable occurrence, 'other' = anything else. Helps determine if occurred dates should be set, but not critical."
+        description="'event' = specific datable occurrence (set occurred dates), 'conversation' = general info (no occurred dates)"
     )
 
     # Temporal fields - optional
     occurred_start: Optional[str] = Field(
         default=None,
-        description="WHEN THE EVENT ACTUALLY HAPPENED (not when mentioned). ISO timestamp. For datable events only (fact_kind='event'). Examples: 'went to Tokyo last spring' on June 10 → occurred_start='2024-03-01' (spring start), 'accident yesterday' on March 15 → occurred_start='2024-03-14' (yesterday). Leave null for general info (fact_kind='conversation')."
+        description="WHEN the event happened (ISO timestamp). Only for fact_kind='event'. Leave null for conversations."
     )
     occurred_end: Optional[str] = Field(
         default=None,
-        description="WHEN THE EVENT ACTUALLY ENDED (not when mentioned). ISO timestamp. For datable events with duration (fact_kind='event'). Examples: 'went to Tokyo last spring' → occurred_end='2024-05-31' (spring end). Can be same as occurred_start for single-day events. Leave null for general info."
+        description="WHEN the event ended (ISO timestamp). Only for events with duration. Leave null for conversations."
     )
 
     # Classification (CRITICAL - required)
     # Note: LLM uses "assistant" but we convert to "bank" for storage
     fact_type: Literal["world", "assistant"] = Field(
-        description="REQUIRED: 'world' = everything NOT involving the assistant (user's background, skills, experiences, other people's lives, events). 'assistant' = interactions BY or TO the assistant (user asked assistant, assistant recommended, assistant helped user, etc.)"
+        description="'world' = about the user/others (background, experiences). 'assistant' = interactions with the assistant."
     )
 
-    # Entities and relations
+    # Entities - extracted from 'who' field
     entities: Optional[List[Entity]] = Field(
         default=None,
-        description="ONLY specific, named entities worth tracking: people's names (e.g., 'Sarah', 'Dr. Smith'), organizations (e.g., 'Google', 'MIT'), specific places (e.g., 'Paris', 'Central Park'). DO NOT include: generic relations (mom, friend, boss, colleague), common nouns (apple, car, house), pronouns (he, she), or vague references (someone, a guy). Can be null or empty list [] if no entities."
+        description="Named entities from 'who': people names, organizations, places. NOT generic relations."
     )
     causal_relations: Optional[List[CausalRelation]] = Field(
         default=None,
-        description="Causal links to other facts in this batch. Example: fact about rain causes fact about cancelled game. Can be null or empty list [] if no causal relations."
+        description="Causal links to other facts. Can be null."
     )
 
     @field_validator('entities', mode='before')
@@ -163,25 +177,20 @@ class ExtractedFact(BaseModel):
 
     def build_fact_text(self) -> str:
         """Combine all dimensions into a single comprehensive fact string."""
-        parts = [self.factual_core]
+        parts = [self.what]
 
-        if self.emotional_significance:
-            parts.append(self.emotional_significance)
-        if self.reasoning_motivation:
-            parts.append(self.reasoning_motivation)
-        if self.preferences_opinions:
-            parts.append(self.preferences_opinions)
-        if self.sensory_details:
-            parts.append(self.sensory_details)
-        if self.observations:
-            parts.append(self.observations)
+        # Add 'who' if not N/A
+        if self.who and self.who.upper() != 'N/A':
+            parts.append(f"Involving: {self.who}")
+
+        # Add 'why' if not N/A
+        if self.why and self.why.upper() != 'N/A':
+            parts.append(self.why)
 
-        # Join with appropriate connectors
         if len(parts) == 1:
             return parts[0]
 
-        # Combine: "Core fact - emotional/significance context"
-        return f"{parts[0]} - {' - '.join(parts[1:])}"
+        return " | ".join(parts)
 
 
 class FactExtractionResponse(BaseModel):
@@ -193,27 +202,35 @@ class FactExtractionResponse(BaseModel):
 
 def chunk_text(text: str, max_chars: int) -> List[str]:
     """
-    Split text into chunks at sentence boundaries using LangChain's text splitter.
+    Split text into chunks, preserving conversation structure when possible.
 
-    Uses RecursiveCharacterTextSplitter which intelligently splits at sentence boundaries
-    and allows chunks to slightly exceed max_chars to finish sentences naturally.
+    For JSON conversation arrays (user/assistant turns), splits at turn boundaries
+    while preserving speaker context. For plain text, uses sentence-aware splitting.
 
     Args:
-        text: Input text to chunk
+        text: Input text to chunk (plain text or JSON conversation)
         max_chars: Maximum characters per chunk (default 120k ≈ 30k tokens)
-                   Note: chunks may slightly exceed this to complete sentences
 
     Returns:
         List of text chunks, roughly under max_chars
     """
+    import json
     from langchain_text_splitters import RecursiveCharacterTextSplitter
 
     # If text is small enough, return as-is
     if len(text) <= max_chars:
         return [text]
 
-    # Configure splitter to split at sentence boundaries first
-    # Separators in order of preference: paragraphs, newlines, sentences, words
+    # Try to parse as JSON conversation array
+    try:
+        parsed = json.loads(text)
+        if isinstance(parsed, list) and all(isinstance(turn, dict) for turn in parsed):
+            # This looks like a conversation - chunk at turn boundaries
+            return _chunk_conversation(parsed, max_chars)
+    except (json.JSONDecodeError, ValueError):
+        pass
+
+    # Fall back to sentence-aware text splitting
     splitter = RecursiveCharacterTextSplitter(
         chunk_size=max_chars,
         chunk_overlap=0,
@@ -235,6 +252,45 @@ def chunk_text(text: str, max_chars: int) -> List[str]:
     return splitter.split_text(text)
 
 
+def _chunk_conversation(turns: List[dict], max_chars: int) -> List[str]:
+    """
+    Chunk a conversation array at turn boundaries, preserving complete turns.
+
+    Args:
+        turns: List of conversation turn dicts (with 'role' and 'content' keys)
+        max_chars: Maximum characters per chunk
+
+    Returns:
+        List of JSON-serialized chunks, each containing complete turns
+    """
+    import json
+
+    chunks = []
+    current_chunk = []
+    current_size = 2  # Account for "[]"
+
+    for turn in turns:
+        # Estimate size of this turn when serialized (with comma separator)
+        turn_json = json.dumps(turn, ensure_ascii=False)
+        turn_size = len(turn_json) + 1  # +1 for comma
+
+        # If adding this turn would exceed limit and we have turns, save current chunk
+        if current_size + turn_size > max_chars and current_chunk:
+            chunks.append(json.dumps(current_chunk, ensure_ascii=False))
+            current_chunk = []
+            current_size = 2  # Reset to "[]"
+
+        # Add turn to current chunk
+        current_chunk.append(turn)
+        current_size += turn_size
+
+    # Add final chunk if non-empty
+    if current_chunk:
+        chunks.append(json.dumps(current_chunk, ensure_ascii=False))
+
+    return chunks if chunks else [json.dumps(turns, ensure_ascii=False)]
+
+
 async def _extract_facts_from_chunk(
     chunk: str,
     chunk_index: int,
@@ -261,141 +317,145 @@ async def _extract_facts_from_chunk(
     else:
         fact_types_instruction = "Extract ONLY 'world' and 'assistant' type facts. DO NOT extract opinions - those are extracted separately."
 
-    prompt = f"""Extract comprehensive facts from user text for an AI memory system.
+    prompt = f"""Extract facts from text into structured format with FOUR required dimensions - BE EXTREMELY DETAILED.
 
 {fact_types_instruction}
 
-## CONTEXT
-- Context: {context if context else 'none'}{agent_context}
+Context: {context if context else 'none'}{agent_context}
 
-═══════════════════════════════════════════════════════════════════════════════
-SECTION 1: TEMPORAL HANDLING (CRITICAL)
-═══════════════════════════════════════════════════════════════════════════════
+══════════════════════════════════════════════════════════════════════════
+FACT FORMAT - ALL FIVE DIMENSIONS REQUIRED - MAXIMUM VERBOSITY
+══════════════════════════════════════════════════════════════════════════
 
-### 1.1 DETECT TEMPORAL MARKERS
-Watch for: "yesterday", "last week/month/year/summer", "ago", "tomorrow", "next", "happened", "occurred", past tense verbs ("went", "visited", "saw")
+For EACH fact, CAPTURE ALL DETAILS - NEVER SUMMARIZE OR OMIT:
 
-### 1.2 DUAL FACT CREATION (KEY RULE)
-When text mentions a past/future event → Create TWO facts:
-1. MENTION FACT: "On [context date], it was mentioned that..." (occurred_start = context date)
-2. EVENT FACT: "[Action] in [absolute date]" (occurred_start = actual event date)
+1. **what**: WHAT happened - COMPLETE description with ALL specifics (objects, actions, quantities, details)
+2. **when**: WHEN it happened - ALWAYS include temporal info (dates, times, durations, relative times)
+3. **where**: WHERE it happened or is about - SPECIFIC locations, places, areas, regions (if applicable)
+4. **who**: WHO is involved - ALL people/entities with FULL relationships and background
+5. **why**: WHY it matters - ALL emotions, preferences, motivations, significance, nuance
+   - For assistant facts: MUST include what the user asked/requested that triggered this!
 
-### 1.3 ABSOLUTE DATE CONVERSION
-ALWAYS convert relative → absolute in factual_core text:
-- "yesterday" → "on [date-1]"
-- "last week" → "around [specific week]"
-- "last summer" → "in summer [year] (June-August [year])"
-- "next month" → "in [month name] [year]"
+Plus: fact_type, fact_kind, entities, occurred_start/end (for structured dates), where (structured location)
 
-### 1.4 occurred_start/end FIELDS ⚠️ CRITICAL
+VERBOSITY REQUIREMENT: Include EVERY detail mentioned. More detail is ALWAYS better than less.
 
-**WHAT THEY REPRESENT:**
-- occurred_start/end = WHEN THE EVENT ACTUALLY HAPPENED (NOT when it was mentioned!)
-- These answer: "When did this event occur in reality?"
+══════════════════════════════════════════════════════════════════════════
+COREFERENCE RESOLUTION (CRITICAL)
+══════════════════════════════════════════════════════════════════════════
 
-**WHEN TO SET THEM:**
-✅ SET for datable events (fact_kind="event"):
-   - "went to Tokyo last spring" → occurred_start = March 1, 2024 (spring started)
-   - "accident yesterday" → occurred_start = context date - 1 day
-   - "party next Saturday" → occurred_start = next Saturday's date
+When text uses BOTH a generic relation AND a name for the same person → LINK THEM!
 
-❌ LEAVE NULL for general info (fact_kind="conversation"):
-   - "loves coffee" → no occurred dates (timeless preference)
-   - "works as engineer" → no occurred dates (ongoing state)
-   - "is expanding business" → no occurred dates (ongoing activity)
+Example input: "I went to my college roommate's wedding last June. Emily finally married Sarah after 5 years together."
 
-**KEY DISTINCTION:**
-- occurred_start/end: When the event happened/will happen
-- mentioned_at: When this was said/written (set automatically to context date)
-- These are DIFFERENT! Example: On June 10, saying "went to Tokyo in March" → occurred_start=March, mentioned_at=June 10
+CORRECT output:
+- what: "Emily got married to Sarah at a rooftop garden ceremony"
+- when: "in June 2024, after dating for 5 years"
+- where: "downtown San Francisco, at a rooftop garden venue"
+- who: "Emily (user's college roommate), Sarah (Emily's partner of 5 years)"
+- why: "User found it romantic and beautiful, dreams of similar outdoor ceremony"
+- where (structured): "San Francisco"
 
-**FORMAT:** ISO timestamps "2024-06-15T00:00:00Z"
+WRONG output:
+- what: "User's roommate got married" ← LOSES THE NAME!
+- who: "the roommate" ← WRONG - use the actual name!
+- where: (missing) ← WRONG - include the location!
 
-### 1.5 EXAMPLES - STUDY THESE CAREFULLY
+══════════════════════════════════════════════════════════════════════════
+TEMPORAL HANDLING
+══════════════════════════════════════════════════════════════════════════
 
-**Example 1: "yesterday" temporal detection**
-Input (Context: March 15, 2024): "Hey Taylor! The volunteers were amazing yesterday. But something unexpected happened - a vehicle accident near the center. Everyone was okay though."
+For EVENTS (fact_kind="event"):
+- Convert relative dates → absolute: "yesterday" on March 15 → "March 14, 2024"
+- Set occurred_start/occurred_end to WHEN IT HAPPENED (not when mentioned)
 
-Output (3 facts):
-1. factual_core: "On March 15, 2024, Alex told Taylor that the volunteers were amazing"
-   occurred_start: "2024-03-15T00:00:00Z", entities: ["Alex", "Taylor"]
+For CONVERSATIONS (fact_kind="conversation"):
+- General info, preferences, ongoing states → NO occurred dates
+- Examples: "loves coffee", "works as engineer"
 
-2. factual_core: "On March 15, 2024, Alex mentioned that something unexpected happened the previous day - a vehicle accident"
-   occurred_start: "2024-03-15T00:00:00Z", entities: ["Alex"]
+══════════════════════════════════════════════════════════════════════════
+FACT TYPE
+══════════════════════════════════════════════════════════════════════════
 
-3. factual_core: "On March 14, 2024, a vehicle accident occurred near the center, but everyone was okay"
-   occurred_start: "2024-03-14T00:00:00Z" ← THE ACTUAL EVENT DATE (yesterday from March 15)
+- **world**: User's life, other people, events (would exist without this conversation)
+- **assistant**: Interactions with assistant (requests, recommendations, help)
+  ⚠️ CRITICAL for assistant facts: ALWAYS capture the user's request/question in the fact!
+  Include: what the user asked, what problem they wanted solved, what context they provided
 
-**Example 2: "last spring" temporal detection**
-Input (Context: June 10, 2024): "Casey went to Tokyo last spring. They had an incredible time visiting temples and trying authentic ramen."
+══════════════════════════════════════════════════════════════════════════
+USER PREFERENCES (CRITICAL)
+══════════════════════════════════════════════════════════════════════════
 
-Output (2 facts):
-1. factual_core: "On June 10, 2024, it was mentioned that Casey went to Tokyo the previous spring"
-   occurred_start: "2024-06-10T00:00:00Z", entities: ["Casey", "Tokyo"]
+ALWAYS extract user preferences as separate facts! Watch for these keywords:
+- "enjoy", "like", "love", "prefer", "hate", "dislike", "favorite", "ideal", "dream", "want"
 
-2. factual_core: "Casey went to Tokyo in spring 2024 (March-May 2024) and visited temples and tried authentic ramen"
-   occurred_start: "2024-03-01T00:00:00Z", occurred_end: "2024-05-31T23:59:59Z" ← THE ACTUAL EVENT DATES
-   emotional_significance: "Casey had an incredible time in Tokyo"
-   entities: ["Casey", "Tokyo"]
+Example: "I love Italian food and prefer outdoor dining"
+→ Fact 1: what="User loves Italian food", who="user", why="This is a food preference", entities=["user"]
+→ Fact 2: what="User prefers outdoor dining", who="user", why="This is a dining preference", entities=["user"]
 
-═══════════════════════════════════════════════════════════════════════════════
-SECTION 2: EXTRACTION RULES
-═══════════════════════════════════════════════════════════════════════════════
+══════════════════════════════════════════════════════════════════════════
+ENTITIES - INCLUDE "user" (CRITICAL)
+══════════════════════════════════════════════════════════════════════════
 
-### 2.1 WHAT TO EXTRACT
-✅ User requests to assistant + assistant actions (extract separately)
-✅ Preferences, recommendations, plans, activities, encouragement (with actual content)
-✅ Possessions, achievements, metrics, skills, background facts
+When a fact is ABOUT the user (their preferences, plans, experiences), ALWAYS include "user" in entities!
 
-### 2.2 WHAT TO SKIP
-❌ Greetings, filler ("thanks", "cool"), structural statements
+✅ CORRECT: entities=["user"] for "User loves coffee"
+✅ CORRECT: entities=["user", "Emily"] for "User attended Emily's wedding"
+❌ WRONG: entities=[] for facts about the user
 
-### 2.3 Q&A HANDLING
-- Combine simple informational Q&A into one fact
-- Split user requests to assistant into two facts (request + response)
+══════════════════════════════════════════════════════════════════════════
+EXAMPLES
+══════════════════════════════════════════════════════════════════════════
 
-═══════════════════════════════════════════════════════════════════════════════
-SECTION 3: STRUCTURED DIMENSIONS
-═══════════════════════════════════════════════════════════════════════════════
+Example 1 - World Facts (Context: June 10, 2024):
+Input: "I'm planning my wedding and want a small outdoor ceremony. I just got back from my college roommate Emily's wedding - she married Sarah at a rooftop garden, it was so romantic!"
 
-### 3.1 REQUIRED FIELD
-- **factual_core**: Capture WHAT was said, not just THAT something was said. Complete sentence.
+Output facts:
 
-### 3.2 OPTIONAL FIELDS (use when present in text)
-- **emotional_significance**: Emotions, feelings, qualitative descriptors. Complete sentence with subject.
-- **reasoning_motivation**: Why it happened, intentions, goals. Complete sentence with subject.
-- **preferences_opinions**: Likes, dislikes, beliefs, values. Complete sentence with subject. Use for: "ideal", "favorite", "dream", "perfect"
-- **sensory_details**: Visual, auditory, physical descriptions. Complete sentence. USE EXACT WORDS from text!
-- **observations**: Background facts, possessions, achievements, metrics, skills. Complete sentence with subject.
+1. User's wedding preference
+   - what: "User wants a small outdoor ceremony for their wedding"
+   - who: "user"
+   - why: "User prefers intimate outdoor settings"
+   - fact_type: "world", fact_kind: "conversation"
+   - entities: ["user"]
 
-### 3.3 FORMATTING RULE
-Each dimension MUST be a complete, grammatically correct sentence with subject that can stand alone.
+2. User planning wedding
+   - what: "User is planning their own wedding"
+   - who: "user"
+   - why: "Inspired by Emily's ceremony"
+   - fact_type: "world", fact_kind: "conversation"
+   - entities: ["user"]
 
-═══════════════════════════════════════════════════════════════════════════════
-SECTION 4: FACT CLASSIFICATION
-═══════════════════════════════════════════════════════════════════════════════
+3. Emily's wedding (THE EVENT)
+   - what: "Emily got married to Sarah at a rooftop garden ceremony in the city"
+   - who: "Emily (user's college roommate), Sarah (Emily's partner)"
+   - why: "User found it romantic and beautiful"
+   - fact_type: "world", fact_kind: "event"
+   - occurred_start: "2024-06-09T00:00:00Z" (recently, user "just got back")
+   - entities: ["user", "Emily", "Sarah"]
 
-### 4.1 fact_kind (temporal nature)
-- **conversation**: General info, ongoing activities (no occurred dates)
-- **event**: Specific datable occurrence (MUST set occurred_start/end)
-- **other**: Catch-all
+Example 2 - Assistant Facts (Context: March 5, 2024):
+Input: "User: My API is really slow when we have 1000+ concurrent users. What can I do?
+Assistant: I'd recommend implementing Redis for caching frequently-accessed data, which should reduce your database load by 70-80%."
 
-### 4.2 fact_type (subject matter)
-- **world**: Everything NOT involving assistant (user background, other people, events)
-- **assistant**: Interactions BY or TO assistant (requests, recommendations, actions in THIS conversation)
+Output fact:
+   - what: "Assistant recommended implementing Redis for caching frequently-accessed data to improve API performance"
+   - when: "March 5, 2024 during conversation"
+   - who: "user, assistant"
+   - why: "User asked how to fix slow API performance with 1000+ concurrent users, expected 70-80% reduction in database load"
+   - fact_type: "assistant", fact_kind: "conversation"
+   - entities: ["user"]
 
-Rule: If it would exist without this conversation → world. If only exists because of this conversation → assistant.
+Note how the "why" field captures the FULL STORY: what the user asked AND what outcome was expected!
 
-═══════════════════════════════════════════════════════════════════════════════
-SECTION 5: ENTITIES & CAUSALITY
-═══════════════════════════════════════════════════════════════════════════════
+══════════════════════════════════════════════════════════════════════════
+WHAT TO EXTRACT vs SKIP
+══════════════════════════════════════════════════════════════════════════
+
+✅ EXTRACT: User preferences (ALWAYS as separate facts!), feelings, plans, events, relationships, achievements
+❌ SKIP: Greetings, filler ("thanks", "cool"), purely structural statements"""
 
-### 5.1 ENTITIES
-Extract: People names, organizations, specific places, products
-Skip: Generic relations (mom, friend), pronouns, common nouns
 
-### 5.2 CAUSAL RELATIONS
-Link facts when explicit causation: causes, caused_by, enables, prevents"""
 
 
     import logging
@@ -407,14 +467,16 @@ Link facts when explicit causation: causes, caused_by, enables, prevents"""
     max_retries = 2
     last_error = None
 
-    # inject all the chunk metadata for better reasoning
-    chunk_data = json.dumps({
-        "chunk_index": chunk_index,
-        "total_chunks": total_chunks,
-        "event_date": event_date.isoformat(),
-        "context": context,
-        "chunk_content": chunk
-    })
+    # Build user message with metadata and chunk content in a clear format
+    user_message = f"""Extract facts from the following text chunk.
+
+Chunk: {chunk_index + 1}/{total_chunks}
+Event Date: {event_date.isoformat()}
+Context: {context if context else 'none'}
+
+Text:
+{chunk}"""
+
     for attempt in range(max_retries):
         try:
             extraction_response_json = await llm_config.call(
@@ -425,7 +487,7 @@ Link facts when explicit causation: causes, caused_by, enables, prevents"""
                     },
                     {
                         "role": "user",
-                        "content": chunk_data
+                        "content": user_message
                     }
                 ],
                 response_format=FactExtractionResponse,
@@ -437,32 +499,58 @@ Link facts when explicit causation: causes, caused_by, enables, prevents"""
 
             # Lenient parsing of facts from raw JSON
             chunk_facts = []
+            has_malformed_facts = False
 
             # Handle malformed LLM responses
             if not isinstance(extraction_response_json, dict):
-                logger.warning(
-                    f"LLM returned non-dict JSON: {type(extraction_response_json).__name__}. "
-                    f"Raw: {str(extraction_response_json)[:500]}"
-                )
-                return []
+                if attempt < max_retries - 1:
+                    logger.warning(
+                        f"LLM returned non-dict JSON on attempt {attempt + 1}/{max_retries}: {type(extraction_response_json).__name__}. Retrying..."
+                    )
+                    continue
+                else:
+                    logger.warning(
+                        f"LLM returned non-dict JSON after {max_retries} attempts: {type(extraction_response_json).__name__}. "
+                        f"Raw: {str(extraction_response_json)[:500]}"
+                    )
+                    return []
 
             raw_facts = extraction_response_json.get('facts', [])
             if not raw_facts:
-                logger.warning(
+                logger.debug(
                     f"LLM response missing 'facts' field or returned empty list. "
-                    f"Response: {extraction_response_json}"
+                    f"Response: {extraction_response_json}. "
+                    f"Input: "
+                    f"date: {event_date.isoformat()}, "
+                    f"context: {context if context else 'none'}, "
+                    f"text: {chunk}"
                 )
 
             for i, llm_fact in enumerate(raw_facts):
-                # Skip non-dict entries
+                # Skip non-dict entries but track them for retry
                 if not isinstance(llm_fact, dict):
                     logger.warning(f"Skipping non-dict fact at index {i}")
+                    has_malformed_facts = True
                     continue
 
-                # Critical field: factual_core (MUST have this)
-                factual_core = llm_fact.get('factual_core')
-                if not factual_core:
-                    logger.warning(f"Skipping fact {i}: missing factual_core")
+                # Helper to get non-empty value
+                def get_value(field_name):
+                    value = llm_fact.get(field_name)
+                    if value and value != '' and value != [] and value != {} and str(value).upper() != 'N/A':
+                        return value
+                    return None
+
+                # NEW FORMAT: what, when, who, why (all required)
+                what = get_value('what')
+                when = get_value('when')
+                who = get_value('who')
+                why = get_value('why')
+
+                # Fallback to old format if new fields not present
+                if not what:
+                    what = get_value('factual_core')
+                if not what:
+                    logger.warning(f"Skipping fact {i}: missing 'what' field")
                     continue
 
                 # Critical field: fact_type
@@ -491,34 +579,20 @@ Link facts when explicit causation: causes, caused_by, enables, prevents"""
                 if fact_kind not in ['conversation', 'event', 'other']:
                     fact_kind = 'conversation'
 
-                # Build combined fact text from dimensions
-                dimension_parts = []
+                # Build combined fact text from the 4 dimensions: what | when | who | why
                 fact_data = {}
+                combined_parts = [what]
 
-                # Helper to get non-empty value
-                def get_value(field_name):
-                    value = llm_fact.get(field_name)
-                    if value and value != '' and value != [] and value != {}:
-                        return value
-                    return None
+                if when:
+                    combined_parts.append(f"When: {when}")
 
-                # Collect dimension fields
-                for field in ['emotional_significance', 'reasoning_motivation', 'preferences_opinions',
-                              'sensory_details', 'observations']:
-                    value = get_value(field)
-                    if value:
-                        # Handle case where LLM returns list instead of string
-                        if isinstance(value, list):
-                            value = '; '.join(str(v) for v in value)
-                        fact_data[field] = value
-                        dimension_parts.append(value)
+                if who:
+                    combined_parts.append(f"Involving: {who}")
 
-                # Build combined fact text
-                combined_parts = [factual_core] + dimension_parts
-                if len(combined_parts) == 1:
-                    combined_text = combined_parts[0]
-                else:
-                    combined_text = f"{combined_parts[0]} - {' - '.join(combined_parts[1:])}"
+                if why:
+                    combined_parts.append(why)
+
+                combined_text = " | ".join(combined_parts)
 
                 # Add temporal fields
                 # For events: occurred_start/occurred_end (when the event happened)
@@ -575,7 +649,16 @@ Link facts when explicit causation: causes, caused_by, enables, prevents"""
                     chunk_facts.append(fact)
                 except Exception as e:
                     logger.error(f"Failed to create Fact model for fact {i}: {e}")
+                    has_malformed_facts = True
                     continue
+
+            # If we got malformed facts and haven't exhausted retries, try again
+            if has_malformed_facts and len(chunk_facts) < len(raw_facts) * 0.8 and attempt < max_retries - 1:
+                logger.warning(
+                    f"Got {len(raw_facts) - len(chunk_facts)} malformed facts out of {len(raw_facts)} on attempt {attempt + 1}/{max_retries}. Retrying..."
+                )
+                continue
+
             return chunk_facts
 
         except BadRequestError as e:

hindsight-api/hindsight_api/engine/retain/fact_storage.py

@@ -66,7 +66,8 @@ async def insert_facts_batch(
         access_counts.append(0)  # Initial access count
         metadata_jsons.append(json.dumps(fact.metadata))
         chunk_ids.append(fact.chunk_id)
-        document_ids.append(document_id)
+        # Use per-fact document_id if available, otherwise fallback to batch-level document_id
+        document_ids.append(fact.document_id if fact.document_id else document_id)
 
     # Batch insert all facts
     results = await conn.fetch(
@@ -127,7 +128,8 @@ async def handle_document_tracking(
     bank_id: str,
     document_id: str,
     combined_content: str,
-    is_first_batch: bool
+    is_first_batch: bool,
+    retain_params: Optional[dict] = None
 ) -> None:
     """
     Handle document tracking in the database.
@@ -138,6 +140,7 @@ async def handle_document_tracking(
         document_id: Document identifier
         combined_content: Combined content text from all content items
         is_first_batch: Whether this is the first batch (for chunked operations)
+        retain_params: Optional parameters passed during retain (context, event_date, etc.)
     """
     import hashlib
 
@@ -155,17 +158,19 @@ async def handle_document_tracking(
     # Insert document (or update if exists from concurrent operations)
     await conn.execute(
         """
-        INSERT INTO documents (id, bank_id, original_text, content_hash, metadata)
-        VALUES ($1, $2, $3, $4, $5)
+        INSERT INTO documents (id, bank_id, original_text, content_hash, metadata, retain_params)
+        VALUES ($1, $2, $3, $4, $5, $6)
         ON CONFLICT (id, bank_id) DO UPDATE
         SET original_text = EXCLUDED.original_text,
             content_hash = EXCLUDED.content_hash,
             metadata = EXCLUDED.metadata,
+            retain_params = EXCLUDED.retain_params,
             updated_at = NOW()
         """,
         document_id,
         bank_id,
         combined_content,
         content_hash,
-        json.dumps({})  # Empty metadata dict
+        json.dumps({}),  # Empty metadata dict
+        json.dumps(retain_params) if retain_params else None
     )

hindsight-api/hindsight_api/engine/retain/link_utils.py

@@ -97,37 +97,56 @@ async def extract_entities_batch_optimized(
         if all_entities_flat:
             # [6.2.2] Batch resolve entities
             substep_6_2_2_start = time.time()
-            # Group by date for batch resolution (most will have same date)
+            # Group by date for batch resolution (round to hour to reduce buckets)
             entities_by_date = {}
             for idx, (unit_id, local_idx, fact_date) in enumerate(entity_to_unit):
-                date_key = fact_date
+                # Round to hour to group facts from same time period
+                date_key = fact_date.replace(minute=0, second=0, microsecond=0)
                 if date_key not in entities_by_date:
                     entities_by_date[date_key] = []
                 entities_by_date[date_key].append((idx, all_entities_flat[idx]))
 
-            _log(log_buffer, f"    [6.2.2] Grouped into {len(entities_by_date)} date buckets, resolving...")
+            _log(log_buffer, f"    [6.2.2] Grouped into {len(entities_by_date)} date buckets, resolving in parallel...")
 
-            # Resolve each date group in batch
+            # Resolve all date groups in PARALLEL using asyncio.gather
             resolved_entity_ids = [None] * len(all_entities_flat)
-            for date_idx, (fact_date, entities_group) in enumerate(entities_by_date.items(), 1):
+
+            # Prepare all resolution tasks
+            async def resolve_date_bucket(date_idx, date_key, entities_group):
                 date_bucket_start = time.time()
                 indices = [idx for idx, _ in entities_group]
                 entities_data = [entity_data for _, entity_data in entities_group]
+                # Use the first fact's date for this bucket (all should be in same hour)
+                fact_date = entity_to_unit[indices[0]][2]
 
+                # Pass conn=None to let each parallel task acquire its own connection
                 batch_resolved = await entity_resolver.resolve_entities_batch(
                     bank_id=bank_id,
                     entities_data=entities_data,
                     context=context,
                     unit_event_date=fact_date,
-                    conn=conn
+                    conn=None  # Each task gets its own connection from pool
                 )
 
+                if len(entities_by_date) <= 10:  # Only log individual buckets if there aren't too many
+                    _log(log_buffer, f"      [6.2.2.{date_idx}] Resolved {len(entities_data)} entities in {time.time() - date_bucket_start:.3f}s")
+
+                return indices, batch_resolved
+
+            # Execute all resolution tasks in parallel
+            import asyncio
+            tasks = [
+                resolve_date_bucket(date_idx, date_key, entities_group)
+                for date_idx, (date_key, entities_group) in enumerate(entities_by_date.items(), 1)
+            ]
+            results = await asyncio.gather(*tasks)
+
+            # Map results back to resolved_entity_ids
+            for indices, batch_resolved in results:
                 for idx, entity_id in zip(indices, batch_resolved):
                     resolved_entity_ids[idx] = entity_id
 
-                _log(log_buffer, f"      [6.2.2.{date_idx}] Resolved {len(entities_data)} entities in {time.time() - date_bucket_start:.3f}s")
-
-            _log(log_buffer, f"    [6.2.2] Resolve entities: {len(all_entities_flat)} entities in {time.time() - substep_6_2_2_start:.3f}s")
+            _log(log_buffer, f"    [6.2.2] Resolve entities: {len(all_entities_flat)} entities across {len(entities_by_date)} buckets in {time.time() - substep_6_2_2_start:.3f}s")
 
             # [6.2.3] Create unit-entity links in BATCH
             substep_6_2_3_start = time.time()
@@ -444,17 +463,14 @@ async def insert_entity_links_batch(conn, links: List[tuple]):
     if not links:
         return
 
-    try:
-        await conn.executemany(
-            """
-            INSERT INTO memory_links (from_unit_id, to_unit_id, link_type, weight, entity_id)
-            VALUES ($1, $2, $3, $4, $5)
-            ON CONFLICT (from_unit_id, to_unit_id, link_type, COALESCE(entity_id, '00000000-0000-0000-0000-000000000000'::uuid)) DO NOTHING
-            """,
-            links
-        )
-    except Exception as e:
-        logger.warning(f"Failed to insert entity links: {str(e)}")
+    await conn.executemany(
+        """
+        INSERT INTO memory_links (from_unit_id, to_unit_id, link_type, weight, entity_id)
+        VALUES ($1, $2, $3, $4, $5)
+        ON CONFLICT (from_unit_id, to_unit_id, link_type, COALESCE(entity_id, '00000000-0000-0000-0000-000000000000'::uuid)) DO NOTHING
+        """,
+        links
+    )
 
 
 async def create_causal_links_batch(

hindsight-api/hindsight_api/engine/retain/orchestrator.py

@@ -39,6 +39,7 @@ async def retain_batch(
     task_backend,
     format_date_fn,
     duplicate_checker_fn,
+    regenerate_observations_fn,
     bank_id: str,
     contents_dicts: List[Dict[str, Any]],
     document_id: Optional[str] = None,
@@ -57,6 +58,7 @@ async def retain_batch(
         task_backend: Task backend for background jobs
         format_date_fn: Function to format datetime to readable string
         duplicate_checker_fn: Function to check for duplicate facts
+        regenerate_observations_fn: Async function to regenerate observations for entities
         bank_id: Bank identifier
         contents_dicts: List of content dictionaries
         document_id: Optional document ID
@@ -102,7 +104,7 @@ async def retain_batch(
         agent_name,
         extract_opinions
     )
-    log_buffer.append(f"[1] Extract facts: {len(extracted_facts)} facts from {len(contents)} contents in {time.time() - step_start:.3f}s")
+    log_buffer.append(f"[1] Extract facts: {len(extracted_facts)} facts, {len(chunks)} chunks from {len(contents)} contents in {time.time() - step_start:.3f}s")
 
     if not extracted_facts:
         return [[] for _ in contents]
@@ -124,39 +126,140 @@ async def retain_batch(
         for extracted_fact, embedding in zip(extracted_facts, embeddings)
     ]
 
+    # Track document IDs for logging
+    document_ids_added = []
+
+    # Group contents by document_id for document tracking and chunk storage
+    from collections import defaultdict
+    contents_by_doc = defaultdict(list)
+    for idx, content_dict in enumerate(contents_dicts):
+        doc_id = content_dict.get("document_id")
+        contents_by_doc[doc_id].append((idx, content_dict))
+
     # Step 4: Database transaction
     async with acquire_with_retry(pool) as conn:
         async with conn.transaction():
             # Ensure bank exists
             await fact_storage.ensure_bank_exists(conn, bank_id)
 
-            # Handle document tracking
-            if document_id:
-                combined_content = "\n".join([c.get("content", "") for c in contents_dicts])
-                await fact_storage.handle_document_tracking(
-                    conn, bank_id, document_id, combined_content, is_first_batch
-                )
-            elif chunks:
-                # Generate document_id for chunk storage
-                document_id = str(uuid.uuid4())
-                combined_content = "\n".join([c.get("content", "") for c in contents_dicts])
-                await fact_storage.handle_document_tracking(
-                    conn, bank_id, document_id, combined_content, is_first_batch
-                )
-                log_buffer.append(f"[2.5] Generated document_id: {document_id}")
-
-            # Store chunks and map to facts
+            # Handle document tracking for all documents
             step_start = time.time()
-            chunk_id_map = {}
-            if document_id and chunks:
-                chunk_id_map = await chunk_storage.store_chunks_batch(conn, bank_id, document_id, chunks)
-                log_buffer.append(f"[3] Store chunks: {len(chunks)} chunks in {time.time() - step_start:.3f}s")
+            # Map None document_id to generated UUIDs
+            doc_id_mapping = {}  # Maps original doc_id (including None) to actual doc_id used
 
-                # Map chunk_ids to facts
-                facts_chunk_indices = [fact.chunk_index for fact in extracted_facts]
-                chunk_ids = chunk_storage.map_facts_to_chunks(facts_chunk_indices, chunk_id_map)
-                for processed_fact, chunk_id in zip(processed_facts, chunk_ids):
-                    processed_fact.chunk_id = chunk_id
+            if document_id:
+                # Legacy: single document_id parameter
+                combined_content = "\n".join([c.get("content", "") for c in contents_dicts])
+                retain_params = {}
+                if contents_dicts:
+                    first_item = contents_dicts[0]
+                    if first_item.get("context"):
+                        retain_params["context"] = first_item["context"]
+                    if first_item.get("event_date"):
+                        retain_params["event_date"] = first_item["event_date"].isoformat() if hasattr(first_item["event_date"], "isoformat") else str(first_item["event_date"])
+                    if first_item.get("metadata"):
+                        retain_params["metadata"] = first_item["metadata"]
+
+                await fact_storage.handle_document_tracking(
+                    conn, bank_id, document_id, combined_content, is_first_batch, retain_params
+                )
+                document_ids_added.append(document_id)
+                doc_id_mapping[None] = document_id  # For backwards compatibility
+            else:
+                # Handle per-item document_ids (create documents if any item has document_id or if chunks exist)
+                has_any_doc_ids = any(item.get("document_id") for item in contents_dicts)
+
+                if has_any_doc_ids or chunks:
+                    for original_doc_id, doc_contents in contents_by_doc.items():
+                        actual_doc_id = original_doc_id
+
+                        # Only create document record if:
+                        # 1. Item has explicit document_id, OR
+                        # 2. There are chunks (need document for chunk storage)
+                        should_create_doc = (original_doc_id is not None) or chunks
+
+                        if should_create_doc:
+                            if actual_doc_id is None:
+                                # No document_id but have chunks - generate one
+                                actual_doc_id = str(uuid.uuid4())
+
+                            # Store mapping for later use
+                            doc_id_mapping[original_doc_id] = actual_doc_id
+
+                            # Combine content for this document
+                            combined_content = "\n".join([c.get("content", "") for _, c in doc_contents])
+
+                            # Extract retain params from first content item
+                            retain_params = {}
+                            if doc_contents:
+                                first_item = doc_contents[0][1]
+                                if first_item.get("context"):
+                                    retain_params["context"] = first_item["context"]
+                                if first_item.get("event_date"):
+                                    retain_params["event_date"] = first_item["event_date"].isoformat() if hasattr(first_item["event_date"], "isoformat") else str(first_item["event_date"])
+                                if first_item.get("metadata"):
+                                    retain_params["metadata"] = first_item["metadata"]
+
+                            await fact_storage.handle_document_tracking(
+                                conn, bank_id, actual_doc_id, combined_content, is_first_batch, retain_params
+                            )
+                            document_ids_added.append(actual_doc_id)
+
+            if document_ids_added:
+                log_buffer.append(f"[2.5] Document tracking: {len(document_ids_added)} documents in {time.time() - step_start:.3f}s")
+
+            # Store chunks and map to facts for all documents
+            step_start = time.time()
+            chunk_id_map_by_doc = {}  # Maps (doc_id, chunk_index) -> chunk_id
+
+            if chunks:
+                # Group chunks by their source document
+                chunks_by_doc = defaultdict(list)
+                for chunk in chunks:
+                    # chunk.content_index tells us which content this chunk came from
+                    original_doc_id = contents_dicts[chunk.content_index].get("document_id")
+                    # Map to actual document_id (handles None -> generated UUID mapping)
+                    actual_doc_id = doc_id_mapping.get(original_doc_id, original_doc_id)
+                    if actual_doc_id is None and document_id:
+                        actual_doc_id = document_id
+                    chunks_by_doc[actual_doc_id].append(chunk)
+
+                # Store chunks for each document
+                for doc_id, doc_chunks in chunks_by_doc.items():
+                    chunk_id_map = await chunk_storage.store_chunks_batch(conn, bank_id, doc_id, doc_chunks)
+                    # Store mapping with document context
+                    for chunk_idx, chunk_id in chunk_id_map.items():
+                        chunk_id_map_by_doc[(doc_id, chunk_idx)] = chunk_id
+
+                log_buffer.append(f"[3] Store chunks: {len(chunks)} chunks for {len(chunks_by_doc)} documents in {time.time() - step_start:.3f}s")
+
+                # Map chunk_ids and document_ids to facts
+                for fact, processed_fact in zip(extracted_facts, processed_facts):
+                    # Get the original document_id for this fact's source content
+                    original_doc_id = contents_dicts[fact.content_index].get("document_id")
+                    # Map to actual document_id (handles None -> generated UUID mapping)
+                    actual_doc_id = doc_id_mapping.get(original_doc_id, original_doc_id)
+                    if actual_doc_id is None and document_id:
+                        actual_doc_id = document_id
+
+                    # Set document_id on the fact
+                    processed_fact.document_id = actual_doc_id
+
+                    # Map chunk_id if this fact came from a chunk
+                    if fact.chunk_index is not None:
+                        # Look up chunk_id using (doc_id, chunk_index)
+                        chunk_id = chunk_id_map_by_doc.get((actual_doc_id, fact.chunk_index))
+                        if chunk_id:
+                            processed_fact.chunk_id = chunk_id
+            else:
+                # No chunks - still need to set document_id on facts
+                for fact, processed_fact in zip(extracted_facts, processed_facts):
+                    original_doc_id = contents_dicts[fact.content_index].get("document_id")
+                    # Map to actual document_id (handles None -> generated UUID mapping)
+                    actual_doc_id = doc_id_mapping.get(original_doc_id, original_doc_id)
+                    if actual_doc_id is None and document_id:
+                        actual_doc_id = document_id
+                    processed_fact.document_id = actual_doc_id
 
             # Deduplication
             step_start = time.time()
@@ -171,15 +274,15 @@ async def retain_batch(
             if not non_duplicate_facts:
                 return [[] for _ in contents]
 
-            # Insert facts
+            # Insert facts (document_id is now stored per-fact)
             step_start = time.time()
-            unit_ids = await fact_storage.insert_facts_batch(conn, bank_id, non_duplicate_facts, document_id)
+            unit_ids = await fact_storage.insert_facts_batch(conn, bank_id, non_duplicate_facts)
             log_buffer.append(f"[5] Insert facts: {len(unit_ids)} units in {time.time() - step_start:.3f}s")
 
             # Process entities
             step_start = time.time()
             entity_links = await entity_processing.process_entities_batch(
-                entity_resolver, conn, bank_id, unit_ids, non_duplicate_facts
+                entity_resolver, conn, bank_id, unit_ids, non_duplicate_facts, log_buffer
             )
             log_buffer.append(f"[6] Process entities: {len(entity_links)} links in {time.time() - step_start:.3f}s")
 
@@ -213,20 +316,23 @@ async def retain_batch(
             total_time = time.time() - start_time
             log_buffer.append(f"{'='*60}")
             log_buffer.append(f"RETAIN_BATCH COMPLETE: {len(unit_ids)} units in {total_time:.3f}s")
+            if document_ids_added:
+                log_buffer.append(f"Documents: {', '.join(document_ids_added)}")
             log_buffer.append(f"{'='*60}")
 
             logger.info("\n" + "\n".join(log_buffer) + "\n")
 
-            # Trigger background tasks
-            await _trigger_background_tasks(
-                task_backend,
-                bank_id,
-                unit_ids,
-                non_duplicate_facts,
-                entity_links
-            )
+        # Trigger background tasks AFTER transaction commits
+        await _trigger_background_tasks(
+            task_backend,
+            regenerate_observations_fn,
+            bank_id,
+            unit_ids,
+            non_duplicate_facts,
+            entity_links
+        )
 
-            return result_unit_ids
+        return result_unit_ids
 
 
 def _map_results_to_contents(
@@ -261,12 +367,13 @@ def _map_results_to_contents(
 
 async def _trigger_background_tasks(
     task_backend,
+    regenerate_observations_fn,
     bank_id: str,
     unit_ids: List[str],
     facts: List[ProcessedFact],
     entity_links: List
 ) -> None:
-    """Trigger opinion reinforcement and observation regeneration tasks."""
+    """Trigger opinion reinforcement and observation regeneration (sync)."""
     # Trigger opinion reinforcement if there are entities
     fact_entities = [[e.name for e in fact.entities] for fact in facts]
     if any(fact_entities):
@@ -278,11 +385,11 @@ async def _trigger_background_tasks(
             'unit_entities': fact_entities
         })
 
-    # Trigger observation regeneration for top entities
+    # Regenerate observations synchronously for top entities
     TOP_N_ENTITIES = 5
     MIN_FACTS_THRESHOLD = 5
 
-    if entity_links:
+    if entity_links and regenerate_observations_fn:
         unique_entity_ids = set()
         for link in entity_links:
             # links are tuples: (unit_id, entity_id, confidence)
@@ -290,9 +397,9 @@ async def _trigger_background_tasks(
                 unique_entity_ids.add(str(link[1]))
 
         if unique_entity_ids:
-            await task_backend.submit_task({
-                'type': 'regenerate_observations',
-                'bank_id': bank_id,
-                'entity_ids': list(unique_entity_ids)[:TOP_N_ENTITIES],
-                'min_facts': MIN_FACTS_THRESHOLD
-            })
+            # Run observation regeneration synchronously
+            await regenerate_observations_fn(
+                bank_id=bank_id,
+                entity_ids=list(unique_entity_ids)[:TOP_N_ENTITIES],
+                min_facts=MIN_FACTS_THRESHOLD
+            )

hindsight-api/hindsight_api/engine/retain/types.py

@@ -79,6 +79,7 @@ class ExtractedFact:
     entities: List[str] = field(default_factory=list)
     occurred_start: Optional[datetime] = None
     occurred_end: Optional[datetime] = None
+    where: Optional[str] = None  # WHERE the fact occurred or is about
     causal_relations: List[CausalRelation] = field(default_factory=list)
 
     # Context from the content item
@@ -110,6 +111,9 @@ class ProcessedFact:
     context: str
     metadata: Dict[str, str]
 
+    # Location data
+    where: Optional[str] = None
+
     # Entities
     entities: List[EntityRef] = field(default_factory=list)
 
@@ -119,6 +123,9 @@ class ProcessedFact:
     # Chunk reference
     chunk_id: Optional[str] = None
 
+    # Document reference (denormalized for query performance)
+    document_id: Optional[str] = None
+
     # DB fields (set after insertion)
     unit_id: Optional[UUID] = None
 

hindsight-api/hindsight_api/engine/search/trace.py

@@ -59,7 +59,7 @@ class NodeVisit(BaseModel):
     node_id: str = Field(description="Memory unit ID")
     text: str = Field(description="Memory unit text content")
     context: str = Field(description="Memory unit context")
-    event_date: datetime = Field(description="When the memory occurred")
+    event_date: Optional[datetime] = Field(default=None, description="When the memory occurred")
     access_count: int = Field(description="Number of times accessed before this search")
 
     # How this node was reached
@@ -100,6 +100,7 @@ class RetrievalResult(BaseModel):
     text: str = Field(description="Memory unit text content")
     context: str = Field(default="", description="Memory unit context")
     event_date: Optional[datetime] = Field(default=None, description="When the memory occurred")
+    fact_type: Optional[str] = Field(default=None, description="Fact type (world, bank, opinion)")
     score: float = Field(description="Score from this retrieval method")
     score_name: str = Field(description="Name of the score (e.g., 'similarity', 'bm25_score', 'activation')")
 

hindsight-api/hindsight_api/engine/search/tracer.py

@@ -303,7 +303,9 @@ class SearchTracer:
         """
         retrieval_results = []
         for rank, (doc_id, data) in enumerate(results, start=1):
-            score = data.get(score_field, 0.0)
+            score = data.get(score_field)
+            if score is None:
+                score = 0.0
             retrieval_results.append(
                 RetrievalResult(
                     rank=rank,
@@ -311,6 +313,7 @@ class SearchTracer:
                     text=data.get("text", ""),
                     context=data.get("context", ""),
                     event_date=data.get("event_date"),
+                    fact_type=data.get("fact_type"),
                     score=score,
                     score_name=score_field,
                 )

hindsight-api/hindsight_api/pg0.py

@@ -28,7 +28,8 @@ def get_platform_binary_name() -> str:
 
     Supported platforms:
     - macOS ARM64 (darwin-aarch64)
-    - Linux x86_64
+    - Linux x86_64 (gnu)
+    - Linux ARM64 (gnu)
     - Windows x86_64
     """
     system = platform.system().lower()
@@ -42,19 +43,21 @@ def get_platform_binary_name() -> str:
     else:
         raise RuntimeError(
             f"Embedded PostgreSQL is not supported on architecture: {machine}. "
-            f"Supported architectures: x86_64/amd64 (Linux, Windows), aarch64/arm64 (macOS)"
+            f"Supported architectures: x86_64/amd64 (Linux, Windows), aarch64/arm64 (macOS, Linux)"
         )
 
     if system == "darwin" and arch == "aarch64":
         return "pg0-darwin-aarch64"
     elif system == "linux" and arch == "x86_64":
-        return "pg0-linux-x86_64"
+        return "pg0-linux-x86_64-gnu"
+    elif system == "linux" and arch == "aarch64":
+        return "pg0-linux-aarch64-gnu"
     elif system == "windows" and arch == "x86_64":
         return "pg0-windows-x86_64.exe"
     else:
         raise RuntimeError(
             f"Embedded PostgreSQL is not supported on {system}-{arch}. "
-            f"Supported platforms: darwin-aarch64 (macOS ARM), linux-x86_64, windows-x86_64"
+            f"Supported platforms: darwin-aarch64 (macOS ARM), linux-x86_64-gnu, linux-aarch64-gnu, windows-x86_64"
         )
 
 

hindsight-api/tests/test_fact_extraction_quality.py

@@ -15,7 +15,7 @@ produces semantically correct and complete facts.
 import pytest
 import re
 from datetime import datetime, timezone
-from hindsight_api.engine.fact_extraction import extract_facts_from_text
+from hindsight_api.engine.retain.fact_extraction import extract_facts_from_text
 from hindsight_api import LLMConfig
 
 

hindsight-clients/python/.openapi-generator/FILES

@@ -13,6 +13,7 @@ hindsight_client_api/docs/BankProfileResponse.md
 hindsight_client_api/docs/Budget.md
 hindsight_client_api/docs/ChunkData.md
 hindsight_client_api/docs/ChunkIncludeOptions.md
+hindsight_client_api/docs/ChunkResponse.md
 hindsight_client_api/docs/CreateBankRequest.md
 hindsight_client_api/docs/DefaultApi.md
 hindsight_client_api/docs/DeleteResponse.md
@@ -54,6 +55,7 @@ hindsight_client_api/models/bank_profile_response.py
 hindsight_client_api/models/budget.py
 hindsight_client_api/models/chunk_data.py
 hindsight_client_api/models/chunk_include_options.py
+hindsight_client_api/models/chunk_response.py
 hindsight_client_api/models/create_bank_request.py
 hindsight_client_api/models/delete_response.py
 hindsight_client_api/models/document_response.py
@@ -93,6 +95,7 @@ hindsight_client_api/test/test_bank_profile_response.py
 hindsight_client_api/test/test_budget.py
 hindsight_client_api/test/test_chunk_data.py
 hindsight_client_api/test/test_chunk_include_options.py
+hindsight_client_api/test/test_chunk_response.py
 hindsight_client_api/test/test_create_bank_request.py
 hindsight_client_api/test/test_default_api.py
 hindsight_client_api/test/test_delete_response.py

hindsight-clients/python/hindsight_client_api/__init__.py

@@ -37,6 +37,7 @@ __all__ = [
     "Budget",
     "ChunkData",
     "ChunkIncludeOptions",
+    "ChunkResponse",
     "CreateBankRequest",
     "DeleteResponse",
     "DocumentResponse",
@@ -92,6 +93,7 @@ from hindsight_client_api.models.bank_profile_response import BankProfileRespons
 from hindsight_client_api.models.budget import Budget as Budget
 from hindsight_client_api.models.chunk_data import ChunkData as ChunkData
 from hindsight_client_api.models.chunk_include_options import ChunkIncludeOptions as ChunkIncludeOptions
+from hindsight_client_api.models.chunk_response import ChunkResponse as ChunkResponse
 from hindsight_client_api.models.create_bank_request import CreateBankRequest as CreateBankRequest
 from hindsight_client_api.models.delete_response import DeleteResponse as DeleteResponse
 from hindsight_client_api.models.document_response import DocumentResponse as DocumentResponse

hindsight-clients/python/hindsight_client_api/api/default_api.py

@@ -23,6 +23,7 @@ from hindsight_client_api.models.add_background_request import AddBackgroundRequ
 from hindsight_client_api.models.background_response import BackgroundResponse
 from hindsight_client_api.models.bank_list_response import BankListResponse
 from hindsight_client_api.models.bank_profile_response import BankProfileResponse
+from hindsight_client_api.models.chunk_response import ChunkResponse
 from hindsight_client_api.models.create_bank_request import CreateBankRequest
 from hindsight_client_api.models.delete_response import DeleteResponse
 from hindsight_client_api.models.document_response import DocumentResponse
@@ -2001,6 +2002,269 @@ class DefaultApi:
 
 
 
+    @validate_call
+    async def get_chunk(
+        self,
+        chunk_id: StrictStr,
+        _request_timeout: Union[
+            None,
+            Annotated[StrictFloat, Field(gt=0)],
+            Tuple[
+                Annotated[StrictFloat, Field(gt=0)],
+                Annotated[StrictFloat, Field(gt=0)]
+            ]
+        ] = None,
+        _request_auth: Optional[Dict[StrictStr, Any]] = None,
+        _content_type: Optional[StrictStr] = None,
+        _headers: Optional[Dict[StrictStr, Any]] = None,
+        _host_index: Annotated[StrictInt, Field(ge=0, le=0)] = 0,
+    ) -> ChunkResponse:
+        """Get chunk details
+
+        Get a specific chunk by its ID
+
+        :param chunk_id: (required)
+        :type chunk_id: str
+        :param _request_timeout: timeout setting for this request. If one
+                                 number provided, it will be total request
+                                 timeout. It can also be a pair (tuple) of
+                                 (connection, read) timeouts.
+        :type _request_timeout: int, tuple(int, int), optional
+        :param _request_auth: set to override the auth_settings for an a single
+                              request; this effectively ignores the
+                              authentication in the spec for a single request.
+        :type _request_auth: dict, optional
+        :param _content_type: force content-type for the request.
+        :type _content_type: str, Optional
+        :param _headers: set to override the headers for a single
+                         request; this effectively ignores the headers
+                         in the spec for a single request.
+        :type _headers: dict, optional
+        :param _host_index: set to override the host_index for a single
+                            request; this effectively ignores the host_index
+                            in the spec for a single request.
+        :type _host_index: int, optional
+        :return: Returns the result object.
+        """ # noqa: E501
+
+        _param = self._get_chunk_serialize(
+            chunk_id=chunk_id,
+            _request_auth=_request_auth,
+            _content_type=_content_type,
+            _headers=_headers,
+            _host_index=_host_index
+        )
+
+        _response_types_map: Dict[str, Optional[str]] = {
+            '200': "ChunkResponse",
+            '422': "HTTPValidationError",
+        }
+        response_data = await self.api_client.call_api(
+            *_param,
+            _request_timeout=_request_timeout
+        )
+        await response_data.read()
+        return self.api_client.response_deserialize(
+            response_data=response_data,
+            response_types_map=_response_types_map,
+        ).data
+
+
+    @validate_call
+    async def get_chunk_with_http_info(
+        self,
+        chunk_id: StrictStr,
+        _request_timeout: Union[
+            None,
+            Annotated[StrictFloat, Field(gt=0)],
+            Tuple[
+                Annotated[StrictFloat, Field(gt=0)],
+                Annotated[StrictFloat, Field(gt=0)]
+            ]
+        ] = None,
+        _request_auth: Optional[Dict[StrictStr, Any]] = None,
+        _content_type: Optional[StrictStr] = None,
+        _headers: Optional[Dict[StrictStr, Any]] = None,
+        _host_index: Annotated[StrictInt, Field(ge=0, le=0)] = 0,
+    ) -> ApiResponse[ChunkResponse]:
+        """Get chunk details
+
+        Get a specific chunk by its ID
+
+        :param chunk_id: (required)
+        :type chunk_id: str
+        :param _request_timeout: timeout setting for this request. If one
+                                 number provided, it will be total request
+                                 timeout. It can also be a pair (tuple) of
+                                 (connection, read) timeouts.
+        :type _request_timeout: int, tuple(int, int), optional
+        :param _request_auth: set to override the auth_settings for an a single
+                              request; this effectively ignores the
+                              authentication in the spec for a single request.
+        :type _request_auth: dict, optional
+        :param _content_type: force content-type for the request.
+        :type _content_type: str, Optional
+        :param _headers: set to override the headers for a single
+                         request; this effectively ignores the headers
+                         in the spec for a single request.
+        :type _headers: dict, optional
+        :param _host_index: set to override the host_index for a single
+                            request; this effectively ignores the host_index
+                            in the spec for a single request.
+        :type _host_index: int, optional
+        :return: Returns the result object.
+        """ # noqa: E501
+
+        _param = self._get_chunk_serialize(
+            chunk_id=chunk_id,
+            _request_auth=_request_auth,
+            _content_type=_content_type,
+            _headers=_headers,
+            _host_index=_host_index
+        )
+
+        _response_types_map: Dict[str, Optional[str]] = {
+            '200': "ChunkResponse",
+            '422': "HTTPValidationError",
+        }
+        response_data = await self.api_client.call_api(
+            *_param,
+            _request_timeout=_request_timeout
+        )
+        await response_data.read()
+        return self.api_client.response_deserialize(
+            response_data=response_data,
+            response_types_map=_response_types_map,
+        )
+
+
+    @validate_call
+    async def get_chunk_without_preload_content(
+        self,
+        chunk_id: StrictStr,
+        _request_timeout: Union[
+            None,
+            Annotated[StrictFloat, Field(gt=0)],
+            Tuple[
+                Annotated[StrictFloat, Field(gt=0)],
+                Annotated[StrictFloat, Field(gt=0)]
+            ]
+        ] = None,
+        _request_auth: Optional[Dict[StrictStr, Any]] = None,
+        _content_type: Optional[StrictStr] = None,
+        _headers: Optional[Dict[StrictStr, Any]] = None,
+        _host_index: Annotated[StrictInt, Field(ge=0, le=0)] = 0,
+    ) -> RESTResponseType:
+        """Get chunk details
+
+        Get a specific chunk by its ID
+
+        :param chunk_id: (required)
+        :type chunk_id: str
+        :param _request_timeout: timeout setting for this request. If one
+                                 number provided, it will be total request
+                                 timeout. It can also be a pair (tuple) of
+                                 (connection, read) timeouts.
+        :type _request_timeout: int, tuple(int, int), optional
+        :param _request_auth: set to override the auth_settings for an a single
+                              request; this effectively ignores the
+                              authentication in the spec for a single request.
+        :type _request_auth: dict, optional
+        :param _content_type: force content-type for the request.
+        :type _content_type: str, Optional
+        :param _headers: set to override the headers for a single
+                         request; this effectively ignores the headers
+                         in the spec for a single request.
+        :type _headers: dict, optional
+        :param _host_index: set to override the host_index for a single
+                            request; this effectively ignores the host_index
+                            in the spec for a single request.
+        :type _host_index: int, optional
+        :return: Returns the result object.
+        """ # noqa: E501
+
+        _param = self._get_chunk_serialize(
+            chunk_id=chunk_id,
+            _request_auth=_request_auth,
+            _content_type=_content_type,
+            _headers=_headers,
+            _host_index=_host_index
+        )
+
+        _response_types_map: Dict[str, Optional[str]] = {
+            '200': "ChunkResponse",
+            '422': "HTTPValidationError",
+        }
+        response_data = await self.api_client.call_api(
+            *_param,
+            _request_timeout=_request_timeout
+        )
+        return response_data.response
+
+
+    def _get_chunk_serialize(
+        self,
+        chunk_id,
+        _request_auth,
+        _content_type,
+        _headers,
+        _host_index,
+    ) -> RequestSerialized:
+
+        _host = None
+
+        _collection_formats: Dict[str, str] = {
+        }
+
+        _path_params: Dict[str, str] = {}
+        _query_params: List[Tuple[str, str]] = []
+        _header_params: Dict[str, Optional[str]] = _headers or {}
+        _form_params: List[Tuple[str, str]] = []
+        _files: Dict[
+            str, Union[str, bytes, List[str], List[bytes], List[Tuple[str, bytes]]]
+        ] = {}
+        _body_params: Optional[bytes] = None
+
+        # process the path parameters
+        if chunk_id is not None:
+            _path_params['chunk_id'] = chunk_id
+        # process the query parameters
+        # process the header parameters
+        # process the form parameters
+        # process the body parameter
+
+
+        # set the HTTP header `Accept`
+        if 'Accept' not in _header_params:
+            _header_params['Accept'] = self.api_client.select_header_accept(
+                [
+                    'application/json'
+                ]
+            )
+
+
+        # authentication setting
+        _auth_settings: List[str] = [
+        ]
+
+        return self.api_client.param_serialize(
+            method='GET',
+            resource_path='/v1/default/chunks/{chunk_id}',
+            path_params=_path_params,
+            query_params=_query_params,
+            header_params=_header_params,
+            body=_body_params,
+            post_params=_form_params,
+            files=_files,
+            auth_settings=_auth_settings,
+            collection_formats=_collection_formats,
+            _host=_host,
+            _request_auth=_request_auth
+        )
+
+
+
+
     @validate_call
     async def get_document(
         self,
@@ -5150,7 +5414,7 @@ class DefaultApi:
     ) -> RetainResponse:
         """Retain memories
 
-        Retain memory items with automatic fact extraction.      This is the main endpoint for storing memories. It supports both synchronous and asynchronous processing     via the async parameter.      Features:     - Efficient batch processing     - Automatic fact extraction from natural language     - Entity recognition and linking     - Document tracking with automatic upsert (when document_id is provided)     - Temporal and semantic linking     - Optional asynchronous processing      The system automatically:     1. Extracts semantic facts from the content     2. Generates embeddings     3. Deduplicates similar facts     4. Creates temporal, semantic, and entity links     5. Tracks document metadata      When async=true:     - Returns immediately after queuing the task     - Processing happens in the background     - Use the operations endpoint to monitor progress      When async=false (default):     - Waits for processing to complete     - Returns after all memories are stored      Note: If document_id is provided and already exists, the old document and its memory units will be deleted before creating new ones (upsert behavior).
+        Retain memory items with automatic fact extraction.      This is the main endpoint for storing memories. It supports both synchronous and asynchronous processing     via the async parameter.      Features:     - Efficient batch processing     - Automatic fact extraction from natural language     - Entity recognition and linking     - Document tracking with automatic upsert (when document_id is provided on items)     - Temporal and semantic linking     - Optional asynchronous processing      The system automatically:     1. Extracts semantic facts from the content     2. Generates embeddings     3. Deduplicates similar facts     4. Creates temporal, semantic, and entity links     5. Tracks document metadata      When async=true:     - Returns immediately after queuing the task     - Processing happens in the background     - Use the operations endpoint to monitor progress      When async=false (default):     - Waits for processing to complete     - Returns after all memories are stored      Note: If a memory item has a document_id that already exists, the old document and its memory units will be deleted before creating new ones (upsert behavior). Items with the same document_id are grouped together for efficient processing.
 
         :param bank_id: (required)
         :type bank_id: str
@@ -5222,7 +5486,7 @@ class DefaultApi:
     ) -> ApiResponse[RetainResponse]:
         """Retain memories
 
-        Retain memory items with automatic fact extraction.      This is the main endpoint for storing memories. It supports both synchronous and asynchronous processing     via the async parameter.      Features:     - Efficient batch processing     - Automatic fact extraction from natural language     - Entity recognition and linking     - Document tracking with automatic upsert (when document_id is provided)     - Temporal and semantic linking     - Optional asynchronous processing      The system automatically:     1. Extracts semantic facts from the content     2. Generates embeddings     3. Deduplicates similar facts     4. Creates temporal, semantic, and entity links     5. Tracks document metadata      When async=true:     - Returns immediately after queuing the task     - Processing happens in the background     - Use the operations endpoint to monitor progress      When async=false (default):     - Waits for processing to complete     - Returns after all memories are stored      Note: If document_id is provided and already exists, the old document and its memory units will be deleted before creating new ones (upsert behavior).
+        Retain memory items with automatic fact extraction.      This is the main endpoint for storing memories. It supports both synchronous and asynchronous processing     via the async parameter.      Features:     - Efficient batch processing     - Automatic fact extraction from natural language     - Entity recognition and linking     - Document tracking with automatic upsert (when document_id is provided on items)     - Temporal and semantic linking     - Optional asynchronous processing      The system automatically:     1. Extracts semantic facts from the content     2. Generates embeddings     3. Deduplicates similar facts     4. Creates temporal, semantic, and entity links     5. Tracks document metadata      When async=true:     - Returns immediately after queuing the task     - Processing happens in the background     - Use the operations endpoint to monitor progress      When async=false (default):     - Waits for processing to complete     - Returns after all memories are stored      Note: If a memory item has a document_id that already exists, the old document and its memory units will be deleted before creating new ones (upsert behavior). Items with the same document_id are grouped together for efficient processing.
 
         :param bank_id: (required)
         :type bank_id: str
@@ -5294,7 +5558,7 @@ class DefaultApi:
     ) -> RESTResponseType:
         """Retain memories
 
-        Retain memory items with automatic fact extraction.      This is the main endpoint for storing memories. It supports both synchronous and asynchronous processing     via the async parameter.      Features:     - Efficient batch processing     - Automatic fact extraction from natural language     - Entity recognition and linking     - Document tracking with automatic upsert (when document_id is provided)     - Temporal and semantic linking     - Optional asynchronous processing      The system automatically:     1. Extracts semantic facts from the content     2. Generates embeddings     3. Deduplicates similar facts     4. Creates temporal, semantic, and entity links     5. Tracks document metadata      When async=true:     - Returns immediately after queuing the task     - Processing happens in the background     - Use the operations endpoint to monitor progress      When async=false (default):     - Waits for processing to complete     - Returns after all memories are stored      Note: If document_id is provided and already exists, the old document and its memory units will be deleted before creating new ones (upsert behavior).
+        Retain memory items with automatic fact extraction.      This is the main endpoint for storing memories. It supports both synchronous and asynchronous processing     via the async parameter.      Features:     - Efficient batch processing     - Automatic fact extraction from natural language     - Entity recognition and linking     - Document tracking with automatic upsert (when document_id is provided on items)     - Temporal and semantic linking     - Optional asynchronous processing      The system automatically:     1. Extracts semantic facts from the content     2. Generates embeddings     3. Deduplicates similar facts     4. Creates temporal, semantic, and entity links     5. Tracks document metadata      When async=true:     - Returns immediately after queuing the task     - Processing happens in the background     - Use the operations endpoint to monitor progress      When async=false (default):     - Waits for processing to complete     - Returns after all memories are stored      Note: If a memory item has a document_id that already exists, the old document and its memory units will be deleted before creating new ones (upsert behavior). Items with the same document_id are grouped together for efficient processing.
 
         :param bank_id: (required)
         :type bank_id: str

hindsight-clients/python/hindsight_client_api/docs/ChunkResponse.md

@@ -0,0 +1,35 @@
+# ChunkResponse
+
+Response model for get chunk endpoint.
+
+## Properties
+
+Name | Type | Description | Notes
+------------ | ------------- | ------------- | -------------
+**chunk_id** | **str** |  | 
+**document_id** | **str** |  | 
+**bank_id** | **str** |  | 
+**chunk_index** | **int** |  | 
+**chunk_text** | **str** |  | 
+**created_at** | **str** |  | 
+
+## Example
+
+```python
+from hindsight_client_api.models.chunk_response import ChunkResponse
+
+# TODO update the JSON string below
+json = "{}"
+# create an instance of ChunkResponse from a JSON string
+chunk_response_instance = ChunkResponse.from_json(json)
+# print the JSON string representation of the object
+print(ChunkResponse.to_json())
+
+# convert the object into a dict
+chunk_response_dict = chunk_response_instance.to_dict()
+# create an instance of ChunkResponse from a dict
+chunk_response_from_dict = ChunkResponse.from_dict(chunk_response_dict)
+```
+[[Back to Model list]](../README.md#documentation-for-models) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to README]](../README.md)
+
+

hindsight-clients/python/hindsight_client_api/docs/DefaultApi.md

@@ -11,6 +11,7 @@ Method | HTTP request | Description
 [**delete_document**](DefaultApi.md#delete_document) | **DELETE** /v1/default/banks/{bank_id}/documents/{document_id} | Delete a document
 [**get_agent_stats**](DefaultApi.md#get_agent_stats) | **GET** /v1/default/banks/{bank_id}/stats | Get statistics for memory bank
 [**get_bank_profile**](DefaultApi.md#get_bank_profile) | **GET** /v1/default/banks/{bank_id}/profile | Get memory bank profile
+[**get_chunk**](DefaultApi.md#get_chunk) | **GET** /v1/default/chunks/{chunk_id} | Get chunk details
 [**get_document**](DefaultApi.md#get_document) | **GET** /v1/default/banks/{bank_id}/documents/{document_id} | Get document details
 [**get_entity**](DefaultApi.md#get_entity) | **GET** /v1/default/banks/{bank_id}/entities/{entity_id} | Get entity details
 [**get_graph**](DefaultApi.md#get_graph) | **GET** /v1/default/banks/{bank_id}/graph | Get memory graph data
@@ -525,6 +526,75 @@ No authorization required
 
 [[Back to top]](#) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to Model list]](../README.md#documentation-for-models) [[Back to README]](../README.md)
 
+# **get_chunk**
+> ChunkResponse get_chunk(chunk_id)
+
+Get chunk details
+
+Get a specific chunk by its ID
+
+### Example
+
+
+```python
+import hindsight_client_api
+from hindsight_client_api.models.chunk_response import ChunkResponse
+from hindsight_client_api.rest import ApiException
+from pprint import pprint
+
+# Defining the host is optional and defaults to http://localhost
+# See configuration.py for a list of all supported configuration parameters.
+configuration = hindsight_client_api.Configuration(
+    host = "http://localhost"
+)
+
+
+# Enter a context with an instance of the API client
+async with hindsight_client_api.ApiClient(configuration) as api_client:
+    # Create an instance of the API class
+    api_instance = hindsight_client_api.DefaultApi(api_client)
+    chunk_id = 'chunk_id_example' # str | 
+
+    try:
+        # Get chunk details
+        api_response = await api_instance.get_chunk(chunk_id)
+        print("The response of DefaultApi->get_chunk:\n")
+        pprint(api_response)
+    except Exception as e:
+        print("Exception when calling DefaultApi->get_chunk: %s\n" % e)
+```
+
+
+
+### Parameters
+
+
+Name | Type | Description  | Notes
+------------- | ------------- | ------------- | -------------
+ **chunk_id** | **str**|  | 
+
+### Return type
+
+[**ChunkResponse**](ChunkResponse.md)
+
+### Authorization
+
+No authorization required
+
+### HTTP request headers
+
+ - **Content-Type**: Not defined
+ - **Accept**: application/json
+
+### HTTP response details
+
+| Status code | Description | Response headers |
+|-------------|-------------|------------------|
+**200** | Successful Response |  -  |
+**422** | Validation Error |  -  |
+
+[[Back to top]](#) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to Model list]](../README.md#documentation-for-models) [[Back to README]](../README.md)
+
 # **get_document**
 > DocumentResponse get_document(bank_id, document_id)
 
@@ -1338,7 +1408,7 @@ Retain memory items with automatic fact extraction.
     - Efficient batch processing
     - Automatic fact extraction from natural language
     - Entity recognition and linking
-    - Document tracking with automatic upsert (when document_id is provided)
+    - Document tracking with automatic upsert (when document_id is provided on items)
     - Temporal and semantic linking
     - Optional asynchronous processing
 
@@ -1358,7 +1428,7 @@ Retain memory items with automatic fact extraction.
     - Waits for processing to complete
     - Returns after all memories are stored
 
-    Note: If document_id is provided and already exists, the old document and its memory units will be deleted before creating new ones (upsert behavior).
+    Note: If a memory item has a document_id that already exists, the old document and its memory units will be deleted before creating new ones (upsert behavior). Items with the same document_id are grouped together for efficient processing.
 
 ### Example
 

hindsight-clients/python/hindsight_client_api/docs/MemoryItem.md

@@ -10,6 +10,7 @@ Name | Type | Description | Notes
 **timestamp** | **datetime** |  | [optional] 
 **context** | **str** |  | [optional] 
 **metadata** | **Dict[str, str]** |  | [optional] 
+**document_id** | **str** |  | [optional] 
 
 ## Example
 

hindsight-clients/python/hindsight_client_api/docs/ReflectIncludeOptions.md

@@ -7,7 +7,6 @@ Options for including additional data in reflect results.
 Name | Type | Description | Notes
 ------------ | ------------- | ------------- | -------------
 **facts** | **object** | Options for including facts (based_on) in reflect results. | [optional] 
-**entities** | [**EntityIncludeOptions**](EntityIncludeOptions.md) |  | [optional] 
 
 ## Example
 

hindsight-clients/python/hindsight_client_api/docs/ReflectRequest.md

@@ -10,7 +10,7 @@ Name | Type | Description | Notes
 **budget** | [**Budget**](Budget.md) |  | [optional] 
 **context** | **str** |  | [optional] 
 **filters** | [**List[MetadataFilter]**](MetadataFilter.md) |  | [optional] 
-**include** | [**ReflectIncludeOptions**](ReflectIncludeOptions.md) | Options for including additional data (both disabled by default) | [optional] 
+**include** | [**ReflectIncludeOptions**](ReflectIncludeOptions.md) | Options for including additional data (disabled by default) | [optional] 
 
 ## Example
 

hindsight-clients/python/hindsight_client_api/docs/RetainRequest.md

@@ -7,7 +7,6 @@ Request model for retain endpoint.
 Name | Type | Description | Notes
 ------------ | ------------- | ------------- | -------------
 **items** | [**List[MemoryItem]**](MemoryItem.md) |  | 
-**document_id** | **str** |  | [optional] 
 **var_async** | **bool** | If true, process asynchronously in background. If false, wait for completion (default: false) | [optional] [default to False]
 
 ## Example

hindsight-clients/python/hindsight_client_api/docs/RetainResponse.md

@@ -8,7 +8,6 @@ Name | Type | Description | Notes
 ------------ | ------------- | ------------- | -------------
 **success** | **bool** |  | 
 **bank_id** | **str** |  | 
-**document_id** | **str** |  | [optional] 
 **items_count** | **int** |  | 
 **var_async** | **bool** | Whether the operation was processed asynchronously | 
 

hindsight-clients/python/hindsight_client_api/models/__init__.py

@@ -21,6 +21,7 @@ from hindsight_client_api.models.bank_profile_response import BankProfileRespons
 from hindsight_client_api.models.budget import Budget
 from hindsight_client_api.models.chunk_data import ChunkData
 from hindsight_client_api.models.chunk_include_options import ChunkIncludeOptions
+from hindsight_client_api.models.chunk_response import ChunkResponse
 from hindsight_client_api.models.create_bank_request import CreateBankRequest
 from hindsight_client_api.models.delete_response import DeleteResponse
 from hindsight_client_api.models.document_response import DocumentResponse

hindsight-clients/python/hindsight_client_api/models/chunk_response.py

@@ -0,0 +1,97 @@
+# coding: utf-8
+
+"""
+    Hindsight HTTP API
+
+    HTTP API for Hindsight
+
+    The version of the OpenAPI document: 1.0.0
+    Generated by OpenAPI Generator (https://openapi-generator.tech)
+
+    Do not edit the class manually.
+"""  # noqa: E501
+
+
+from __future__ import annotations
+import pprint
+import re  # noqa: F401
+import json
+
+from pydantic import BaseModel, ConfigDict, StrictInt, StrictStr
+from typing import Any, ClassVar, Dict, List
+from typing import Optional, Set
+from typing_extensions import Self
+
+class ChunkResponse(BaseModel):
+    """
+    Response model for get chunk endpoint.
+    """ # noqa: E501
+    chunk_id: StrictStr
+    document_id: StrictStr
+    bank_id: StrictStr
+    chunk_index: StrictInt
+    chunk_text: StrictStr
+    created_at: StrictStr
+    __properties: ClassVar[List[str]] = ["chunk_id", "document_id", "bank_id", "chunk_index", "chunk_text", "created_at"]
+
+    model_config = ConfigDict(
+        populate_by_name=True,
+        validate_assignment=True,
+        protected_namespaces=(),
+    )
+
+
+    def to_str(self) -> str:
+        """Returns the string representation of the model using alias"""
+        return pprint.pformat(self.model_dump(by_alias=True))
+
+    def to_json(self) -> str:
+        """Returns the JSON representation of the model using alias"""
+        # TODO: pydantic v2: use .model_dump_json(by_alias=True, exclude_unset=True) instead
+        return json.dumps(self.to_dict())
+
+    @classmethod
+    def from_json(cls, json_str: str) -> Optional[Self]:
+        """Create an instance of ChunkResponse from a JSON string"""
+        return cls.from_dict(json.loads(json_str))
+
+    def to_dict(self) -> Dict[str, Any]:
+        """Return the dictionary representation of the model using alias.
+
+        This has the following differences from calling pydantic's
+        `self.model_dump(by_alias=True)`:
+
+        * `None` is only added to the output dict for nullable fields that
+          were set at model initialization. Other fields with value `None`
+          are ignored.
+        """
+        excluded_fields: Set[str] = set([
+        ])
+
+        _dict = self.model_dump(
+            by_alias=True,
+            exclude=excluded_fields,
+            exclude_none=True,
+        )
+        return _dict
+
+    @classmethod
+    def from_dict(cls, obj: Optional[Dict[str, Any]]) -> Optional[Self]:
+        """Create an instance of ChunkResponse from a dict"""
+        if obj is None:
+            return None
+
+        if not isinstance(obj, dict):
+            return cls.model_validate(obj)
+
+        _obj = cls.model_validate({
+            "chunk_id": obj.get("chunk_id"),
+            "document_id": obj.get("document_id"),
+            "bank_id": obj.get("bank_id"),
+            "chunk_index": obj.get("chunk_index"),
+            "chunk_text": obj.get("chunk_text"),
+            "created_at": obj.get("created_at")
+        })
+        return _obj
+
+

hindsight-clients/python/hindsight_client_api/models/memory_item.py

@@ -31,7 +31,8 @@ class MemoryItem(BaseModel):
     timestamp: Optional[datetime] = None
     context: Optional[StrictStr] = None
     metadata: Optional[Dict[str, StrictStr]] = None
-    __properties: ClassVar[List[str]] = ["content", "timestamp", "context", "metadata"]
+    document_id: Optional[StrictStr] = None
+    __properties: ClassVar[List[str]] = ["content", "timestamp", "context", "metadata", "document_id"]
 
     model_config = ConfigDict(
         populate_by_name=True,
@@ -87,6 +88,11 @@ class MemoryItem(BaseModel):
         if self.metadata is None and "metadata" in self.model_fields_set:
             _dict['metadata'] = None
 
+        # set to None if document_id (nullable) is None
+        # and model_fields_set contains the field
+        if self.document_id is None and "document_id" in self.model_fields_set:
+            _dict['document_id'] = None
+
         return _dict
 
     @classmethod
@@ -102,7 +108,8 @@ class MemoryItem(BaseModel):
             "content": obj.get("content"),
             "timestamp": obj.get("timestamp"),
             "context": obj.get("context"),
-            "metadata": obj.get("metadata")
+            "metadata": obj.get("metadata"),
+            "document_id": obj.get("document_id")
         })
         return _obj
 

hindsight-clients/python/hindsight_client_api/models/reflect_include_options.py

@@ -19,7 +19,6 @@ import json
 
 from pydantic import BaseModel, ConfigDict, Field
 from typing import Any, ClassVar, Dict, List, Optional
-from hindsight_client_api.models.entity_include_options import EntityIncludeOptions
 from typing import Optional, Set
 from typing_extensions import Self
 
@@ -28,8 +27,7 @@ class ReflectIncludeOptions(BaseModel):
     Options for including additional data in reflect results.
     """ # noqa: E501
     facts: Optional[Dict[str, Any]] = Field(default=None, description="Options for including facts (based_on) in reflect results.")
-    entities: Optional[EntityIncludeOptions] = None
-    __properties: ClassVar[List[str]] = ["facts", "entities"]
+    __properties: ClassVar[List[str]] = ["facts"]
 
     model_config = ConfigDict(
         populate_by_name=True,
@@ -70,14 +68,6 @@ class ReflectIncludeOptions(BaseModel):
             exclude=excluded_fields,
             exclude_none=True,
         )
-        # override the default output from pydantic by calling `to_dict()` of entities
-        if self.entities:
-            _dict['entities'] = self.entities.to_dict()
-        # set to None if entities (nullable) is None
-        # and model_fields_set contains the field
-        if self.entities is None and "entities" in self.model_fields_set:
-            _dict['entities'] = None
-
         return _dict
 
     @classmethod
@@ -90,8 +80,7 @@ class ReflectIncludeOptions(BaseModel):
             return cls.model_validate(obj)
 
         _obj = cls.model_validate({
-            "facts": obj.get("facts"),
-            "entities": EntityIncludeOptions.from_dict(obj["entities"]) if obj.get("entities") is not None else None
+            "facts": obj.get("facts")
         })
         return _obj
 

hindsight-clients/python/hindsight_client_api/models/reflect_request.py

@@ -33,7 +33,7 @@ class ReflectRequest(BaseModel):
     budget: Optional[Budget] = None
     context: Optional[StrictStr] = None
     filters: Optional[List[MetadataFilter]] = None
-    include: Optional[ReflectIncludeOptions] = Field(default=None, description="Options for including additional data (both disabled by default)")
+    include: Optional[ReflectIncludeOptions] = Field(default=None, description="Options for including additional data (disabled by default)")
     __properties: ClassVar[List[str]] = ["query", "budget", "context", "filters", "include"]
 
     model_config = ConfigDict(

hindsight-clients/python/hindsight_client_api/models/retain_request.py

@@ -17,7 +17,7 @@ import pprint
 import re  # noqa: F401
 import json
 
-from pydantic import BaseModel, ConfigDict, Field, StrictBool, StrictStr
+from pydantic import BaseModel, ConfigDict, Field, StrictBool
 from typing import Any, ClassVar, Dict, List, Optional
 from hindsight_client_api.models.memory_item import MemoryItem
 from typing import Optional, Set
@@ -28,9 +28,8 @@ class RetainRequest(BaseModel):
     Request model for retain endpoint.
     """ # noqa: E501
     items: List[MemoryItem]
-    document_id: Optional[StrictStr] = None
     var_async: Optional[StrictBool] = Field(default=False, description="If true, process asynchronously in background. If false, wait for completion (default: false)", alias="async")
-    __properties: ClassVar[List[str]] = ["items", "document_id", "async"]
+    __properties: ClassVar[List[str]] = ["items", "async"]
 
     model_config = ConfigDict(
         populate_by_name=True,
@@ -78,11 +77,6 @@ class RetainRequest(BaseModel):
                 if _item_items:
                     _items.append(_item_items.to_dict())
             _dict['items'] = _items
-        # set to None if document_id (nullable) is None
-        # and model_fields_set contains the field
-        if self.document_id is None and "document_id" in self.model_fields_set:
-            _dict['document_id'] = None
-
         return _dict
 
     @classmethod
@@ -96,7 +90,6 @@ class RetainRequest(BaseModel):
 
         _obj = cls.model_validate({
             "items": [MemoryItem.from_dict(_item) for _item in obj["items"]] if obj.get("items") is not None else None,
-            "document_id": obj.get("document_id"),
             "async": obj.get("async") if obj.get("async") is not None else False
         })
         return _obj

hindsight-clients/python/hindsight_client_api/models/retain_response.py

@@ -18,7 +18,7 @@ import re  # noqa: F401
 import json
 
 from pydantic import BaseModel, ConfigDict, Field, StrictBool, StrictInt, StrictStr
-from typing import Any, ClassVar, Dict, List, Optional
+from typing import Any, ClassVar, Dict, List
 from typing import Optional, Set
 from typing_extensions import Self
 
@@ -28,10 +28,9 @@ class RetainResponse(BaseModel):
     """ # noqa: E501
     success: StrictBool
     bank_id: StrictStr
-    document_id: Optional[StrictStr] = None
     items_count: StrictInt
     var_async: StrictBool = Field(description="Whether the operation was processed asynchronously", alias="async")
-    __properties: ClassVar[List[str]] = ["success", "bank_id", "document_id", "items_count", "async"]
+    __properties: ClassVar[List[str]] = ["success", "bank_id", "items_count", "async"]
 
     model_config = ConfigDict(
         populate_by_name=True,
@@ -72,11 +71,6 @@ class RetainResponse(BaseModel):
             exclude=excluded_fields,
             exclude_none=True,
         )
-        # set to None if document_id (nullable) is None
-        # and model_fields_set contains the field
-        if self.document_id is None and "document_id" in self.model_fields_set:
-            _dict['document_id'] = None
-
         return _dict
 
     @classmethod
@@ -91,7 +85,6 @@ class RetainResponse(BaseModel):
         _obj = cls.model_validate({
             "success": obj.get("success"),
             "bank_id": obj.get("bank_id"),
-            "document_id": obj.get("document_id"),
             "items_count": obj.get("items_count"),
             "async": obj.get("async")
         })

hindsight-clients/python/hindsight_client_api/test/test_chunk_response.py

@@ -0,0 +1,62 @@
+# coding: utf-8
+
+"""
+    Hindsight HTTP API
+
+    HTTP API for Hindsight
+
+    The version of the OpenAPI document: 1.0.0
+    Generated by OpenAPI Generator (https://openapi-generator.tech)
+
+    Do not edit the class manually.
+"""  # noqa: E501
+
+
+import unittest
+
+from hindsight_client_api.models.chunk_response import ChunkResponse
+
+class TestChunkResponse(unittest.TestCase):
+    """ChunkResponse unit test stubs"""
+
+    def setUp(self):
+        pass
+
+    def tearDown(self):
+        pass
+
+    def make_instance(self, include_optional) -> ChunkResponse:
+        """Test ChunkResponse
+            include_optional is a boolean, when False only required
+            params are included, when True both required and
+            optional params are included """
+        # uncomment below to create an instance of `ChunkResponse`
+        """
+        model = ChunkResponse()
+        if include_optional:
+            return ChunkResponse(
+                chunk_id = '',
+                document_id = '',
+                bank_id = '',
+                chunk_index = 56,
+                chunk_text = '',
+                created_at = ''
+            )
+        else:
+            return ChunkResponse(
+                chunk_id = '',
+                document_id = '',
+                bank_id = '',
+                chunk_index = 56,
+                chunk_text = '',
+                created_at = '',
+        )
+        """
+
+    def testChunkResponse(self):
+        """Test ChunkResponse"""
+        # inst_req_only = self.make_instance(include_optional=False)
+        # inst_req_and_optional = self.make_instance(include_optional=True)
+
+if __name__ == '__main__':
+    unittest.main()

hindsight-clients/python/hindsight_client_api/test/test_default_api.py

@@ -75,6 +75,13 @@ class TestDefaultApi(unittest.IsolatedAsyncioTestCase):
         """
         pass
 
+    async def test_get_chunk(self) -> None:
+        """Test case for get_chunk
+
+        Get chunk details
+        """
+        pass
+
     async def test_get_document(self) -> None:
         """Test case for get_document
 

hindsight-clients/python/hindsight_client_api/test/test_memory_item.py

@@ -40,7 +40,8 @@ class TestMemoryItem(unittest.TestCase):
                 context = '',
                 metadata = {
                     'key' : ''
-                    }
+                    },
+                document_id = ''
             )
         else:
             return MemoryItem(

hindsight-clients/python/hindsight_client_api/test/test_reflect_include_options.py

@@ -35,9 +35,7 @@ class TestReflectIncludeOptions(unittest.TestCase):
         model = ReflectIncludeOptions()
         if include_optional:
             return ReflectIncludeOptions(
-                facts = hindsight_client_api.models.facts_include_options.FactsIncludeOptions(),
-                entities = hindsight_client_api.models.entity_include_options.EntityIncludeOptions(
-                    max_tokens = 56, )
+                facts = hindsight_client_api.models.facts_include_options.FactsIncludeOptions()
             )
         else:
             return ReflectIncludeOptions(

hindsight-clients/python/hindsight_client_api/test/test_reflect_request.py

@@ -42,9 +42,7 @@ class TestReflectRequest(unittest.TestCase):
                     {key=source, match_unset=true, value=slack}
                     ],
                 include = hindsight_client_api.models.reflect_include_options.ReflectIncludeOptions(
-                    facts = hindsight_client_api.models.facts_include_options.FactsIncludeOptions(), 
-                    entities = hindsight_client_api.models.entity_include_options.EntityIncludeOptions(
-                        max_tokens = 56, ), )
+                    facts = hindsight_client_api.models.facts_include_options.FactsIncludeOptions(), )
             )
         else:
             return ReflectRequest(

hindsight-clients/python/hindsight_client_api/test/test_retain_request.py

@@ -36,15 +36,14 @@ class TestRetainRequest(unittest.TestCase):
         if include_optional:
             return RetainRequest(
                 items = [
-                    {content=Alice mentioned she's working on a new ML model, context=team meeting, metadata={channel=engineering, source=slack}, timestamp=2024-01-15T10:30:00Z}
+                    {content=Alice mentioned she's working on a new ML model, context=team meeting, document_id=meeting_notes_2024_01_15, metadata={channel=engineering, source=slack}, timestamp=2024-01-15T10:30:00Z}
                     ],
-                document_id = '',
                 var_async = True
             )
         else:
             return RetainRequest(
                 items = [
-                    {content=Alice mentioned she's working on a new ML model, context=team meeting, metadata={channel=engineering, source=slack}, timestamp=2024-01-15T10:30:00Z}
+                    {content=Alice mentioned she's working on a new ML model, context=team meeting, document_id=meeting_notes_2024_01_15, metadata={channel=engineering, source=slack}, timestamp=2024-01-15T10:30:00Z}
                     ],
         )
         """

hindsight-clients/python/hindsight_client_api/test/test_retain_response.py

@@ -37,7 +37,6 @@ class TestRetainResponse(unittest.TestCase):
             return RetainResponse(
                 success = True,
                 bank_id = '',
-                document_id = '',
                 items_count = 56,
                 var_async = True
             )

hindsight-clients/rust/Cargo.lock

@@ -1680,9 +1680,9 @@ checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be"
 
 [[package]]
 name = "uuid"
-version = "1.18.1"
+version = "1.19.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2f87b8aa10b915a06587d0dec516c282ff295b475d94abf425d62b57710070a2"
+checksum = "e2e054861b4bd027cd373e18e8d8d8e6548085000e41290d95ce0c373a654b4a"
 
 [[package]]
 name = "vcpkg"

hindsight-clients/rust/target/release/.fingerprint/hindsight-client-26b0bc308cced3ae/lib-hindsight_client

@@ -1 +0,0 @@
-49b8126e9760ef6c

hindsight-clients/rust/target/release/.fingerprint/hindsight-client-45e816fa8febabac/run-build-script-build-script-build

@@ -1 +0,0 @@
-15024a40d442b93a

hindsight-clients/rust/target/release/.fingerprint/hindsight-client-45e816fa8febabac/run-build-script-build-script-build.json

@@ -1 +0,0 @@
-{"rustc":16243257175721966122,"features":"","declared_features":"","target":0,"profile":0,"path":0,"deps":[[350039288653093011,"build_script_build",false,7490509127350218047]],"local":[{"RerunIfChanged":{"output":"release/build/hindsight-client-45e816fa8febabac/output","paths":["/Users/nicoloboschi/dev/memory-poc/openapi.json"]}}],"rustflags":[],"config":0,"compile_kind":0}

hindsight-clients/rust/target/release/.fingerprint/hindsight-client-58618863206160a4/build-script-build-script-build

@@ -0,0 +1 @@
+92e66ba8891fb9ec

hindsight-clients/rust/target/release/.fingerprint/hindsight-client-58618863206160a4/build-script-build-script-build.json

@@ -1 +1 @@
-{"rustc":16243257175721966122,"features":"[]","declared_features":"[]","target":5408242616063297496,"profile":1369601567987815722,"path":13767053534773805487,"deps":[[7988640081342112296,"syn",false,1809862803220272063],[9423015880379144908,"prettyplease",false,5158570001680563136],[9738901266855342370,"progenitor",false,5156049257603408733],[12832915883349295919,"serde_json",false,7203318985267246464],[16847286912798951732,"openapiv3",false,15609397813544834071]],"local":[{"CheckDepInfo":{"dep_info":"release/.fingerprint/hindsight-client-b1a7aa8cd48fa221/dep-build-script-build-script-build","checksum":false}}],"rustflags":[],"config":2069994364910194474,"compile_kind":0}
+{"rustc":16243257175721966122,"features":"[]","declared_features":"[]","target":5408242616063297496,"profile":1369601567987815722,"path":13767053534773805487,"deps":[[7988640081342112296,"syn",false,1809862803220272063],[9423015880379144908,"prettyplease",false,5158570001680563136],[9738901266855342370,"progenitor",false,6987356646142300397],[12832915883349295919,"serde_json",false,7203318985267246464],[16847286912798951732,"openapiv3",false,15609397813544834071]],"local":[{"CheckDepInfo":{"dep_info":"release/.fingerprint/hindsight-client-58618863206160a4/dep-build-script-build-script-build","checksum":false}}],"rustflags":[],"config":2069994364910194474,"compile_kind":0}

hindsight-clients/rust/target/release/.fingerprint/hindsight-client-793f05aca97996e5/run-build-script-build-script-build

@@ -0,0 +1 @@
+17b2b80455e1755a

hindsight-clients/rust/target/release/.fingerprint/hindsight-client-793f05aca97996e5/run-build-script-build-script-build.json

@@ -0,0 +1 @@
+{"rustc":16243257175721966122,"features":"","declared_features":"","target":0,"profile":0,"path":0,"deps":[[350039288653093011,"build_script_build",false,17057699739739088530]],"local":[{"RerunIfChanged":{"output":"release/build/hindsight-client-793f05aca97996e5/output","paths":["/Users/nicoloboschi/dev/memory-poc/openapi.json"]}}],"rustflags":[],"config":0,"compile_kind":0}

hindsight-clients/rust/target/release/.fingerprint/hindsight-client-b1a7aa8cd48fa221/build-script-build-script-build

@@ -1 +0,0 @@
-3fed876a5da2f367

hindsight-clients/rust/target/release/.fingerprint/hindsight-client-b2e1c0bce7e404b8/lib-hindsight_client

@@ -0,0 +1 @@
+b85c02be490b8ea0

hindsight-clients/rust/target/release/.fingerprint/hindsight-client-b2e1c0bce7e404b8/lib-hindsight_client.json

@@ -1 +1 @@
-{"rustc":16243257175721966122,"features":"[]","declared_features":"[]","target":6828276606420267087,"profile":2040997289075261528,"path":10763286916239946207,"deps":[[350039288653093011,"build_script_build",false,4231486804270383637],[503842845364652431,"chrono",false,11144948474405894614],[1046219396048762255,"progenitor_client",false,13036815077551201483],[2620434475832828286,"http",false,9979032511492736550],[5404511084185685755,"url",false,17079058419592311478],[5802782114936492624,"reqwest",false,16326245460864990945],[7720834239451334583,"tokio",false,814226396053303386],[8008191657135824715,"thiserror",false,1675330904495433212],[12832915883349295919,"serde_json",false,11663418101978700483],[13548984313718623784,"serde",false,17261882564294632758]],"local":[{"CheckDepInfo":{"dep_info":"release/.fingerprint/hindsight-client-26b0bc308cced3ae/dep-lib-hindsight_client","checksum":false}}],"rustflags":[],"config":2069994364910194474,"compile_kind":0}
+{"rustc":16243257175721966122,"features":"[]","declared_features":"[]","target":6828276606420267087,"profile":2040997289075261528,"path":10763286916239946207,"deps":[[350039288653093011,"build_script_build",false,6518363790956343831],[503842845364652431,"chrono",false,11144948474405894614],[1046219396048762255,"progenitor_client",false,13036815077551201483],[2620434475832828286,"http",false,9979032511492736550],[5404511084185685755,"url",false,17079058419592311478],[5802782114936492624,"reqwest",false,16326245460864990945],[7720834239451334583,"tokio",false,814226396053303386],[8008191657135824715,"thiserror",false,1675330904495433212],[12832915883349295919,"serde_json",false,11663418101978700483],[13548984313718623784,"serde",false,17261882564294632758]],"local":[{"CheckDepInfo":{"dep_info":"release/.fingerprint/hindsight-client-b2e1c0bce7e404b8/dep-lib-hindsight_client","checksum":false}}],"rustflags":[],"config":2069994364910194474,"compile_kind":0}

hindsight-clients/rust/target/release/.fingerprint/progenitor-09c624ef88805fe3/lib-progenitor

@@ -0,0 +1 @@
+edd0465cd913f860

hindsight-clients/rust/target/release/.fingerprint/progenitor-09c624ef88805fe3/lib-progenitor.json

@@ -1 +1 @@
-{"rustc":16243257175721966122,"features":"[\"default\", \"macro\"]","declared_features":"[\"default\", \"macro\"]","target":15608857702111660434,"profile":1369601567987815722,"path":14901966660390340651,"deps":[[1046219396048762255,"progenitor_client",false,13306704853339479171],[3039535961030183584,"progenitor_impl",false,636270600742616262],[17067139923740644357,"progenitor_macro",false,179726542608778300]],"local":[{"CheckDepInfo":{"dep_info":"release/.fingerprint/progenitor-4bdd0c7fc86c6021/dep-lib-progenitor","checksum":false}}],"rustflags":[],"config":2069994364910194474,"compile_kind":0}
+{"rustc":16243257175721966122,"features":"[\"default\", \"macro\"]","declared_features":"[\"default\", \"macro\"]","target":15608857702111660434,"profile":1369601567987815722,"path":14901966660390340651,"deps":[[1046219396048762255,"progenitor_client",false,13306704853339479171],[3039535961030183584,"progenitor_impl",false,3223650479284347882],[17067139923740644357,"progenitor_macro",false,7805686184841859657]],"local":[{"CheckDepInfo":{"dep_info":"release/.fingerprint/progenitor-09c624ef88805fe3/dep-lib-progenitor","checksum":false}}],"rustflags":[],"config":2069994364910194474,"compile_kind":0}

hindsight-clients/rust/target/release/.fingerprint/progenitor-4bdd0c7fc86c6021/lib-progenitor

@@ -1 +0,0 @@
-5d23de5b7bf78d47

hindsight-clients/rust/target/release/.fingerprint/progenitor-impl-06826d04079242ad/lib-progenitor_impl

@@ -0,0 +1 @@
+eae3c20108b5bc2c

hindsight-clients/rust/target/release/.fingerprint/progenitor-impl-06826d04079242ad/lib-progenitor_impl.json

@@ -0,0 +1 @@
+{"rustc":16243257175721966122,"features":"[]","declared_features":"[]","target":14575771264175982451,"profile":1369601567987815722,"path":4143174102873047870,"deps":[[1548027836057496652,"unicode_ident",false,12497779118727399146],[2620434475832828286,"http",false,1650492083869495292],[3056178850035811329,"regex",false,16159500298532302859],[4336745513838352383,"thiserror",false,16080867181668872609],[6240934600354534560,"indexmap",false,12677675679014318805],[6913375703034175521,"schemars",false,374665543383128733],[7988640081342112296,"syn",false,1809862803220272063],[9869581871423326951,"quote",false,16408282429201193700],[11401754758611382041,"typify",false,17824555848214860990],[12832915883349295919,"serde_json",false,7203318985267246464],[13077543566650298139,"heck",false,13265169220388563925],[13548984313718623784,"serde",false,18392579626400240657],[14285738760999836560,"proc_macro2",false,14273862529107951632],[16847286912798951732,"openapiv3",false,15609397813544834071]],"local":[{"CheckDepInfo":{"dep_info":"release/.fingerprint/progenitor-impl-06826d04079242ad/dep-lib-progenitor_impl","checksum":false}}],"rustflags":[],"config":2069994364910194474,"compile_kind":0}

hindsight-clients/rust/target/release/.fingerprint/progenitor-impl-4be51208161b9376/lib-progenitor_impl

@@ -1 +0,0 @@
-c6bc4982bd7cd408

hindsight-clients/rust/target/release/.fingerprint/progenitor-impl-4be51208161b9376/lib-progenitor_impl.json

@@ -1 +0,0 @@
-{"rustc":16243257175721966122,"features":"[]","declared_features":"[]","target":14575771264175982451,"profile":1369601567987815722,"path":4143174102873047870,"deps":[[1548027836057496652,"unicode_ident",false,12497779118727399146],[2620434475832828286,"http",false,1650492083869495292],[3056178850035811329,"regex",false,16159500298532302859],[4336745513838352383,"thiserror",false,16080867181668872609],[6240934600354534560,"indexmap",false,12677675679014318805],[6913375703034175521,"schemars",false,15965657796090931825],[7988640081342112296,"syn",false,1809862803220272063],[9869581871423326951,"quote",false,16408282429201193700],[11401754758611382041,"typify",false,1878054387901356869],[12832915883349295919,"serde_json",false,7203318985267246464],[13077543566650298139,"heck",false,13265169220388563925],[13548984313718623784,"serde",false,18392579626400240657],[14285738760999836560,"proc_macro2",false,14273862529107951632],[16847286912798951732,"openapiv3",false,15609397813544834071]],"local":[{"CheckDepInfo":{"dep_info":"release/.fingerprint/progenitor-impl-4be51208161b9376/dep-lib-progenitor_impl","checksum":false}}],"rustflags":[],"config":2069994364910194474,"compile_kind":0}

hindsight-clients/rust/target/release/.fingerprint/progenitor-macro-2aa1b4026eb3ec1c/lib-progenitor_macro

@@ -0,0 +1 @@
+4952ac933a5e536c

hindsight-clients/rust/target/release/.fingerprint/progenitor-macro-2aa1b4026eb3ec1c/lib-progenitor_macro.json

@@ -0,0 +1 @@
+{"rustc":16243257175721966122,"features":"[]","declared_features":"[]","target":12433518205030116463,"profile":1369601567987815722,"path":3059927606038109866,"deps":[[3039535961030183584,"progenitor_impl",false,3223650479284347882],[6913375703034175521,"schemars",false,374665543383128733],[7988640081342112296,"syn",false,1809862803220272063],[9614479274285663593,"serde_yaml",false,166363834370521707],[9869581871423326951,"quote",false,16408282429201193700],[12832915883349295919,"serde_json",false,7203318985267246464],[13548984313718623784,"serde",false,18392579626400240657],[14285738760999836560,"proc_macro2",false,14273862529107951632],[16847286912798951732,"openapiv3",false,15609397813544834071],[18142522549889578203,"serde_tokenstream",false,16295073279524653793]],"local":[{"CheckDepInfo":{"dep_info":"release/.fingerprint/progenitor-macro-2aa1b4026eb3ec1c/dep-lib-progenitor_macro","checksum":false}}],"rustflags":[],"config":2069994364910194474,"compile_kind":0}

hindsight-clients/rust/target/release/.fingerprint/progenitor-macro-be18053f5df03ea5/lib-progenitor_macro

@@ -1 +0,0 @@
-3cc4e79856847e02

hindsight-clients/rust/target/release/.fingerprint/progenitor-macro-be18053f5df03ea5/lib-progenitor_macro.json

@@ -1 +0,0 @@
-{"rustc":16243257175721966122,"features":"[]","declared_features":"[]","target":12433518205030116463,"profile":1369601567987815722,"path":3059927606038109866,"deps":[[3039535961030183584,"progenitor_impl",false,636270600742616262],[6913375703034175521,"schemars",false,15965657796090931825],[7988640081342112296,"syn",false,1809862803220272063],[9614479274285663593,"serde_yaml",false,166363834370521707],[9869581871423326951,"quote",false,16408282429201193700],[12832915883349295919,"serde_json",false,7203318985267246464],[13548984313718623784,"serde",false,18392579626400240657],[14285738760999836560,"proc_macro2",false,14273862529107951632],[16847286912798951732,"openapiv3",false,15609397813544834071],[18142522549889578203,"serde_tokenstream",false,16295073279524653793]],"local":[{"CheckDepInfo":{"dep_info":"release/.fingerprint/progenitor-macro-be18053f5df03ea5/dep-lib-progenitor_macro","checksum":false}}],"rustflags":[],"config":2069994364910194474,"compile_kind":0}

hindsight-clients/rust/target/release/.fingerprint/schemars-1d824015212552b8/lib-schemars

@@ -1 +0,0 @@
-712a72052d6991dd

hindsight-clients/rust/target/release/.fingerprint/schemars-2baa628a98a02c17/lib-schemars

@@ -0,0 +1 @@
+9d1aee9e53143305

hindsight-clients/rust/target/release/.fingerprint/schemars-2baa628a98a02c17/lib-schemars.json

@@ -1 +1 @@
-{"rustc":16243257175721966122,"features":"[\"chrono\", \"default\", \"derive\", \"schemars_derive\", \"uuid1\"]","declared_features":"[\"arrayvec\", \"arrayvec05\", \"arrayvec07\", \"bigdecimal\", \"bigdecimal03\", \"bigdecimal04\", \"bytes\", \"chrono\", \"default\", \"derive\", \"derive_json_schema\", \"either\", \"enumset\", \"impl_json_schema\", \"indexmap\", \"indexmap1\", \"indexmap2\", \"preserve_order\", \"raw_value\", \"rust_decimal\", \"schemars_derive\", \"semver\", \"smallvec\", \"smol_str\", \"ui_test\", \"url\", \"uuid\", \"uuid08\", \"uuid1\"]","target":11155677158530064643,"profile":1369601567987815722,"path":5926874010205357219,"deps":[[503842845364652431,"chrono",false,2900766844280536775],[6913375703034175521,"build_script_build",false,7206007404470304092],[6982418085031928086,"dyn_clone",false,6834683740842263458],[12832915883349295919,"serde_json",false,7203318985267246464],[13548984313718623784,"serde",false,18392579626400240657],[15267671913832104935,"uuid1",false,12241668191619465921],[16071897500792579091,"schemars_derive",false,1760453691084183604]],"local":[{"CheckDepInfo":{"dep_info":"release/.fingerprint/schemars-1d824015212552b8/dep-lib-schemars","checksum":false}}],"rustflags":[],"config":2069994364910194474,"compile_kind":0}
+{"rustc":16243257175721966122,"features":"[\"chrono\", \"default\", \"derive\", \"schemars_derive\", \"uuid1\"]","declared_features":"[\"arrayvec\", \"arrayvec05\", \"arrayvec07\", \"bigdecimal\", \"bigdecimal03\", \"bigdecimal04\", \"bytes\", \"chrono\", \"default\", \"derive\", \"derive_json_schema\", \"either\", \"enumset\", \"impl_json_schema\", \"indexmap\", \"indexmap1\", \"indexmap2\", \"preserve_order\", \"raw_value\", \"rust_decimal\", \"schemars_derive\", \"semver\", \"smallvec\", \"smol_str\", \"ui_test\", \"url\", \"uuid\", \"uuid08\", \"uuid1\"]","target":11155677158530064643,"profile":1369601567987815722,"path":5926874010205357219,"deps":[[503842845364652431,"chrono",false,2900766844280536775],[1420800981318104879,"uuid1",false,18252328355564986311],[6913375703034175521,"build_script_build",false,7206007404470304092],[6982418085031928086,"dyn_clone",false,6834683740842263458],[12832915883349295919,"serde_json",false,7203318985267246464],[13548984313718623784,"serde",false,18392579626400240657],[16071897500792579091,"schemars_derive",false,1760453691084183604]],"local":[{"CheckDepInfo":{"dep_info":"release/.fingerprint/schemars-2baa628a98a02c17/dep-lib-schemars","checksum":false}}],"rustflags":[],"config":2069994364910194474,"compile_kind":0}

hindsight-clients/rust/target/release/.fingerprint/typify-3c13a635718c0fbd/lib-typify

@@ -0,0 +1 @@
+be54f5fb1a8b5df7

hindsight-clients/rust/target/release/.fingerprint/typify-3c13a635718c0fbd/lib-typify.json

@@ -1 +1 @@
-{"rustc":16243257175721966122,"features":"[\"default\", \"macro\", \"typify-macro\"]","declared_features":"[\"default\", \"macro\", \"typify-macro\"]","target":14975903297306792855,"profile":1369601567987815722,"path":17345980241392545380,"deps":[[12189557469245296852,"typify_impl",false,11312802444564477127],[12514255388840618205,"typify_macro",false,8783384910811377035]],"local":[{"CheckDepInfo":{"dep_info":"release/.fingerprint/typify-65e7243859581cdf/dep-lib-typify","checksum":false}}],"rustflags":[],"config":2069994364910194474,"compile_kind":0}
+{"rustc":16243257175721966122,"features":"[\"default\", \"macro\", \"typify-macro\"]","declared_features":"[\"default\", \"macro\", \"typify-macro\"]","target":14975903297306792855,"profile":1369601567987815722,"path":17345980241392545380,"deps":[[12189557469245296852,"typify_impl",false,8737027169081722424],[12514255388840618205,"typify_macro",false,17335598172921498906]],"local":[{"CheckDepInfo":{"dep_info":"release/.fingerprint/typify-3c13a635718c0fbd/dep-lib-typify","checksum":false}}],"rustflags":[],"config":2069994364910194474,"compile_kind":0}

hindsight-clients/rust/target/release/.fingerprint/typify-65e7243859581cdf/lib-typify

@@ -1 +0,0 @@
-455bb9f38330101a

hindsight-clients/rust/target/release/.fingerprint/typify-impl-c6bad80e900da2bc/lib-typify_impl

@@ -0,0 +1 @@
+38860d66eb274079

hindsight-clients/rust/target/release/.fingerprint/typify-impl-c6bad80e900da2bc/lib-typify_impl.json

@@ -0,0 +1 @@
+{"rustc":16243257175721966122,"features":"[]","declared_features":"[]","target":2642133076118073701,"profile":1369601567987815722,"path":17859204502057325984,"deps":[[57391913602052214,"regress",false,5853959305665143200],[1548027836057496652,"unicode_ident",false,12497779118727399146],[4336745513838352383,"thiserror",false,16080867181668872609],[6913375703034175521,"schemars",false,374665543383128733],[7988640081342112296,"syn",false,1809862803220272063],[9869581871423326951,"quote",false,16408282429201193700],[12832915883349295919,"serde_json",false,7203318985267246464],[13066042571740262168,"log",false,5499292635580693977],[13077543566650298139,"heck",false,13265169220388563925],[13548984313718623784,"serde",false,18392579626400240657],[14285738760999836560,"proc_macro2",false,14273862529107951632],[18361894353739432590,"semver",false,6556068567130520783]],"local":[{"CheckDepInfo":{"dep_info":"release/.fingerprint/typify-impl-c6bad80e900da2bc/dep-lib-typify_impl","checksum":false}}],"rustflags":[],"config":2069994364910194474,"compile_kind":0}

hindsight-clients/rust/target/release/.fingerprint/typify-impl-eae5a0de0558fb19/lib-typify_impl

@@ -1 +0,0 @@
-c7b46fb2e225ff9c

hindsight-clients/rust/target/release/.fingerprint/typify-impl-eae5a0de0558fb19/lib-typify_impl.json

@@ -1 +0,0 @@
-{"rustc":16243257175721966122,"features":"[]","declared_features":"[]","target":2642133076118073701,"profile":1369601567987815722,"path":17859204502057325984,"deps":[[57391913602052214,"regress",false,5853959305665143200],[1548027836057496652,"unicode_ident",false,12497779118727399146],[4336745513838352383,"thiserror",false,16080867181668872609],[6913375703034175521,"schemars",false,15965657796090931825],[7988640081342112296,"syn",false,1809862803220272063],[9869581871423326951,"quote",false,16408282429201193700],[12832915883349295919,"serde_json",false,7203318985267246464],[13066042571740262168,"log",false,5499292635580693977],[13077543566650298139,"heck",false,13265169220388563925],[13548984313718623784,"serde",false,18392579626400240657],[14285738760999836560,"proc_macro2",false,14273862529107951632],[18361894353739432590,"semver",false,6556068567130520783]],"local":[{"CheckDepInfo":{"dep_info":"release/.fingerprint/typify-impl-eae5a0de0558fb19/dep-lib-typify_impl","checksum":false}}],"rustflags":[],"config":2069994364910194474,"compile_kind":0}

hindsight-clients/rust/target/release/.fingerprint/typify-macro-13de632972e7e35b/lib-typify_macro

@@ -0,0 +1 @@
+1a8de5b9b06a94f0

hindsight-clients/rust/target/release/.fingerprint/typify-macro-13de632972e7e35b/lib-typify_macro.json

@@ -0,0 +1 @@
+{"rustc":16243257175721966122,"features":"[]","declared_features":"[]","target":4711089848534984104,"profile":1369601567987815722,"path":16211961820950626129,"deps":[[6913375703034175521,"schemars",false,374665543383128733],[7988640081342112296,"syn",false,1809862803220272063],[9869581871423326951,"quote",false,16408282429201193700],[12189557469245296852,"typify_impl",false,8737027169081722424],[12832915883349295919,"serde_json",false,7203318985267246464],[13548984313718623784,"serde",false,18392579626400240657],[14285738760999836560,"proc_macro2",false,14273862529107951632],[18142522549889578203,"serde_tokenstream",false,16295073279524653793],[18361894353739432590,"semver",false,6556068567130520783]],"local":[{"CheckDepInfo":{"dep_info":"release/.fingerprint/typify-macro-13de632972e7e35b/dep-lib-typify_macro","checksum":false}}],"rustflags":[],"config":2069994364910194474,"compile_kind":0}

hindsight-clients/rust/target/release/.fingerprint/typify-macro-fd19a21f23250962/lib-typify_macro

@@ -1 +0,0 @@
-8b2d703e0adae479

hindsight-clients/rust/target/release/.fingerprint/typify-macro-fd19a21f23250962/lib-typify_macro.json

@@ -1 +0,0 @@
-{"rustc":16243257175721966122,"features":"[]","declared_features":"[]","target":4711089848534984104,"profile":1369601567987815722,"path":16211961820950626129,"deps":[[6913375703034175521,"schemars",false,15965657796090931825],[7988640081342112296,"syn",false,1809862803220272063],[9869581871423326951,"quote",false,16408282429201193700],[12189557469245296852,"typify_impl",false,11312802444564477127],[12832915883349295919,"serde_json",false,7203318985267246464],[13548984313718623784,"serde",false,18392579626400240657],[14285738760999836560,"proc_macro2",false,14273862529107951632],[18142522549889578203,"serde_tokenstream",false,16295073279524653793],[18361894353739432590,"semver",false,6556068567130520783]],"local":[{"CheckDepInfo":{"dep_info":"release/.fingerprint/typify-macro-fd19a21f23250962/dep-lib-typify_macro","checksum":false}}],"rustflags":[],"config":2069994364910194474,"compile_kind":0}

hindsight-clients/rust/target/release/.fingerprint/uuid-892ce5c89892b489/lib-uuid

@@ -0,0 +1 @@
+c79f3d35ef4b4dfd

hindsight-clients/rust/target/release/.fingerprint/uuid-892ce5c89892b489/lib-uuid.json

@@ -0,0 +1 @@
+{"rustc":16243257175721966122,"features":"[]","declared_features":"[\"arbitrary\", \"atomic\", \"borsh\", \"bytemuck\", \"default\", \"fast-rng\", \"js\", \"macro-diagnostics\", \"md5\", \"rng\", \"rng-getrandom\", \"rng-rand\", \"serde\", \"sha1\", \"slog\", \"std\", \"uuid-rng-internal-lib\", \"v1\", \"v3\", \"v4\", \"v5\", \"v6\", \"v7\", \"v8\", \"zerocopy\"]","target":2422778461497348360,"profile":10765049016586272810,"path":2526824666322011227,"deps":[],"local":[{"CheckDepInfo":{"dep_info":"release/.fingerprint/uuid-892ce5c89892b489/dep-lib-uuid","checksum":false}}],"rustflags":[],"config":2069994364910194474,"compile_kind":0}

hindsight-clients/rust/target/release/.fingerprint/uuid-b261ab99bb391ac4/lib-uuid

@@ -1 +0,0 @@
-c142ad735c24e3a9