add llms.txt

hindsight-api/hindsight_api/engine/search/tracer.py

@@ -368,7 +368,7 @@ class SearchTracer:
 
             # Extract score components (only include non-None values)
             score_components = {}
-            for key in ["semantic_similarity", "bm25_score", "rrf_score", "recency_normalized", "frequency_normalized"]:
+            for key in ["semantic_similarity", "bm25_score", "rrf_score", "recency_normalized", "frequency_normalized", "cross_encoder_score", "cross_encoder_score_normalized"]:
                 if key in result and result[key] is not None:
                     score_components[key] = result[key]
 

hindsight-docs/docs/cookbook/per-user-memory.md

@@ -47,14 +47,29 @@ def on_user_signup(user_id: str):
     )
 ```
 
-### 2. Save Conversations After Each Session
+### 2. Manage Conversation Sessions
+
+Use `document_id` to group messages belonging to the same conversation. When you retain with the same `document_id`, Hindsight replaces the previous version (upsert behavior), keeping the memory up-to-date as the conversation evolves.
 
 ```python
-async def save_conversation(user_id: str, messages: list):
-    await client.retain(
-        bank_id=f"user-{user_id}",
-        content=messages  # [{"role": "user", "content": "..."}, ...]
-    )
+import uuid
+
+class ConversationSession:
+    def __init__(self, user_id: str):
+        self.user_id = user_id
+        self.session_id = str(uuid.uuid4())  # Unique ID for this conversation
+        self.messages = []
+
+    def add_message(self, role: str, content: str):
+        self.messages.append({"role": role, "content": content})
+
+    async def save(self, client: HindsightClient):
+        """Save the entire conversation. Replaces previous version if session_id exists."""
+        await client.retain(
+            bank_id=f"user-{self.user_id}",
+            content=self.messages,
+            document_id=self.session_id  # Same ID = upsert (replace old version)
+        )
 ```
 
 ### 3. Recall Context Before Responding
@@ -71,38 +86,70 @@ async def get_context(user_id: str, query: str):
 ### 4. Complete Agent Loop
 
 ```python
-async def handle_message(user_id: str, user_message: str):
-    # 1. Recall relevant context
+async def handle_message(session: ConversationSession, user_message: str):
+    # 1. Add user message to session
+    session.add_message("user", user_message)
+
+    # 2. Recall relevant context from past conversations
     context = await client.recall(
-        bank_id=f"user-{user_id}",
+        bank_id=f"user-{session.user_id}",
         query=user_message
     )
 
-    # 2. Build prompt with memory
+    # 3. Build prompt with memory
     prompt = f"""You are a helpful assistant with memory of past conversations.
 
 ## What you remember about this user
 {format_results(context.results)}
 
-## Current message
-{user_message}
+## Current conversation
+{format_messages(session.messages)}
 """
 
-    # 3. Generate response
+    # 4. Generate response
     response = await llm.complete(prompt)
 
-    # 4. Save the conversation
-    await client.retain(
-        bank_id=f"user-{user_id}",
-        content=[
-            {"role": "user", "content": user_message},
-            {"role": "assistant", "content": response}
-        ]
-    )
+    # 5. Add assistant response to session
+    session.add_message("assistant", response)
+
+    # 6. Save the updated conversation (upserts based on session_id)
+    await session.save(client)
 
     return response
 ```
 
+### 5. Starting a New Conversation
+
+```python
+# Each new conversation gets a new session with a unique ID
+session = ConversationSession(user_id="alice")
+
+# Multiple exchanges in the same conversation
+await handle_message(session, "Hi! I'm working on a Python project")
+await handle_message(session, "Can you help me with async/await?")
+
+# Start a new conversation later (new session_id)
+new_session = ConversationSession(user_id="alice")
+await handle_message(new_session, "Different topic today...")
+```
+
+## How Document ID Works
+
+The `document_id` parameter is key to managing evolving conversations:
+
+| Scenario | Behavior |
+|----------|----------|
+| First retain with `document_id="session_123"` | Creates new document |
+| Retain again with same `document_id="session_123"` | **Replaces** previous version (upsert) |
+| Retain with different `document_id="session_456"` | Creates separate document |
+| Retain without `document_id` | Creates new document each time |
+
+This upsert behavior means:
+- You always retain the **full conversation** state
+- Facts are re-extracted from the complete conversation
+- No duplicate or stale facts from old versions
+- Memory stays consistent as conversations evolve
+
 ## What Gets Remembered
 
 Hindsight automatically extracts and connects:

hindsight-docs/package.json

@@ -4,8 +4,9 @@
   "private": true,
   "scripts": {
     "docusaurus": "docusaurus",
-    "start": "docusaurus start",
-    "build": "docusaurus build",
+    "generate-llms": "node scripts/generate-llms-full.js",
+    "start": "npm run generate-llms && docusaurus start",
+    "build": "npm run generate-llms && docusaurus build",
     "swizzle": "docusaurus swizzle",
     "deploy": "docusaurus deploy",
     "clear": "docusaurus clear",

hindsight-docs/scripts/generate-llms-full.js

@@ -0,0 +1,150 @@
+#!/usr/bin/env node
+
+/**
+ * Generates llms-full.txt by concatenating all documentation markdown files.
+ * This file is used by LLMs to understand the full documentation.
+ *
+ * Usage: node scripts/generate-llms-full.js
+ *
+ * Output: static/llms-full.txt (served at /llms-full.txt)
+ */
+
+const fs = require('fs');
+const path = require('path');
+
+const DOCS_DIR = path.join(__dirname, '..', 'docs');
+const OUTPUT_FILE = path.join(__dirname, '..', 'static', 'llms-full.txt');
+
+// Order matters - more important docs first
+const DOC_ORDER = [
+  'developer/index.md',
+  'developer/api/quickstart.md',
+  'developer/api/main-methods.md',
+  'developer/retain.md',
+  'developer/retrieval.md',
+  'developer/reflect.md',
+  'developer/api/retain.md',
+  'developer/api/recall.md',
+  'developer/api/reflect.md',
+  'developer/api/memory-banks.md',
+  'developer/api/entities.md',
+  'developer/api/documents.md',
+  'developer/api/operations.md',
+  'developer/installation.md',
+  'developer/configuration.md',
+  'developer/models.md',
+  'developer/rag-vs-hindsight.md',
+  'sdks/python.md',
+  'sdks/nodejs.md',
+  'sdks/cli.md',
+  'sdks/mcp.md',
+  'cookbook/index.md',
+  'cookbook/per-user-memory.md',
+  'cookbook/support-agent-with-shared-knowledge.md',
+];
+
+function getAllMarkdownFiles(dir, baseDir = dir) {
+  const files = [];
+  const entries = fs.readdirSync(dir, { withFileTypes: true });
+
+  for (const entry of entries) {
+    const fullPath = path.join(dir, entry.name);
+    if (entry.isDirectory()) {
+      files.push(...getAllMarkdownFiles(fullPath, baseDir));
+    } else if (entry.name.endsWith('.md') || entry.name.endsWith('.mdx')) {
+      const relativePath = path.relative(baseDir, fullPath);
+      files.push(relativePath);
+    }
+  }
+
+  return files;
+}
+
+function stripFrontmatter(content) {
+  // Remove YAML frontmatter (between --- markers)
+  const frontmatterRegex = /^---\n[\s\S]*?\n---\n/;
+  return content.replace(frontmatterRegex, '');
+}
+
+function cleanMarkdown(content) {
+  let cleaned = stripFrontmatter(content);
+
+  // Remove import statements
+  cleaned = cleaned.replace(/^import\s+.*$/gm, '');
+
+  // Remove empty lines at start
+  cleaned = cleaned.replace(/^\n+/, '');
+
+  return cleaned;
+}
+
+function generateLlmsFullTxt() {
+  console.log('Generating llms-full.txt...');
+
+  // Get all markdown files
+  const allFiles = getAllMarkdownFiles(DOCS_DIR);
+
+  // Create ordered list: prioritized files first, then remaining files
+  const orderedFiles = [];
+  const remainingFiles = new Set(allFiles);
+
+  // Add prioritized files in order
+  for (const file of DOC_ORDER) {
+    if (remainingFiles.has(file)) {
+      orderedFiles.push(file);
+      remainingFiles.delete(file);
+    }
+  }
+
+  // Add remaining files (sorted alphabetically)
+  const sortedRemaining = Array.from(remainingFiles).sort();
+  orderedFiles.push(...sortedRemaining);
+
+  // Build the output
+  const sections = [];
+
+  // Header
+  sections.push(`# Hindsight Documentation
+
+> Agent Memory that Works Like Human Memory
+
+This file contains the complete Hindsight documentation for LLM consumption.
+Generated: ${new Date().toISOString()}
+
+---
+`);
+
+  // Process each file
+  for (const file of orderedFiles) {
+    const filePath = path.join(DOCS_DIR, file);
+
+    if (!fs.existsSync(filePath)) {
+      console.warn(`  Warning: ${file} not found, skipping`);
+      continue;
+    }
+
+    const content = fs.readFileSync(filePath, 'utf-8');
+    const cleanedContent = cleanMarkdown(content);
+
+    if (cleanedContent.trim()) {
+      // Add file path as context
+      sections.push(`\n## File: ${file}\n`);
+      sections.push(cleanedContent);
+      sections.push('\n---\n');
+      console.log(`  Added: ${file}`);
+    }
+  }
+
+  // Write output
+  const output = sections.join('\n');
+  fs.writeFileSync(OUTPUT_FILE, output);
+
+  const stats = fs.statSync(OUTPUT_FILE);
+  const sizeKb = (stats.size / 1024).toFixed(1);
+
+  console.log(`\nGenerated: ${OUTPUT_FILE}`);
+  console.log(`Size: ${sizeKb} KB`);
+  console.log(`Files included: ${orderedFiles.length}`);
+}
+
+generateLlmsFullTxt();

llms.txt

@@ -0,0 +1,88 @@
+# Hindsight
+
+> Agent Memory that Works Like Human Memory
+
+Hindsight is an agent memory system that gives AI agents persistent, structured memory across sessions. It extracts facts, entities, and relationships from conversations and enables temporal reasoning, opinion formation, and multi-strategy retrieval.
+
+For complete documentation, see: https://vectorize-io.github.io/hindsight/llms-full.txt
+
+## Core Operations
+
+- **Retain**: Store memories (extracts facts, entities, relationships automatically)
+- **Recall**: Retrieve memories (semantic, keyword, graph, temporal search)
+- **Reflect**: Deep analysis to form opinions and insights
+
+## Quick Start
+
+```python
+from hindsight import HindsightClient
+
+client = HindsightClient(base_url="http://localhost:8888")
+
+# Store
+client.retain(bank_id="my-agent", content="Alice works at Google as a software engineer")
+
+# Query
+results = client.recall(bank_id="my-agent", query="What does Alice do?")
+
+# Reflect
+response = client.reflect(bank_id="my-agent", query="Tell me about Alice")
+```
+
+## Key Concepts
+
+### Memory Banks
+Each bank is an isolated memory store. One bank per user/agent. Banks contain facts, entities, documents, and their relationships.
+
+### Memory Types
+- World facts: General knowledge
+- Experience facts: Personal experiences
+- Opinion facts: Beliefs with confidence scores
+
+### Document ID for Evolving Conversations
+Use `document_id` to group messages in a conversation. Retaining with the same `document_id` replaces the previous version (upsert), keeping memory consistent as conversations evolve.
+
+```python
+client.retain(
+    bank_id="user-123",
+    content=messages,
+    document_id="session_abc"  # Same ID = replace old version
+)
+```
+
+## Documentation
+
+- Docs: https://vectorize-io.github.io/hindsight
+- GitHub: https://github.com/vectorize-io/hindsight
+- Python client: pip install hindsight-client
+- TypeScript client: npm install @vectorize-io/hindsight-client
+
+## API Reference
+
+Base URL: http://localhost:8888 (default)
+
+### POST /v1/default/banks/{bank_id}/retain
+Store memories in a bank.
+
+### POST /v1/default/banks/{bank_id}/recall
+Retrieve memories matching a query.
+
+### POST /v1/default/banks/{bank_id}/reflect
+Analyze memories and form opinions/insights.
+
+### GET /v1/default/banks/{bank_id}/profile
+Get bank profile (disposition, background).
+
+### PUT /v1/default/banks/{bank_id}/profile
+Update bank disposition and background.
+
+## Architecture Patterns
+
+### Per-User Memory
+One bank per user. Simplest pattern for chatbots and assistants.
+
+### Support Agent + Shared Knowledge
+User bank + shared docs bank. Client orchestrates queries to both banks and merges results.
+
+### With Curated Learnings
+User bank + shared docs + learnings bank. Promote verified solutions to shared learnings.