fix: improve openclaw test coverage (#396)
* fix: improve openclaw test coverage * test(openclaw): export stripMemoryTags/extractRecallQuery and add hook integration tests - Extract stripMemoryTags and extractRecallQuery as exported pure functions from index.ts so hooks share one implementation and tests cover the real code - Update before_agent_start to call extractRecallQuery; update agent_end to call stripMemoryTags instead of duplicating the regex inline - Rewrite index.test.ts to import the real functions (no more local duplicate) and add 11 tests for extractRecallQuery covering all envelope-stripping cases - Add tests/hooks.integration.test.ts: loads the plugin via mock MoltbotPluginAPI in HTTP mode, spies on client.recall/retain, and exercises all hook behaviours: excluded providers, short messages, memory injection format, tag stripping, transcript formatting, array content blocks, metadata, document_id derivation
This commit is contained in:
Nicolò Boschi
GitHub
parent
7eafba661e
commit
6c695eb9f8
9 changed files with 1240 additions and 135 deletions
101
.github/workflows/test.yml
vendored
101
.github/workflows/test.yml
vendored
|
|
@ -726,6 +726,107 @@ jobs:
|
|||
echo "=== API Server Logs ==="
|
||||
cat /tmp/api-server.log || echo "No API server log found"
|
||||
|
||||
test-openclaw-integration:
|
||||
runs-on: ubuntu-latest
|
||||
env:
|
||||
HINDSIGHT_API_LLM_PROVIDER: groq
|
||||
HINDSIGHT_API_LLM_API_KEY: ${{ secrets.GROQ_API_KEY }}
|
||||
HINDSIGHT_API_LLM_MODEL: openai/gpt-oss-20b
|
||||
HINDSIGHT_API_URL: http://localhost:8888
|
||||
HINDSIGHT_EMBED_PACKAGE_PATH: ${{ github.workspace }}/hindsight-embed
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
UV_INDEX: pytorch=https://download.pytorch.org/whl/cpu
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- name: Install uv
|
||||
uses: astral-sh/setup-uv@v5
|
||||
with:
|
||||
enable-cache: true
|
||||
prune-cache: false
|
||||
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version-file: ".python-version"
|
||||
|
||||
- name: Set up Node.js
|
||||
uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version: '22'
|
||||
|
||||
- name: Build API
|
||||
working-directory: ./hindsight-api
|
||||
run: uv build
|
||||
|
||||
- name: Install API dependencies
|
||||
working-directory: ./hindsight-api
|
||||
run: uv sync --frozen --no-install-project --index-strategy unsafe-best-match
|
||||
|
||||
- name: Install embed dependencies
|
||||
working-directory: ./hindsight-embed
|
||||
run: uv sync --frozen --index-strategy unsafe-best-match
|
||||
|
||||
- name: Cache HuggingFace models
|
||||
uses: actions/cache@v4
|
||||
with:
|
||||
path: ~/.cache/huggingface
|
||||
key: ${{ runner.os }}-huggingface-${{ hashFiles('hindsight-api/pyproject.toml') }}
|
||||
restore-keys: |
|
||||
${{ runner.os }}-huggingface-
|
||||
|
||||
- name: Pre-download models
|
||||
working-directory: ./hindsight-api
|
||||
run: |
|
||||
uv run python -c "
|
||||
from sentence_transformers import SentenceTransformer, CrossEncoder
|
||||
print('Downloading embedding model...')
|
||||
SentenceTransformer('BAAI/bge-small-en-v1.5')
|
||||
print('Downloading cross-encoder model...')
|
||||
CrossEncoder('cross-encoder/ms-marco-MiniLM-L-6-v2')
|
||||
print('Models downloaded successfully')
|
||||
"
|
||||
|
||||
- name: Install openclaw integration dependencies
|
||||
working-directory: ./hindsight-integrations/openclaw
|
||||
run: npm ci
|
||||
|
||||
- name: Create .env file
|
||||
run: |
|
||||
cat > .env << EOF
|
||||
HINDSIGHT_API_LLM_PROVIDER=${{ env.HINDSIGHT_API_LLM_PROVIDER }}
|
||||
HINDSIGHT_API_LLM_API_KEY=${{ env.HINDSIGHT_API_LLM_API_KEY }}
|
||||
HINDSIGHT_API_LLM_MODEL=${{ env.HINDSIGHT_API_LLM_MODEL }}
|
||||
EOF
|
||||
|
||||
- name: Start API server
|
||||
run: |
|
||||
./scripts/dev/start-api.sh > /tmp/api-server.log 2>&1 &
|
||||
echo "Waiting for API server to be ready..."
|
||||
for i in {1..60}; do
|
||||
if curl -sf http://localhost:8888/health > /dev/null 2>&1; then
|
||||
echo "API server is ready after ${i}s"
|
||||
break
|
||||
fi
|
||||
if [ $i -eq 60 ]; then
|
||||
echo "API server failed to start after 60s"
|
||||
cat /tmp/api-server.log
|
||||
exit 1
|
||||
fi
|
||||
sleep 1
|
||||
done
|
||||
|
||||
- name: Run openclaw integration tests
|
||||
working-directory: ./hindsight-integrations/openclaw
|
||||
run: npm run test:integration
|
||||
|
||||
- name: Show API server logs
|
||||
if: always()
|
||||
run: |
|
||||
echo "=== API Server Logs ==="
|
||||
cat /tmp/api-server.log || echo "No API server log found"
|
||||
|
||||
test-integration:
|
||||
runs-on: ubuntu-latest
|
||||
env:
|
||||
|
|
|
|||
|
|
@ -34,8 +34,9 @@
|
|||
"build": "tsc",
|
||||
"dev": "tsc --watch",
|
||||
"clean": "rm -rf dist",
|
||||
"test": "vitest run",
|
||||
"test:watch": "vitest",
|
||||
"test": "vitest run src",
|
||||
"test:watch": "vitest src",
|
||||
"test:integration": "vitest run --config vitest.integration.config.ts",
|
||||
"prepublishOnly": "npm run clean && npm run build"
|
||||
},
|
||||
"dependencies": {
|
||||
|
|
|
|||
|
|
@ -237,20 +237,7 @@ export class HindsightClient {
|
|||
throw new Error(`Failed to recall memories (HTTP ${res.status}): ${text}`);
|
||||
}
|
||||
|
||||
const response = await res.json() as { results?: any[] };
|
||||
const results = response.results || [];
|
||||
|
||||
return {
|
||||
results: results.map((r: any) => ({
|
||||
content: r.text || r.content || '',
|
||||
score: r.score ?? 1.0,
|
||||
metadata: {
|
||||
document_id: r.document_id,
|
||||
chunk_id: r.chunk_id,
|
||||
...r.metadata,
|
||||
},
|
||||
})),
|
||||
};
|
||||
return res.json() as Promise<RecallResponse>;
|
||||
}
|
||||
|
||||
private async recallSubprocess(request: RecallRequest, timeoutMs?: number): Promise<RecallResponse> {
|
||||
|
|
@ -265,21 +252,7 @@ export class HindsightClient {
|
|||
timeout: timeoutMs ?? 30_000, // subprocess gets a longer default
|
||||
});
|
||||
|
||||
// Parse JSON output - returns { entities: {...}, results: [...] }
|
||||
const response = JSON.parse(stdout);
|
||||
const results = response.results || [];
|
||||
|
||||
return {
|
||||
results: results.map((r: any) => ({
|
||||
content: r.text || r.content || '',
|
||||
score: 1.0, // CLI doesn't return scores
|
||||
metadata: {
|
||||
document_id: r.document_id,
|
||||
chunk_id: r.chunk_id,
|
||||
...r.metadata,
|
||||
},
|
||||
})),
|
||||
};
|
||||
return JSON.parse(stdout) as RecallResponse;
|
||||
} catch (error) {
|
||||
throw new Error(`Failed to recall memories: ${error}`, { cause: error });
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,83 +1,143 @@
|
|||
import { describe, it, expect } from 'vitest';
|
||||
import { stripMemoryTags, extractRecallQuery } from './index.js';
|
||||
|
||||
/**
|
||||
* Unit tests for the memory feedback loop fix.
|
||||
* Verifies that <hindsight_memories> and <relevant_memories> tags
|
||||
* are stripped from content before RETAIN to prevent duplicates.
|
||||
*/
|
||||
describe('Memory Tag Stripping', () => {
|
||||
/**
|
||||
* Simulates the tag stripping logic from agent_end hook
|
||||
*/
|
||||
function stripMemoryTags(content: string): string {
|
||||
// Strip plugin-injected memory tags to prevent feedback loop
|
||||
content = content.replace(/<hindsight_memories>[\s\S]*?<\/hindsight_memories>/g, '');
|
||||
content = content.replace(/<relevant_memories>[\s\S]*?<\/relevant_memories>/g, '');
|
||||
return content;
|
||||
}
|
||||
// ---------------------------------------------------------------------------
|
||||
// stripMemoryTags
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
it('should strip simple hindsight_memories tags', () => {
|
||||
const input = 'User: Hello\n<hindsight_memories>\nRelevant memories here...\n</hindsight_memories>\nAssistant: How can I help?';
|
||||
const expected = 'User: Hello\n\nAssistant: How can I help?';
|
||||
const result = stripMemoryTags(input);
|
||||
expect(result).toBe(expected);
|
||||
describe('stripMemoryTags', () => {
|
||||
it('strips simple hindsight_memories tags', () => {
|
||||
const input =
|
||||
'User: Hello\n<hindsight_memories>\nRelevant memories here...\n</hindsight_memories>\nAssistant: How can I help?';
|
||||
expect(stripMemoryTags(input)).toBe('User: Hello\n\nAssistant: How can I help?');
|
||||
});
|
||||
|
||||
it('should strip relevant_memories tags', () => {
|
||||
it('strips relevant_memories tags', () => {
|
||||
const input = 'Before\n<relevant_memories>\nSome data\n</relevant_memories>\nAfter';
|
||||
const expected = 'Before\n\nAfter';
|
||||
const result = stripMemoryTags(input);
|
||||
expect(result).toBe(expected);
|
||||
expect(stripMemoryTags(input)).toBe('Before\n\nAfter');
|
||||
});
|
||||
|
||||
it('should strip multiple hindsight_memories blocks', () => {
|
||||
const input = 'Start\n<hindsight_memories>\nBlock 1\n</hindsight_memories>\nMiddle\n<hindsight_memories>\nBlock 2\n</hindsight_memories>\nEnd';
|
||||
const expected = 'Start\n\nMiddle\n\nEnd';
|
||||
const result = stripMemoryTags(input);
|
||||
expect(result).toBe(expected);
|
||||
it('strips multiple hindsight_memories blocks', () => {
|
||||
const input =
|
||||
'Start\n<hindsight_memories>\nBlock 1\n</hindsight_memories>\nMiddle\n<hindsight_memories>\nBlock 2\n</hindsight_memories>\nEnd';
|
||||
expect(stripMemoryTags(input)).toBe('Start\n\nMiddle\n\nEnd');
|
||||
});
|
||||
|
||||
it('should handle multiline memory blocks with JSON', () => {
|
||||
const input = 'User: What is the weather?\n<hindsight_memories>\nRelevant memories:\n{\n "memory": "User likes sunny weather"\n}\n</hindsight_memories>\nAssistant: Let me check';
|
||||
const expected = 'User: What is the weather?\n\nAssistant: Let me check';
|
||||
it('handles multiline memory blocks with JSON', () => {
|
||||
const input =
|
||||
'User: What is the weather?\n<hindsight_memories>\n[\n {"memory": "User likes sunny weather"}\n]\n</hindsight_memories>\nAssistant: Let me check';
|
||||
const result = stripMemoryTags(input);
|
||||
expect(result).toBe(expected);
|
||||
expect(result).toBe('User: What is the weather?\n\nAssistant: Let me check');
|
||||
});
|
||||
|
||||
it('should preserve content without memory tags', () => {
|
||||
it('preserves content without memory tags', () => {
|
||||
const input = 'User: Hello\nAssistant: Hi there!';
|
||||
const expected = 'User: Hello\nAssistant: Hi there!';
|
||||
const result = stripMemoryTags(input);
|
||||
expect(result).toBe(expected);
|
||||
expect(stripMemoryTags(input)).toBe(input);
|
||||
});
|
||||
|
||||
it('should handle nested-like content without actual nesting', () => {
|
||||
const input = '<hindsight_memories>Outer start\n</hindsight_memories>\nSafe content\n<hindsight_memories>\nOuter end</hindsight_memories>';
|
||||
const expected = '\nSafe content\n';
|
||||
const result = stripMemoryTags(input);
|
||||
expect(result).toBe(expected);
|
||||
it('strips both tag types in same content', () => {
|
||||
const input =
|
||||
'A\n<hindsight_memories>\nH mem\n</hindsight_memories>\nB\n<relevant_memories>\nR mem\n</relevant_memories>\nC';
|
||||
expect(stripMemoryTags(input)).toBe('A\n\nB\n\nC');
|
||||
});
|
||||
|
||||
it('should strip both tag types in same content', () => {
|
||||
const input = 'A\n<hindsight_memories>\nH mem\n</hindsight_memories>\nB\n<relevant_memories>\nR mem\n</relevant_memories>\nC';
|
||||
const expected = 'A\n\nB\n\nC';
|
||||
const result = stripMemoryTags(input);
|
||||
expect(result).toBe(expected);
|
||||
});
|
||||
|
||||
it('should handle real-world agent conversation with injected memories', () => {
|
||||
const input = '[role: system]\n<hindsight_memories>\nRelevant memories from past conversations (score 1=highest, prioritize recent when conflicting):\n[\n {\n "content": "User prefers dark mode",\n "relevance_score": 0.95\n }\n]\n\nUser message: How do I enable dark mode?\n</hindsight_memories>\n[system:end]\n\n[role: user]\nHow do I enable dark mode?\n[user:end]\n\n[role: assistant]\nBased on your previous preference, let me help you enable dark mode.\n[assistant:end]';
|
||||
it('strips tags from a real-world agent conversation with injected memories', () => {
|
||||
const input =
|
||||
'[role: system]\n<hindsight_memories>\nRelevant memories:\n[{"text": "User prefers dark mode"}]\nUser message: How do I enable dark mode?\n</hindsight_memories>\n[system:end]\n\n[role: user]\nHow do I enable dark mode?\n[user:end]\n\n[role: assistant]\nLet me help you enable dark mode.\n[assistant:end]';
|
||||
|
||||
const result = stripMemoryTags(input);
|
||||
|
||||
// Should not contain the memory tags
|
||||
expect(result).not.toContain('<hindsight_memories>');
|
||||
expect(result).not.toContain('</hindsight_memories>');
|
||||
expect(result).not.toContain('Relevant memories from past conversations');
|
||||
|
||||
// Should still contain the actual conversation
|
||||
expect(result).not.toContain('User prefers dark mode');
|
||||
expect(result).toContain('[role: user]');
|
||||
expect(result).toContain('How do I enable dark mode?');
|
||||
expect(result).toContain('[role: assistant]');
|
||||
});
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// extractRecallQuery
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
describe('extractRecallQuery', () => {
|
||||
it('returns rawMessage when it is long enough', () => {
|
||||
expect(extractRecallQuery('What is my favorite food?', undefined)).toBe(
|
||||
'What is my favorite food?',
|
||||
);
|
||||
});
|
||||
|
||||
it('returns null when rawMessage is too short and prompt is absent', () => {
|
||||
expect(extractRecallQuery('Hi', undefined)).toBeNull();
|
||||
expect(extractRecallQuery('', '')).toBeNull();
|
||||
expect(extractRecallQuery(undefined, undefined)).toBeNull();
|
||||
});
|
||||
|
||||
it('returns null when both rawMessage and prompt are too short', () => {
|
||||
expect(extractRecallQuery('Hey', 'Hey')).toBeNull();
|
||||
});
|
||||
|
||||
it('falls back to prompt when rawMessage is absent', () => {
|
||||
const result = extractRecallQuery(undefined, 'What programming language do I prefer?');
|
||||
expect(result).toBe('What programming language do I prefer?');
|
||||
});
|
||||
|
||||
it('strips leading System: lines from prompt', () => {
|
||||
const prompt = 'System: You are an agent.\nSystem: Use tools wisely.\n\nWhat is my name?';
|
||||
const result = extractRecallQuery(undefined, prompt);
|
||||
expect(result).not.toContain('System:');
|
||||
expect(result).toContain('What is my name?');
|
||||
});
|
||||
|
||||
it('strips [Channel] envelope header and returns inner message', () => {
|
||||
const prompt = '[Telegram Chat]\nWhat is my favorite hobby?';
|
||||
const result = extractRecallQuery(undefined, prompt);
|
||||
expect(result).toBe('What is my favorite hobby?');
|
||||
});
|
||||
|
||||
it('strips [from: SenderName] footer from group chat prompts', () => {
|
||||
const prompt = '[Slack Channel #general]\nWhat should I eat for lunch?\n[from: Alice]';
|
||||
const result = extractRecallQuery(undefined, prompt);
|
||||
expect(result).not.toContain('[from: Alice]');
|
||||
expect(result).toContain('What should I eat for lunch?');
|
||||
});
|
||||
|
||||
it('handles full envelope with System lines, channel header, and from footer', () => {
|
||||
const prompt =
|
||||
'System: You are a helpful agent.\n\n[Discord Server]\nRemind me what I said about Python?\n[from: Bob]';
|
||||
const result = extractRecallQuery(undefined, prompt);
|
||||
expect(result).not.toContain('System:');
|
||||
expect(result).not.toContain('[Discord');
|
||||
expect(result).not.toContain('[from: Bob]');
|
||||
expect(result).toContain('Remind me what I said about Python?');
|
||||
});
|
||||
|
||||
it('strips session abort hint from prompt', () => {
|
||||
const prompt =
|
||||
'Note: The previous agent run was aborted by the user\n\n[Telegram]\nWhat is my cat\'s name?';
|
||||
const result = extractRecallQuery(undefined, prompt);
|
||||
expect(result).not.toContain('Note: The previous agent run was aborted');
|
||||
expect(result).toContain("What is my cat's name?");
|
||||
});
|
||||
|
||||
it('returns null when prompt reduces to < 5 chars after stripping', () => {
|
||||
// Envelope with almost-empty inner message
|
||||
const prompt = '[Telegram Chat]\nHi';
|
||||
const result = extractRecallQuery(undefined, prompt);
|
||||
expect(result).toBeNull();
|
||||
});
|
||||
|
||||
it('prefers rawMessage over prompt even when prompt is longer', () => {
|
||||
const rawMessage = 'What do I like to eat?';
|
||||
const prompt = '[Telegram]\nWhat do I like to eat?\n[from: Alice]';
|
||||
const result = extractRecallQuery(rawMessage, prompt);
|
||||
// Should return the clean rawMessage verbatim
|
||||
expect(result).toBe(rawMessage);
|
||||
expect(result).not.toContain('[from: Alice]');
|
||||
});
|
||||
|
||||
it('trims whitespace from result', () => {
|
||||
const result = extractRecallQuery(' What is my job? ', undefined);
|
||||
expect(result).toBe('What is my job?');
|
||||
});
|
||||
});
|
||||
|
|
|
|||
|
|
@ -138,6 +138,67 @@ const __dirname = dirname(__filename);
|
|||
// Default bank name (fallback when channel context not available)
|
||||
const DEFAULT_BANK_NAME = 'openclaw';
|
||||
|
||||
/**
|
||||
* Strip plugin-injected memory tags from content to prevent retain feedback loop.
|
||||
* Removes <hindsight_memories> and <relevant_memories> blocks that were injected
|
||||
* during before_agent_start so they don't get re-stored into the memory bank.
|
||||
*/
|
||||
export function stripMemoryTags(content: string): string {
|
||||
content = content.replace(/<hindsight_memories>[\s\S]*?<\/hindsight_memories>/g, '');
|
||||
content = content.replace(/<relevant_memories>[\s\S]*?<\/relevant_memories>/g, '');
|
||||
return content;
|
||||
}
|
||||
|
||||
/**
|
||||
* Extract a recall query from a hook event's rawMessage or prompt.
|
||||
*
|
||||
* Prefers rawMessage (clean user text). Falls back to prompt, stripping
|
||||
* envelope formatting (System: lines, [Channel ...] headers, [from: X] footers).
|
||||
*
|
||||
* Returns null when no usable query (< 5 chars) can be extracted.
|
||||
*/
|
||||
export function extractRecallQuery(
|
||||
rawMessage: string | undefined,
|
||||
prompt: string | undefined,
|
||||
): string | null {
|
||||
let recallQuery = rawMessage;
|
||||
if (!recallQuery || typeof recallQuery !== 'string' || recallQuery.trim().length < 5) {
|
||||
recallQuery = prompt;
|
||||
if (!recallQuery || typeof recallQuery !== 'string' || recallQuery.length < 5) {
|
||||
return null;
|
||||
}
|
||||
|
||||
// Strip envelope-formatted prompts from any channel
|
||||
let cleaned = recallQuery;
|
||||
|
||||
// Remove leading "System: ..." lines (from prependSystemEvents)
|
||||
cleaned = cleaned.replace(/^(?:System:.*\n)+\n?/, '');
|
||||
|
||||
// Remove session abort hint
|
||||
cleaned = cleaned.replace(
|
||||
/^Note: The previous agent run was aborted[^\n]*\n\n/,
|
||||
'',
|
||||
);
|
||||
|
||||
// Extract message after [ChannelName ...] envelope header
|
||||
const envelopeMatch = cleaned.match(
|
||||
/\[[A-Z][A-Za-z]*(?:\s[^\]]+)?\]\s*([\s\S]+)$/,
|
||||
);
|
||||
if (envelopeMatch) {
|
||||
cleaned = envelopeMatch[1];
|
||||
}
|
||||
|
||||
// Remove trailing [from: SenderName] metadata (group chats)
|
||||
cleaned = cleaned.replace(/\n\[from:[^\]]*\]\s*$/, '');
|
||||
|
||||
recallQuery = cleaned.trim() || recallQuery;
|
||||
}
|
||||
|
||||
const trimmed = recallQuery.trim();
|
||||
if (trimmed.length < 5) return null;
|
||||
return trimmed;
|
||||
}
|
||||
|
||||
/**
|
||||
* Agent context passed to plugin hooks.
|
||||
* These fields are populated by OpenClaw when invoking hooks.
|
||||
|
|
@ -671,44 +732,11 @@ export default function (api: MoltbotPluginAPI) {
|
|||
|
||||
// Get the user's latest message for recall — only the raw user text, not the full prompt
|
||||
// rawMessage is clean user text; prompt includes envelope, system events, media notes, etc.
|
||||
let recallQuery = event.rawMessage;
|
||||
if (!recallQuery || typeof recallQuery !== 'string' || recallQuery.trim().length < 5) {
|
||||
// Fall back to prompt but strip envelope formatting
|
||||
recallQuery = event.prompt;
|
||||
if (!recallQuery || typeof recallQuery !== 'string' || recallQuery.length < 5) {
|
||||
const extracted = extractRecallQuery(event.rawMessage, event.prompt);
|
||||
if (!extracted) {
|
||||
return;
|
||||
}
|
||||
|
||||
// Strip envelope-formatted prompts from any channel
|
||||
let cleaned = recallQuery;
|
||||
|
||||
// Remove leading "System: ..." lines (from prependSystemEvents)
|
||||
cleaned = cleaned.replace(/^(?:System:.*\n)+\n?/, '');
|
||||
|
||||
// Remove session abort hint
|
||||
cleaned = cleaned.replace(
|
||||
/^Note: The previous agent run was aborted[^\n]*\n\n/,
|
||||
'',
|
||||
);
|
||||
|
||||
// Extract message after [ChannelName ...] envelope header
|
||||
const envelopeMatch = cleaned.match(
|
||||
/\[[A-Z][A-Za-z]*(?:\s[^\]]+)?\]\s*([\s\S]+)$/,
|
||||
);
|
||||
if (envelopeMatch) {
|
||||
cleaned = envelopeMatch[1];
|
||||
}
|
||||
|
||||
// Remove trailing [from: SenderName] metadata (group chats)
|
||||
cleaned = cleaned.replace(/\n\[from:[^\]]*\]\s*$/, '');
|
||||
|
||||
recallQuery = cleaned.trim() || recallQuery;
|
||||
}
|
||||
|
||||
let prompt = recallQuery.trim();
|
||||
if (prompt.length < 5) {
|
||||
return; // Skip very short messages after extraction
|
||||
}
|
||||
let prompt = extracted;
|
||||
|
||||
// Truncate — Hindsight API recall has a 500 token limit; 800 chars stays safely under even with non-ASCII
|
||||
const MAX_RECALL_QUERY_CHARS = 800;
|
||||
|
|
@ -758,7 +786,7 @@ export default function (api: MoltbotPluginAPI) {
|
|||
const memoriesJson = JSON.stringify(response.results, null, 2);
|
||||
|
||||
const contextMessage = `<hindsight_memories>
|
||||
Relevant memories from past conversations (score 1=highest, prioritize recent when conflicting):
|
||||
Relevant memories from past conversations (prioritize recent when conflicting):
|
||||
${memoriesJson}
|
||||
|
||||
User message: ${prompt}
|
||||
|
|
@ -835,10 +863,7 @@ User message: ${prompt}
|
|||
}
|
||||
|
||||
// Strip plugin-injected memory tags to prevent feedback loop
|
||||
// Remove <hindsight_memories> blocks injected during before_agent_start
|
||||
content = content.replace(/<hindsight_memories>[\s\S]*?<\/hindsight_memories>/g, '');
|
||||
// Remove any <relevant_memories> blocks (legacy/alternative format)
|
||||
content = content.replace(/<relevant_memories>[\s\S]*?<\/relevant_memories>/g, '');
|
||||
content = stripMemoryTags(content);
|
||||
|
||||
return `[role: ${role}]\n${content}\n[${role}:end]`;
|
||||
})
|
||||
|
|
|
|||
|
|
@ -72,16 +72,24 @@ export interface RecallRequest {
|
|||
|
||||
export interface RecallResponse {
|
||||
results: MemoryResult[];
|
||||
entities: Record<string, unknown> | null;
|
||||
trace: unknown | null;
|
||||
chunks: unknown | null;
|
||||
}
|
||||
|
||||
export interface MemoryResult {
|
||||
content: string;
|
||||
score: number;
|
||||
metadata?: {
|
||||
document_id?: string;
|
||||
created_at?: string;
|
||||
source?: string;
|
||||
};
|
||||
id: string;
|
||||
text: string;
|
||||
type: string;
|
||||
entities: string[];
|
||||
context: string;
|
||||
occurred_start: string | null;
|
||||
occurred_end: string | null;
|
||||
mentioned_at: string | null;
|
||||
document_id: string | null;
|
||||
metadata: Record<string, unknown> | null;
|
||||
chunk_id: string | null;
|
||||
tags: string[];
|
||||
}
|
||||
|
||||
export interface CreateBankRequest {
|
||||
|
|
|
|||
542
hindsight-integrations/openclaw/tests/hooks.integration.test.ts
Normal file
542
hindsight-integrations/openclaw/tests/hooks.integration.test.ts
Normal file
|
|
@ -0,0 +1,542 @@
|
|||
/**
|
||||
* Integration tests for the OpenClaw plugin hooks.
|
||||
*
|
||||
* Loads the plugin with a mock MoltbotPluginAPI in HTTP mode, then triggers
|
||||
* `before_agent_start` and `agent_end` hooks with realistic event payloads.
|
||||
* Client methods (recall / retain) are spied on to verify the plugin
|
||||
* orchestrates them correctly without requiring a full LLM pipeline.
|
||||
*
|
||||
* Requirements:
|
||||
* Running Hindsight API at HINDSIGHT_API_URL (default: http://localhost:8888)
|
||||
*
|
||||
* Run:
|
||||
* npm run test:integration
|
||||
*/
|
||||
|
||||
import { describe, it, expect, beforeAll, afterAll, afterEach, vi } from 'vitest';
|
||||
import type { HindsightClient } from '../src/client.js';
|
||||
import type { MoltbotPluginAPI, PluginConfig } from '../src/types.js';
|
||||
import type { RecallResponse, RetainResponse } from '../src/types.js';
|
||||
|
||||
const HINDSIGHT_API_URL = process.env.HINDSIGHT_API_URL || 'http://localhost:8888';
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Helpers
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
async function waitForApi(url: string, maxMs = 5000): Promise<boolean> {
|
||||
const deadline = Date.now() + maxMs;
|
||||
while (Date.now() < deadline) {
|
||||
try {
|
||||
const res = await fetch(`${url}/health`, { signal: AbortSignal.timeout(1000) });
|
||||
if (res.ok) return true;
|
||||
} catch {
|
||||
/* not ready yet */
|
||||
}
|
||||
await new Promise((r) => setTimeout(r, 500));
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
interface MockApiHandle {
|
||||
api: MoltbotPluginAPI;
|
||||
/** Trigger a registered hook and return the last handler's return value. */
|
||||
trigger(event: string, eventData: unknown, ctx?: unknown): Promise<unknown>;
|
||||
startServices(): Promise<void>;
|
||||
stopServices(): Promise<void>;
|
||||
}
|
||||
|
||||
function createMockApi(pluginConfig: Partial<PluginConfig> = {}): MockApiHandle {
|
||||
const handlers = new Map<string, ((event: unknown, ctx?: unknown) => unknown)[]>();
|
||||
const services: { id: string; start(): Promise<void>; stop(): Promise<void> }[] = [];
|
||||
|
||||
const api: MoltbotPluginAPI = {
|
||||
config: {
|
||||
plugins: {
|
||||
entries: {
|
||||
'hindsight-openclaw': { enabled: true, config: pluginConfig as PluginConfig },
|
||||
},
|
||||
},
|
||||
},
|
||||
registerService(svc: any) {
|
||||
services.push(svc);
|
||||
},
|
||||
on(event: string, handler: any) {
|
||||
const list = handlers.get(event) ?? [];
|
||||
list.push(handler);
|
||||
handlers.set(event, list);
|
||||
},
|
||||
};
|
||||
|
||||
return {
|
||||
api,
|
||||
async trigger(event, eventData, ctx) {
|
||||
const list = handlers.get(event) ?? [];
|
||||
let result: unknown;
|
||||
for (const h of list) result = await h(eventData, ctx);
|
||||
return result;
|
||||
},
|
||||
async startServices() {
|
||||
for (const svc of services) await svc.start();
|
||||
},
|
||||
async stopServices() {
|
||||
for (const svc of services) await svc.stop();
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
const EMPTY_RECALL: RecallResponse = { results: [], entities: null, trace: null, chunks: null };
|
||||
const OK_RETAIN: RetainResponse = { message: 'queued', document_id: 'test', memory_unit_ids: [] };
|
||||
|
||||
function makeMemoryResult(text: string) {
|
||||
return {
|
||||
id: `mem-${Math.random().toString(36).slice(2)}`,
|
||||
text,
|
||||
type: 'fact',
|
||||
entities: [],
|
||||
context: '',
|
||||
occurred_start: null,
|
||||
occurred_end: null,
|
||||
mentioned_at: null,
|
||||
document_id: null,
|
||||
metadata: null,
|
||||
chunk_id: null,
|
||||
tags: [],
|
||||
};
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Module-level state shared across all hook describe blocks
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
let apiReachable = false;
|
||||
let triggerHook: MockApiHandle['trigger'];
|
||||
let stopServicesFn: () => Promise<void>;
|
||||
let recallSpy: ReturnType<typeof vi.spyOn<HindsightClient, 'recall'>>;
|
||||
let retainSpy: ReturnType<typeof vi.spyOn<HindsightClient, 'retain'>>;
|
||||
|
||||
beforeAll(async () => {
|
||||
apiReachable = await waitForApi(HINDSIGHT_API_URL, 8000);
|
||||
if (!apiReachable) {
|
||||
console.warn(
|
||||
`[Hooks Integration] Hindsight API not reachable at ${HINDSIGHT_API_URL} – skipping hook tests.`,
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
// Reset module registry so we get a fresh module with clean state.
|
||||
vi.resetModules();
|
||||
|
||||
// Provide LLM config — used by plugin init even in HTTP mode.
|
||||
process.env.HINDSIGHT_API_LLM_PROVIDER = 'openai';
|
||||
process.env.HINDSIGHT_API_LLM_API_KEY = 'test-key-hooks';
|
||||
// Point the plugin at the running test API.
|
||||
process.env.HINDSIGHT_EMBED_API_URL = HINDSIGHT_API_URL;
|
||||
|
||||
const mod = await import('../src/index.js');
|
||||
const pluginFn = mod.default;
|
||||
const getClient = mod.getClient;
|
||||
|
||||
const handle = createMockApi({
|
||||
dynamicBankId: true,
|
||||
excludeProviders: ['slack'],
|
||||
// No bankMission — keeps init lean
|
||||
});
|
||||
triggerHook = handle.trigger;
|
||||
stopServicesFn = handle.stopServices;
|
||||
|
||||
// Load the plugin — registers hooks and starts background init.
|
||||
pluginFn(handle.api);
|
||||
|
||||
// service.start() awaits initPromise and health-checks the external API.
|
||||
await handle.startServices();
|
||||
|
||||
// After startServices the client must be ready.
|
||||
const c = getClient();
|
||||
if (!c) throw new Error('[Hooks Integration] Client not initialized after service start');
|
||||
|
||||
recallSpy = vi.spyOn(c, 'recall') as ReturnType<typeof vi.spyOn<HindsightClient, 'recall'>>;
|
||||
retainSpy = vi.spyOn(c, 'retain') as ReturnType<typeof vi.spyOn<HindsightClient, 'retain'>>;
|
||||
}, 30_000);
|
||||
|
||||
afterAll(async () => {
|
||||
vi.restoreAllMocks();
|
||||
delete process.env.HINDSIGHT_API_LLM_PROVIDER;
|
||||
delete process.env.HINDSIGHT_API_LLM_API_KEY;
|
||||
delete process.env.HINDSIGHT_EMBED_API_URL;
|
||||
if (stopServicesFn) await stopServicesFn().catch(() => {});
|
||||
}, 15_000);
|
||||
|
||||
afterEach(() => {
|
||||
// Reset spy call history between tests; don't remove the implementation.
|
||||
recallSpy?.mockReset();
|
||||
retainSpy?.mockReset();
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// before_agent_start
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
describe('before_agent_start hook', () => {
|
||||
it('skips recall for excluded providers and returns undefined', async () => {
|
||||
if (!apiReachable) return;
|
||||
|
||||
const result = await triggerHook(
|
||||
'before_agent_start',
|
||||
{ rawMessage: 'What are my preferences?', prompt: 'What are my preferences?' },
|
||||
{ messageProvider: 'slack', senderId: 'U001' },
|
||||
);
|
||||
|
||||
expect(recallSpy).not.toHaveBeenCalled();
|
||||
expect(result).toBeUndefined();
|
||||
});
|
||||
|
||||
it('skips recall when rawMessage is too short and returns undefined', async () => {
|
||||
if (!apiReachable) return;
|
||||
|
||||
const result = await triggerHook(
|
||||
'before_agent_start',
|
||||
{ rawMessage: 'Hi', prompt: 'Hi' },
|
||||
{ messageProvider: 'telegram', senderId: 'U001' },
|
||||
);
|
||||
|
||||
expect(recallSpy).not.toHaveBeenCalled();
|
||||
expect(result).toBeUndefined();
|
||||
});
|
||||
|
||||
it('returns undefined when recall finds no results', async () => {
|
||||
if (!apiReachable) return;
|
||||
recallSpy.mockResolvedValue(EMPTY_RECALL);
|
||||
|
||||
const result = await triggerHook(
|
||||
'before_agent_start',
|
||||
{ rawMessage: 'What programming language do I like?', prompt: '' },
|
||||
{ messageProvider: 'telegram', senderId: 'U002' },
|
||||
);
|
||||
|
||||
expect(recallSpy).toHaveBeenCalledOnce();
|
||||
expect(result).toBeUndefined();
|
||||
});
|
||||
|
||||
it('returns { prependContext } with <hindsight_memories> when recall returns results', async () => {
|
||||
if (!apiReachable) return;
|
||||
recallSpy.mockResolvedValue({
|
||||
results: [makeMemoryResult('User likes Python')],
|
||||
entities: null,
|
||||
trace: null,
|
||||
chunks: null,
|
||||
});
|
||||
|
||||
const result = (await triggerHook(
|
||||
'before_agent_start',
|
||||
{ rawMessage: 'What programming language do I prefer?', prompt: '' },
|
||||
{ messageProvider: 'telegram', senderId: 'U003' },
|
||||
)) as { prependContext: string };
|
||||
|
||||
expect(result).toBeDefined();
|
||||
expect(result.prependContext).toContain('<hindsight_memories>');
|
||||
expect(result.prependContext).toContain('User likes Python');
|
||||
expect(result.prependContext).toContain('</hindsight_memories>');
|
||||
});
|
||||
|
||||
it('injects all memory result fields in the prependContext JSON', async () => {
|
||||
if (!apiReachable) return;
|
||||
const mem = makeMemoryResult('User prefers dark mode');
|
||||
mem.tags = ['preference'];
|
||||
mem.entities = ['dark_mode'];
|
||||
recallSpy.mockResolvedValue({
|
||||
results: [mem],
|
||||
entities: null,
|
||||
trace: null,
|
||||
chunks: null,
|
||||
});
|
||||
|
||||
const result = (await triggerHook(
|
||||
'before_agent_start',
|
||||
{ rawMessage: 'Do I prefer dark or light mode?', prompt: '' },
|
||||
{ messageProvider: 'telegram', senderId: 'U004' },
|
||||
)) as { prependContext: string };
|
||||
|
||||
// The prependContext should be valid JSON containing all MemoryResult fields
|
||||
const jsonStart = result.prependContext.indexOf('[');
|
||||
const jsonEnd = result.prependContext.lastIndexOf(']') + 1;
|
||||
const parsed = JSON.parse(result.prependContext.slice(jsonStart, jsonEnd)) as unknown[];
|
||||
expect(parsed).toHaveLength(1);
|
||||
const first = parsed[0] as Record<string, unknown>;
|
||||
expect(first.id).toBe(mem.id);
|
||||
expect(first.text).toBe('User prefers dark mode');
|
||||
expect(first.type).toBe('fact');
|
||||
expect(first.tags).toEqual(['preference']);
|
||||
expect(first.entities).toEqual(['dark_mode']);
|
||||
});
|
||||
|
||||
it('extracts the inner query from an envelope-formatted prompt when rawMessage is absent', async () => {
|
||||
if (!apiReachable) return;
|
||||
recallSpy.mockResolvedValue(EMPTY_RECALL);
|
||||
|
||||
const envelopePrompt = '[Telegram Chat]\nWhat is my favorite food?\n[from: Alice]';
|
||||
await triggerHook(
|
||||
'before_agent_start',
|
||||
{ rawMessage: '', prompt: envelopePrompt },
|
||||
{ messageProvider: 'telegram', senderId: 'U005' },
|
||||
);
|
||||
|
||||
expect(recallSpy).toHaveBeenCalledOnce();
|
||||
const [callArgs] = recallSpy.mock.calls[0];
|
||||
// The query passed to recall must NOT contain envelope artifacts
|
||||
expect(callArgs.query).not.toContain('[Telegram');
|
||||
expect(callArgs.query).not.toContain('[from: Alice]');
|
||||
expect(callArgs.query).toContain('What is my favorite food?');
|
||||
});
|
||||
|
||||
it('passes max_tokens to recall', async () => {
|
||||
if (!apiReachable) return;
|
||||
recallSpy.mockResolvedValue(EMPTY_RECALL);
|
||||
|
||||
await triggerHook(
|
||||
'before_agent_start',
|
||||
{ rawMessage: 'Tell me about my hobbies please.', prompt: '' },
|
||||
{ messageProvider: 'telegram', senderId: 'U006' },
|
||||
);
|
||||
|
||||
expect(recallSpy).toHaveBeenCalledOnce();
|
||||
const [callArgs] = recallSpy.mock.calls[0];
|
||||
expect(callArgs.max_tokens).toBeGreaterThan(0);
|
||||
});
|
||||
|
||||
it('includes the user message in the prependContext block', async () => {
|
||||
if (!apiReachable) return;
|
||||
recallSpy.mockResolvedValue({
|
||||
results: [makeMemoryResult('User loves hiking')],
|
||||
entities: null,
|
||||
trace: null,
|
||||
chunks: null,
|
||||
});
|
||||
|
||||
const result = (await triggerHook(
|
||||
'before_agent_start',
|
||||
{ rawMessage: 'What outdoor activities do I enjoy?', prompt: '' },
|
||||
{ messageProvider: 'telegram', senderId: 'U007' },
|
||||
)) as { prependContext: string };
|
||||
|
||||
expect(result.prependContext).toContain('What outdoor activities do I enjoy?');
|
||||
});
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// agent_end hook
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
describe('agent_end hook', () => {
|
||||
it('skips retain when success is false', async () => {
|
||||
if (!apiReachable) return;
|
||||
|
||||
await triggerHook(
|
||||
'agent_end',
|
||||
{ success: false, messages: [{ role: 'user', content: 'Hello there world!' }] },
|
||||
{ messageProvider: 'telegram', senderId: 'U010' },
|
||||
);
|
||||
|
||||
expect(retainSpy).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('skips retain when messages array is empty', async () => {
|
||||
if (!apiReachable) return;
|
||||
|
||||
await triggerHook(
|
||||
'agent_end',
|
||||
{ success: true, messages: [] },
|
||||
{ messageProvider: 'telegram', senderId: 'U011' },
|
||||
);
|
||||
|
||||
expect(retainSpy).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('skips retain for excluded providers', async () => {
|
||||
if (!apiReachable) return;
|
||||
|
||||
await triggerHook(
|
||||
'agent_end',
|
||||
{
|
||||
success: true,
|
||||
messages: [{ role: 'user', content: 'I work as a software engineer.' }],
|
||||
},
|
||||
{ messageProvider: 'slack', senderId: 'U012' },
|
||||
);
|
||||
|
||||
expect(retainSpy).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('calls retain with correctly formatted transcript for string content', async () => {
|
||||
if (!apiReachable) return;
|
||||
retainSpy.mockResolvedValue(OK_RETAIN);
|
||||
|
||||
await triggerHook(
|
||||
'agent_end',
|
||||
{
|
||||
success: true,
|
||||
messages: [
|
||||
{ role: 'user', content: 'I love TypeScript.' },
|
||||
{ role: 'assistant', content: 'TypeScript is great!' },
|
||||
],
|
||||
},
|
||||
{ messageProvider: 'telegram', senderId: 'U013', sessionKey: 'sess-ts-test' },
|
||||
);
|
||||
|
||||
expect(retainSpy).toHaveBeenCalledOnce();
|
||||
const [req] = retainSpy.mock.calls[0];
|
||||
expect(req.content).toContain('[role: user]');
|
||||
expect(req.content).toContain('I love TypeScript.');
|
||||
expect(req.content).toContain('[user:end]');
|
||||
expect(req.content).toContain('[role: assistant]');
|
||||
expect(req.content).toContain('TypeScript is great!');
|
||||
expect(req.content).toContain('[assistant:end]');
|
||||
});
|
||||
|
||||
it('includes session key in document_id', async () => {
|
||||
if (!apiReachable) return;
|
||||
retainSpy.mockResolvedValue(OK_RETAIN);
|
||||
|
||||
await triggerHook(
|
||||
'agent_end',
|
||||
{
|
||||
success: true,
|
||||
messages: [{ role: 'user', content: 'My favourite colour is blue.' }],
|
||||
},
|
||||
{ messageProvider: 'telegram', senderId: 'U014', sessionKey: 'sess-colour' },
|
||||
);
|
||||
|
||||
expect(retainSpy).toHaveBeenCalledOnce();
|
||||
const [req] = retainSpy.mock.calls[0];
|
||||
expect(req.document_id).toContain('sess-colour');
|
||||
});
|
||||
|
||||
it('populates metadata with channel_type, channel_id, and sender_id', async () => {
|
||||
if (!apiReachable) return;
|
||||
retainSpy.mockResolvedValue(OK_RETAIN);
|
||||
|
||||
await triggerHook(
|
||||
'agent_end',
|
||||
{
|
||||
success: true,
|
||||
messages: [{ role: 'user', content: 'My cat is named Whiskers.' }],
|
||||
},
|
||||
{
|
||||
messageProvider: 'telegram',
|
||||
channelId: 'chat-999',
|
||||
senderId: 'U015',
|
||||
sessionKey: 'sess-cat',
|
||||
},
|
||||
);
|
||||
|
||||
expect(retainSpy).toHaveBeenCalledOnce();
|
||||
const [req] = retainSpy.mock.calls[0];
|
||||
expect(req.metadata?.channel_type).toBe('telegram');
|
||||
expect(req.metadata?.channel_id).toBe('chat-999');
|
||||
expect(req.metadata?.sender_id).toBe('U015');
|
||||
expect(req.metadata?.retained_at).toBeDefined();
|
||||
expect(req.metadata?.message_count).toBe('1');
|
||||
});
|
||||
|
||||
it('strips <hindsight_memories> tags from content before retaining', async () => {
|
||||
if (!apiReachable) return;
|
||||
retainSpy.mockResolvedValue(OK_RETAIN);
|
||||
|
||||
const contentWithMemories =
|
||||
'<hindsight_memories>\nRelevant memories:\n[{"text":"old fact"}]\n</hindsight_memories>\nI enjoy reading science fiction.';
|
||||
|
||||
await triggerHook(
|
||||
'agent_end',
|
||||
{
|
||||
success: true,
|
||||
messages: [{ role: 'user', content: contentWithMemories }],
|
||||
},
|
||||
{ messageProvider: 'telegram', senderId: 'U016', sessionKey: 'sess-strip' },
|
||||
);
|
||||
|
||||
expect(retainSpy).toHaveBeenCalledOnce();
|
||||
const [req] = retainSpy.mock.calls[0];
|
||||
expect(req.content).not.toContain('<hindsight_memories>');
|
||||
expect(req.content).not.toContain('</hindsight_memories>');
|
||||
expect(req.content).not.toContain('old fact');
|
||||
expect(req.content).toContain('I enjoy reading science fiction.');
|
||||
});
|
||||
|
||||
it('strips <relevant_memories> tags from content before retaining', async () => {
|
||||
if (!apiReachable) return;
|
||||
retainSpy.mockResolvedValue(OK_RETAIN);
|
||||
|
||||
const contentWithLegacyTag =
|
||||
'<relevant_memories>\nSome old memories\n</relevant_memories>\nI am learning Rust.';
|
||||
|
||||
await triggerHook(
|
||||
'agent_end',
|
||||
{
|
||||
success: true,
|
||||
messages: [{ role: 'user', content: contentWithLegacyTag }],
|
||||
},
|
||||
{ messageProvider: 'telegram', senderId: 'U017', sessionKey: 'sess-legacy' },
|
||||
);
|
||||
|
||||
expect(retainSpy).toHaveBeenCalledOnce();
|
||||
const [req] = retainSpy.mock.calls[0];
|
||||
expect(req.content).not.toContain('<relevant_memories>');
|
||||
expect(req.content).toContain('I am learning Rust.');
|
||||
});
|
||||
|
||||
it('handles array content blocks (structured message format)', async () => {
|
||||
if (!apiReachable) return;
|
||||
retainSpy.mockResolvedValue(OK_RETAIN);
|
||||
|
||||
await triggerHook(
|
||||
'agent_end',
|
||||
{
|
||||
success: true,
|
||||
messages: [
|
||||
{
|
||||
role: 'user',
|
||||
content: [
|
||||
{ type: 'text', text: 'I prefer dark mode in all my editors.' },
|
||||
{ type: 'image', source: 'data:...' }, // non-text block — should be ignored
|
||||
],
|
||||
},
|
||||
],
|
||||
},
|
||||
{ messageProvider: 'telegram', senderId: 'U018', sessionKey: 'sess-array' },
|
||||
);
|
||||
|
||||
expect(retainSpy).toHaveBeenCalledOnce();
|
||||
const [req] = retainSpy.mock.calls[0];
|
||||
expect(req.content).toContain('I prefer dark mode in all my editors.');
|
||||
// Image block text should not appear
|
||||
expect(req.content).not.toContain('data:');
|
||||
});
|
||||
|
||||
it('retains a multi-turn conversation in the correct transcript format', async () => {
|
||||
if (!apiReachable) return;
|
||||
retainSpy.mockResolvedValue(OK_RETAIN);
|
||||
|
||||
await triggerHook(
|
||||
'agent_end',
|
||||
{
|
||||
success: true,
|
||||
messages: [
|
||||
{ role: 'user', content: 'My name is Carol.' },
|
||||
{ role: 'assistant', content: 'Nice to meet you, Carol!' },
|
||||
{ role: 'user', content: 'I work as a data scientist.' },
|
||||
{ role: 'assistant', content: "That's a fascinating career!" },
|
||||
],
|
||||
},
|
||||
{ messageProvider: 'telegram', senderId: 'U019', sessionKey: 'sess-multi' },
|
||||
);
|
||||
|
||||
expect(retainSpy).toHaveBeenCalledOnce();
|
||||
const [req] = retainSpy.mock.calls[0];
|
||||
|
||||
// Each message should appear in the correct envelope format
|
||||
expect(req.content).toContain('[role: user]\nMy name is Carol.\n[user:end]');
|
||||
expect(req.content).toContain('[role: assistant]\nNice to meet you, Carol!\n[assistant:end]');
|
||||
expect(req.content).toContain('[role: user]\nI work as a data scientist.\n[user:end]');
|
||||
expect(req.metadata?.message_count).toBe('4');
|
||||
});
|
||||
});
|
||||
385
hindsight-integrations/openclaw/tests/integration.test.ts
Normal file
385
hindsight-integrations/openclaw/tests/integration.test.ts
Normal file
|
|
@ -0,0 +1,385 @@
|
|||
/**
|
||||
* Integration tests for the Hindsight OpenClaw integration.
|
||||
*
|
||||
* Tests both HTTP mode (direct API calls) and Embed mode (subprocess/daemon).
|
||||
*
|
||||
* Requirements:
|
||||
* HTTP mode: Running Hindsight API at HINDSIGHT_API_URL (default: http://localhost:8888)
|
||||
* Embed mode: hindsight-embed package at HINDSIGHT_EMBED_PACKAGE_PATH
|
||||
* + LLM credentials (HINDSIGHT_API_LLM_PROVIDER / HINDSIGHT_API_LLM_API_KEY)
|
||||
*
|
||||
* Run:
|
||||
* npm run test:integration
|
||||
*/
|
||||
|
||||
import { describe, it, expect, beforeAll, afterAll } from 'vitest';
|
||||
import { join, dirname } from 'path';
|
||||
import { fileURLToPath } from 'url';
|
||||
import { HindsightClient } from '../src/client.js';
|
||||
import { HindsightEmbedManager } from '../src/embed-manager.js';
|
||||
|
||||
const __filename = fileURLToPath(import.meta.url);
|
||||
const __dirname = dirname(__filename);
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Test configuration (driven by environment variables)
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
const HINDSIGHT_API_URL = process.env.HINDSIGHT_API_URL || 'http://localhost:8888';
|
||||
const LLM_PROVIDER = process.env.HINDSIGHT_API_LLM_PROVIDER || '';
|
||||
const LLM_API_KEY = process.env.HINDSIGHT_API_LLM_API_KEY || '';
|
||||
const LLM_MODEL = process.env.HINDSIGHT_API_LLM_MODEL || '';
|
||||
|
||||
// Embed package path – defaults to the sibling hindsight-embed directory in the repo
|
||||
const EMBED_PACKAGE_PATH =
|
||||
process.env.HINDSIGHT_EMBED_PACKAGE_PATH ||
|
||||
join(__dirname, '..', '..', '..', 'hindsight-embed');
|
||||
|
||||
// Port for the test embed daemon (different from production default 9077 to avoid conflicts)
|
||||
const EMBED_TEST_PORT = 19077;
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Helpers
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
function randomBankId(): string {
|
||||
return `openclaw_test_${Math.random().toString(36).slice(2, 14)}`;
|
||||
}
|
||||
|
||||
async function waitForApi(url: string, maxMs = 5000): Promise<boolean> {
|
||||
const deadline = Date.now() + maxMs;
|
||||
while (Date.now() < deadline) {
|
||||
try {
|
||||
const res = await fetch(`${url}/health`, { signal: AbortSignal.timeout(1000) });
|
||||
if (res.ok) return true;
|
||||
} catch {
|
||||
// not ready yet
|
||||
}
|
||||
await new Promise((r) => setTimeout(r, 500));
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// HTTP Mode Tests
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
describe('HindsightClient – HTTP Mode', () => {
|
||||
let client: HindsightClient;
|
||||
|
||||
beforeAll(async () => {
|
||||
const reachable = await waitForApi(HINDSIGHT_API_URL);
|
||||
if (!reachable) {
|
||||
throw new Error(
|
||||
`Hindsight API not reachable at ${HINDSIGHT_API_URL}. ` +
|
||||
'Start the server before running integration tests.',
|
||||
);
|
||||
}
|
||||
|
||||
client = new HindsightClient({
|
||||
llmProvider: LLM_PROVIDER || 'openai',
|
||||
llmApiKey: LLM_API_KEY || 'test-key',
|
||||
llmModel: LLM_MODEL || undefined,
|
||||
apiUrl: HINDSIGHT_API_URL,
|
||||
});
|
||||
});
|
||||
|
||||
it('should retain a conversation', async () => {
|
||||
const bankId = randomBankId();
|
||||
client.setBankId(bankId);
|
||||
|
||||
const response = await client.retain({
|
||||
content:
|
||||
'[role: user]\nMy name is Alice and I love hiking.\n[user:end]\n\n' +
|
||||
'[role: assistant]\nNice to meet you, Alice!\n[assistant:end]',
|
||||
document_id: 'http-retain-test-1',
|
||||
metadata: { channel_type: 'slack', sender_id: 'U001' },
|
||||
});
|
||||
|
||||
expect(response).toBeDefined();
|
||||
expect(response.message).toBeDefined();
|
||||
expect(response.document_id).toBe('http-retain-test-1');
|
||||
});
|
||||
|
||||
it('should retain with auto-generated document id', async () => {
|
||||
const bankId = randomBankId();
|
||||
client.setBankId(bankId);
|
||||
|
||||
const response = await client.retain({
|
||||
content: '[role: user]\nI work at TechCorp as a software engineer.\n[user:end]',
|
||||
});
|
||||
|
||||
expect(response).toBeDefined();
|
||||
expect(response.document_id).toBe('conversation');
|
||||
});
|
||||
|
||||
it('should recall from an empty bank without error', async () => {
|
||||
const bankId = randomBankId();
|
||||
client.setBankId(bankId);
|
||||
|
||||
const response = await client.recall({ query: 'What do I like?', max_tokens: 512 });
|
||||
|
||||
expect(response).toBeDefined();
|
||||
expect(Array.isArray(response.results)).toBe(true);
|
||||
});
|
||||
|
||||
it('should set bank mission without throwing', async () => {
|
||||
const bankId = randomBankId();
|
||||
client.setBankId(bankId);
|
||||
|
||||
// setBankMission on a non-existent bank logs a warning but does not throw
|
||||
await expect(
|
||||
client.setBankMission('You are an assistant helping users via Slack.'),
|
||||
).resolves.not.toThrow();
|
||||
});
|
||||
|
||||
it('should set bank mission after retain creates the bank', async () => {
|
||||
const bankId = randomBankId();
|
||||
client.setBankId(bankId);
|
||||
|
||||
// Create the bank by retaining something first
|
||||
await client.retain({ content: '[role: user]\nHello\n[user:end]' });
|
||||
|
||||
// Now set the mission – bank exists so this should succeed
|
||||
await expect(
|
||||
client.setBankMission('You are a helpful AI assistant.'),
|
||||
).resolves.not.toThrow();
|
||||
});
|
||||
|
||||
it('should retain and then recall relevant memories', async () => {
|
||||
const bankId = randomBankId();
|
||||
client.setBankId(bankId);
|
||||
|
||||
await client.retain({
|
||||
content:
|
||||
'[role: user]\nMy favorite programming language is Python.\n[user:end]\n\n' +
|
||||
'[role: assistant]\nPython is a great choice!\n[assistant:end]',
|
||||
document_id: `session-${Date.now()}`,
|
||||
});
|
||||
|
||||
const response = await client.recall({
|
||||
query: 'What programming language do I like?',
|
||||
max_tokens: 1024,
|
||||
});
|
||||
|
||||
expect(response).toBeDefined();
|
||||
expect(Array.isArray(response.results)).toBe(true);
|
||||
});
|
||||
|
||||
it('should silently truncate recall queries over 800 chars', async () => {
|
||||
const bankId = randomBankId();
|
||||
client.setBankId(bankId);
|
||||
|
||||
const longQuery = 'Tell me about my interests. '.repeat(50); // > 800 chars
|
||||
const response = await client.recall({ query: longQuery, max_tokens: 512 });
|
||||
|
||||
expect(response).toBeDefined();
|
||||
expect(Array.isArray(response.results)).toBe(true);
|
||||
});
|
||||
|
||||
it('should use custom max_tokens in recall request', async () => {
|
||||
const bankId = randomBankId();
|
||||
client.setBankId(bankId);
|
||||
|
||||
const response = await client.recall({ query: 'anything', max_tokens: 256 });
|
||||
|
||||
expect(response).toBeDefined();
|
||||
expect(Array.isArray(response.results)).toBe(true);
|
||||
});
|
||||
|
||||
it('should map recall results to MemoryResult shape', async () => {
|
||||
const bankId = randomBankId();
|
||||
client.setBankId(bankId);
|
||||
|
||||
await client.retain({
|
||||
content:
|
||||
'[role: user]\nI enjoy reading science fiction books.\n[user:end]\n\n' +
|
||||
'[role: assistant]\nSounds like a great hobby!\n[assistant:end]',
|
||||
document_id: 'mapping-test',
|
||||
});
|
||||
|
||||
const response = await client.recall({ query: 'What are my hobbies?', max_tokens: 1024 });
|
||||
|
||||
for (const result of response.results) {
|
||||
expect(typeof result.id).toBe('string');
|
||||
expect(typeof result.text).toBe('string');
|
||||
expect(typeof result.type).toBe('string');
|
||||
expect(Array.isArray(result.entities)).toBe(true);
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Embed Mode Tests (subprocess / daemon)
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
describe('HindsightClient – Embed Mode (Subprocess)', () => {
|
||||
let client: HindsightClient;
|
||||
let embedManager: HindsightEmbedManager;
|
||||
|
||||
const hasEmbedCredentials = Boolean(LLM_PROVIDER && LLM_API_KEY);
|
||||
|
||||
beforeAll(async () => {
|
||||
if (!hasEmbedCredentials) {
|
||||
console.warn(
|
||||
'[Integration] Skipping embed mode tests: ' +
|
||||
'HINDSIGHT_API_LLM_PROVIDER and HINDSIGHT_API_LLM_API_KEY must both be set.',
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
embedManager = new HindsightEmbedManager(
|
||||
EMBED_TEST_PORT,
|
||||
LLM_PROVIDER,
|
||||
LLM_API_KEY,
|
||||
LLM_MODEL || undefined,
|
||||
undefined, // no custom base URL
|
||||
0, // never idle-timeout
|
||||
'latest',
|
||||
EMBED_PACKAGE_PATH,
|
||||
);
|
||||
|
||||
await embedManager.start();
|
||||
|
||||
client = new HindsightClient({
|
||||
llmProvider: LLM_PROVIDER,
|
||||
llmApiKey: LLM_API_KEY,
|
||||
llmModel: LLM_MODEL || undefined,
|
||||
embedPackagePath: EMBED_PACKAGE_PATH,
|
||||
});
|
||||
}, 120_000); // daemon startup can take up to 2 minutes
|
||||
|
||||
afterAll(async () => {
|
||||
if (embedManager) {
|
||||
await embedManager.stop();
|
||||
}
|
||||
}, 30_000);
|
||||
|
||||
it('should retain a conversation via subprocess', async () => {
|
||||
if (!hasEmbedCredentials) return;
|
||||
|
||||
const bankId = randomBankId();
|
||||
client.setBankId(bankId);
|
||||
|
||||
const response = await client.retain({
|
||||
content:
|
||||
'[role: user]\nI love hiking in the mountains.\n[user:end]\n\n' +
|
||||
'[role: assistant]\nSounds adventurous!\n[assistant:end]',
|
||||
document_id: 'embed-retain-test-1',
|
||||
});
|
||||
|
||||
expect(response).toBeDefined();
|
||||
expect(response.message).toBeDefined();
|
||||
expect(response.document_id).toBe('embed-retain-test-1');
|
||||
}, 60_000);
|
||||
|
||||
it('should retain with auto-generated document id via subprocess', async () => {
|
||||
if (!hasEmbedCredentials) return;
|
||||
|
||||
const bankId = randomBankId();
|
||||
client.setBankId(bankId);
|
||||
|
||||
const response = await client.retain({
|
||||
content: '[role: user]\nI am a TypeScript developer.\n[user:end]',
|
||||
});
|
||||
|
||||
expect(response).toBeDefined();
|
||||
expect(response.document_id).toBe('conversation');
|
||||
}, 60_000);
|
||||
|
||||
it('should recall from an empty bank without error via subprocess', async () => {
|
||||
if (!hasEmbedCredentials) return;
|
||||
|
||||
const bankId = randomBankId();
|
||||
client.setBankId(bankId);
|
||||
|
||||
const response = await client.recall({ query: 'What do I like?', max_tokens: 512 });
|
||||
|
||||
expect(response).toBeDefined();
|
||||
expect(Array.isArray(response.results)).toBe(true);
|
||||
}, 60_000);
|
||||
|
||||
it('should set bank mission via subprocess without throwing', async () => {
|
||||
if (!hasEmbedCredentials) return;
|
||||
|
||||
const bankId = randomBankId();
|
||||
client.setBankId(bankId);
|
||||
|
||||
// Create bank by retaining first, then set mission
|
||||
await client.retain({ content: '[role: user]\nHello\n[user:end]' });
|
||||
|
||||
await expect(
|
||||
client.setBankMission('Test mission for embed integration tests.'),
|
||||
).resolves.not.toThrow();
|
||||
}, 60_000);
|
||||
|
||||
it('should retain and then recall relevant memories via subprocess', async () => {
|
||||
if (!hasEmbedCredentials) return;
|
||||
|
||||
const bankId = randomBankId();
|
||||
client.setBankId(bankId);
|
||||
|
||||
await client.retain({
|
||||
content:
|
||||
'[role: user]\nMy cat is named Whiskers and she is 3 years old.\n[user:end]\n\n' +
|
||||
'[role: assistant]\nWhat a lovely name!\n[assistant:end]',
|
||||
document_id: `embed-e2e-${Date.now()}`,
|
||||
});
|
||||
|
||||
const response = await client.recall({
|
||||
query: "What is my cat's name?",
|
||||
max_tokens: 1024,
|
||||
});
|
||||
|
||||
expect(response).toBeDefined();
|
||||
expect(Array.isArray(response.results)).toBe(true);
|
||||
}, 60_000);
|
||||
|
||||
it('should map recall results to MemoryResult shape via subprocess', async () => {
|
||||
if (!hasEmbedCredentials) return;
|
||||
|
||||
const bankId = randomBankId();
|
||||
client.setBankId(bankId);
|
||||
|
||||
await client.retain({
|
||||
content:
|
||||
'[role: user]\nI enjoy cooking Italian food.\n[user:end]\n\n' +
|
||||
'[role: assistant]\nItalian cuisine is delicious!\n[assistant:end]',
|
||||
document_id: 'embed-shape-test',
|
||||
});
|
||||
|
||||
const response = await client.recall({ query: 'What food do I like?', max_tokens: 1024 });
|
||||
|
||||
for (const result of response.results) {
|
||||
expect(typeof result.id).toBe('string');
|
||||
expect(typeof result.text).toBe('string');
|
||||
expect(typeof result.type).toBe('string');
|
||||
expect(Array.isArray(result.entities)).toBe(true);
|
||||
}
|
||||
}, 60_000);
|
||||
|
||||
it('should handle full end-to-end workflow via subprocess', async () => {
|
||||
if (!hasEmbedCredentials) return;
|
||||
|
||||
const bankId = randomBankId();
|
||||
client.setBankId(bankId);
|
||||
|
||||
// Step 1: Retain
|
||||
const retainResp = await client.retain({
|
||||
content:
|
||||
'[role: user]\nI am learning Rust programming.\n[user:end]\n\n' +
|
||||
'[role: assistant]\nRust is a powerful systems language!\n[assistant:end]',
|
||||
document_id: `embed-workflow-${Date.now()}`,
|
||||
metadata: { channel_type: 'telegram', sender_id: '999' },
|
||||
});
|
||||
expect(retainResp).toBeDefined();
|
||||
|
||||
// Step 2: Recall
|
||||
const recallResp = await client.recall({
|
||||
query: 'What am I learning?',
|
||||
max_tokens: 1024,
|
||||
});
|
||||
expect(recallResp).toBeDefined();
|
||||
expect(Array.isArray(recallResp.results)).toBe(true);
|
||||
}, 60_000);
|
||||
});
|
||||
10
hindsight-integrations/openclaw/vitest.integration.config.ts
Normal file
10
hindsight-integrations/openclaw/vitest.integration.config.ts
Normal file
|
|
@ -0,0 +1,10 @@
|
|||
import { defineConfig } from 'vitest/config';
|
||||
|
||||
export default defineConfig({
|
||||
test: {
|
||||
include: ['tests/**/*.test.ts'],
|
||||
testTimeout: 120_000,
|
||||
hookTimeout: 120_000,
|
||||
reporters: ['verbose'],
|
||||
},
|
||||
});
|
||||
Loading…
Reference in a new issue