fix: openclaw tests + split doc-examples CI per language (#503)
* fix: update openclaw tests to use before_prompt_build hook and split doc-examples CI per language - Update hooks.integration.test.ts: rename describe block and all triggerHook calls from 'before_agent_start' to 'before_prompt_build' to match the hook registered in index.ts (changed in PR #480) - Fix 'includes the user message' test: prependContext contains memories (bullet list), not the raw user query; update assertion accordingly - Split test-doc-examples CI job into a matrix over [python, node, cli, go] so each language runs in parallel; language-specific setup steps (Rust/CLI build, Node.js, Python client, TypeScript client) are conditional on matrix.language to avoid unnecessary work * fix: spy on HindsightClient prototype to intercept all per-bank client instances getClientForContext creates new HindsightClient instances per bank when dynamicBankId is true, so vi.spyOn(c, 'recall') on the default client never captured calls. Spy on HindsightClient.prototype instead so all dynamically created bank clients are intercepted.
This commit is contained in:
parent
0ad8c2d09c
commit
e4dd654ec5
2 changed files with 40 additions and 26 deletions
16
.github/workflows/test.yml
vendored
16
.github/workflows/test.yml
vendored
|
|
@ -1321,6 +1321,11 @@ jobs:
|
||||||
|
|
||||||
test-doc-examples:
|
test-doc-examples:
|
||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
|
strategy:
|
||||||
|
fail-fast: false
|
||||||
|
matrix:
|
||||||
|
language: [python, node, cli, go]
|
||||||
|
name: test-doc-examples (${{ matrix.language }})
|
||||||
env:
|
env:
|
||||||
HINDSIGHT_API_LLM_PROVIDER: vertexai
|
HINDSIGHT_API_LLM_PROVIDER: vertexai
|
||||||
HINDSIGHT_API_LLM_VERTEXAI_SERVICE_ACCOUNT_KEY: /tmp/gcp-credentials.json
|
HINDSIGHT_API_LLM_VERTEXAI_SERVICE_ACCOUNT_KEY: /tmp/gcp-credentials.json
|
||||||
|
|
@ -1339,9 +1344,11 @@ jobs:
|
||||||
echo "HINDSIGHT_API_LLM_VERTEXAI_PROJECT_ID=$PROJECT_ID" >> $GITHUB_ENV
|
echo "HINDSIGHT_API_LLM_VERTEXAI_PROJECT_ID=$PROJECT_ID" >> $GITHUB_ENV
|
||||||
|
|
||||||
- name: Install Rust
|
- name: Install Rust
|
||||||
|
if: matrix.language == 'cli'
|
||||||
uses: dtolnay/rust-toolchain@stable
|
uses: dtolnay/rust-toolchain@stable
|
||||||
|
|
||||||
- name: Cache cargo
|
- name: Cache cargo
|
||||||
|
if: matrix.language == 'cli'
|
||||||
uses: actions/cache@v4
|
uses: actions/cache@v4
|
||||||
with:
|
with:
|
||||||
path: |
|
path: |
|
||||||
|
|
@ -1351,6 +1358,7 @@ jobs:
|
||||||
key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }}
|
key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }}
|
||||||
|
|
||||||
- name: Build CLI
|
- name: Build CLI
|
||||||
|
if: matrix.language == 'cli'
|
||||||
working-directory: hindsight-cli
|
working-directory: hindsight-cli
|
||||||
run: |
|
run: |
|
||||||
cargo build --release
|
cargo build --release
|
||||||
|
|
@ -1368,6 +1376,7 @@ jobs:
|
||||||
python-version-file: ".python-version"
|
python-version-file: ".python-version"
|
||||||
|
|
||||||
- name: Set up Node.js
|
- name: Set up Node.js
|
||||||
|
if: matrix.language == 'node'
|
||||||
uses: actions/setup-node@v4
|
uses: actions/setup-node@v4
|
||||||
with:
|
with:
|
||||||
node-version: '20'
|
node-version: '20'
|
||||||
|
|
@ -1381,10 +1390,12 @@ jobs:
|
||||||
uv sync --frozen --no-install-project --index-strategy unsafe-best-match
|
uv sync --frozen --no-install-project --index-strategy unsafe-best-match
|
||||||
|
|
||||||
- name: Install Python client dependencies
|
- name: Install Python client dependencies
|
||||||
|
if: matrix.language == 'python'
|
||||||
working-directory: ./hindsight-clients/python
|
working-directory: ./hindsight-clients/python
|
||||||
run: uv sync --frozen --extra test --index-strategy unsafe-best-match
|
run: uv sync --frozen --extra test --index-strategy unsafe-best-match
|
||||||
|
|
||||||
- name: Install TypeScript client
|
- name: Install TypeScript client
|
||||||
|
if: matrix.language == 'node'
|
||||||
run: |
|
run: |
|
||||||
npm ci --workspace=hindsight-clients/typescript
|
npm ci --workspace=hindsight-clients/typescript
|
||||||
npm run build --workspace=hindsight-clients/typescript
|
npm run build --workspace=hindsight-clients/typescript
|
||||||
|
|
@ -1436,10 +1447,11 @@ jobs:
|
||||||
done
|
done
|
||||||
|
|
||||||
- name: Configure CLI
|
- name: Configure CLI
|
||||||
|
if: matrix.language == 'cli'
|
||||||
run: hindsight configure --api-url http://localhost:8888
|
run: hindsight configure --api-url http://localhost:8888
|
||||||
|
|
||||||
- name: Run all doc examples
|
- name: Run doc examples (${{ matrix.language }})
|
||||||
run: ./scripts/test-doc-examples.sh
|
run: ./scripts/test-doc-examples.sh --lang ${{ matrix.language }}
|
||||||
|
|
||||||
- name: Show API server logs
|
- name: Show API server logs
|
||||||
if: always()
|
if: always()
|
||||||
|
|
|
||||||
|
|
@ -2,7 +2,7 @@
|
||||||
* Integration tests for the OpenClaw plugin hooks.
|
* Integration tests for the OpenClaw plugin hooks.
|
||||||
*
|
*
|
||||||
* Loads the plugin with a mock MoltbotPluginAPI in HTTP mode, then triggers
|
* Loads the plugin with a mock MoltbotPluginAPI in HTTP mode, then triggers
|
||||||
* `before_agent_start` and `agent_end` hooks with realistic event payloads.
|
* `before_prompt_build` and `agent_end` hooks with realistic event payloads.
|
||||||
* Client methods (recall / retain) are spied on to verify the plugin
|
* Client methods (recall / retain) are spied on to verify the plugin
|
||||||
* orchestrates them correctly without requiring a full LLM pipeline.
|
* orchestrates them correctly without requiring a full LLM pipeline.
|
||||||
*
|
*
|
||||||
|
|
@ -134,6 +134,7 @@ beforeAll(async () => {
|
||||||
process.env.HINDSIGHT_EMBED_API_URL = HINDSIGHT_API_URL;
|
process.env.HINDSIGHT_EMBED_API_URL = HINDSIGHT_API_URL;
|
||||||
|
|
||||||
const mod = await import('../src/index.js');
|
const mod = await import('../src/index.js');
|
||||||
|
const { HindsightClient } = await import('../src/client.js');
|
||||||
const pluginFn = mod.default;
|
const pluginFn = mod.default;
|
||||||
const getClient = mod.getClient;
|
const getClient = mod.getClient;
|
||||||
|
|
||||||
|
|
@ -156,11 +157,11 @@ beforeAll(async () => {
|
||||||
await handle.startServices();
|
await handle.startServices();
|
||||||
|
|
||||||
// After startServices the client must be ready.
|
// After startServices the client must be ready.
|
||||||
const c = getClient();
|
if (!getClient()) throw new Error('[Hooks Integration] Client not initialized after service start');
|
||||||
if (!c) throw new Error('[Hooks Integration] Client not initialized after service start');
|
|
||||||
|
|
||||||
recallSpy = vi.spyOn(c, 'recall') as ReturnType<typeof vi.spyOn<HindsightClient, 'recall'>>;
|
// Spy on the prototype so all per-bank instances created by getClientForContext are intercepted.
|
||||||
retainSpy = vi.spyOn(c, 'retain') as ReturnType<typeof vi.spyOn<HindsightClient, 'retain'>>;
|
recallSpy = vi.spyOn(HindsightClient.prototype, 'recall') as ReturnType<typeof vi.spyOn<HindsightClient, 'recall'>>;
|
||||||
|
retainSpy = vi.spyOn(HindsightClient.prototype, 'retain') as ReturnType<typeof vi.spyOn<HindsightClient, 'retain'>>;
|
||||||
}, 30_000);
|
}, 30_000);
|
||||||
|
|
||||||
afterAll(async () => {
|
afterAll(async () => {
|
||||||
|
|
@ -181,13 +182,13 @@ afterEach(() => {
|
||||||
// before_agent_start
|
// before_agent_start
|
||||||
// ---------------------------------------------------------------------------
|
// ---------------------------------------------------------------------------
|
||||||
|
|
||||||
describe('before_agent_start hook', () => {
|
describe('before_prompt_build hook', () => {
|
||||||
it('skips recall for excluded providers and returns undefined', async () => {
|
it('skips recall for excluded providers and returns undefined', async () => {
|
||||||
if (!apiReachable) return;
|
if (!apiReachable) return;
|
||||||
|
|
||||||
const result = await triggerHook(
|
const result = await triggerHook(
|
||||||
'before_agent_start',
|
'before_prompt_build',
|
||||||
{ rawMessage: 'What are my preferences?', prompt: 'What are my preferences?' },
|
{ rawMessage: 'What are my preferences?', prompt: 'What are my preferences?', messages: [] },
|
||||||
{ messageProvider: 'slack', senderId: 'U001' },
|
{ messageProvider: 'slack', senderId: 'U001' },
|
||||||
);
|
);
|
||||||
|
|
||||||
|
|
@ -199,8 +200,8 @@ describe('before_agent_start hook', () => {
|
||||||
if (!apiReachable) return;
|
if (!apiReachable) return;
|
||||||
|
|
||||||
const result = await triggerHook(
|
const result = await triggerHook(
|
||||||
'before_agent_start',
|
'before_prompt_build',
|
||||||
{ rawMessage: 'Hi', prompt: 'Hi' },
|
{ rawMessage: 'Hi', prompt: 'Hi', messages: [] },
|
||||||
{ messageProvider: 'telegram', senderId: 'U001' },
|
{ messageProvider: 'telegram', senderId: 'U001' },
|
||||||
);
|
);
|
||||||
|
|
||||||
|
|
@ -213,8 +214,8 @@ describe('before_agent_start hook', () => {
|
||||||
recallSpy.mockResolvedValue(EMPTY_RECALL);
|
recallSpy.mockResolvedValue(EMPTY_RECALL);
|
||||||
|
|
||||||
const result = await triggerHook(
|
const result = await triggerHook(
|
||||||
'before_agent_start',
|
'before_prompt_build',
|
||||||
{ rawMessage: 'What programming language do I like?', prompt: '' },
|
{ rawMessage: 'What programming language do I like?', prompt: '', messages: [] },
|
||||||
{ messageProvider: 'telegram', senderId: 'U002' },
|
{ messageProvider: 'telegram', senderId: 'U002' },
|
||||||
);
|
);
|
||||||
|
|
||||||
|
|
@ -232,8 +233,8 @@ describe('before_agent_start hook', () => {
|
||||||
});
|
});
|
||||||
|
|
||||||
const result = (await triggerHook(
|
const result = (await triggerHook(
|
||||||
'before_agent_start',
|
'before_prompt_build',
|
||||||
{ rawMessage: 'What programming language do I prefer?', prompt: '' },
|
{ rawMessage: 'What programming language do I prefer?', prompt: '', messages: [] },
|
||||||
{ messageProvider: 'telegram', senderId: 'U003' },
|
{ messageProvider: 'telegram', senderId: 'U003' },
|
||||||
)) as { prependContext: string };
|
)) as { prependContext: string };
|
||||||
|
|
||||||
|
|
@ -256,8 +257,8 @@ describe('before_agent_start hook', () => {
|
||||||
});
|
});
|
||||||
|
|
||||||
const result = (await triggerHook(
|
const result = (await triggerHook(
|
||||||
'before_agent_start',
|
'before_prompt_build',
|
||||||
{ rawMessage: 'Do I prefer dark or light mode?', prompt: '' },
|
{ rawMessage: 'Do I prefer dark or light mode?', prompt: '', messages: [] },
|
||||||
{ messageProvider: 'telegram', senderId: 'U004' },
|
{ messageProvider: 'telegram', senderId: 'U004' },
|
||||||
)) as { prependContext: string };
|
)) as { prependContext: string };
|
||||||
|
|
||||||
|
|
@ -273,8 +274,8 @@ describe('before_agent_start hook', () => {
|
||||||
|
|
||||||
const envelopePrompt = '[Telegram Chat]\nWhat is my favorite food?\n[from: Alice]';
|
const envelopePrompt = '[Telegram Chat]\nWhat is my favorite food?\n[from: Alice]';
|
||||||
await triggerHook(
|
await triggerHook(
|
||||||
'before_agent_start',
|
'before_prompt_build',
|
||||||
{ rawMessage: '', prompt: envelopePrompt },
|
{ rawMessage: '', prompt: envelopePrompt, messages: [] },
|
||||||
{ messageProvider: 'telegram', senderId: 'U005' },
|
{ messageProvider: 'telegram', senderId: 'U005' },
|
||||||
);
|
);
|
||||||
|
|
||||||
|
|
@ -317,8 +318,8 @@ describe('before_agent_start hook', () => {
|
||||||
recallSpy.mockResolvedValue(EMPTY_RECALL);
|
recallSpy.mockResolvedValue(EMPTY_RECALL);
|
||||||
|
|
||||||
await triggerHook(
|
await triggerHook(
|
||||||
'before_agent_start',
|
'before_prompt_build',
|
||||||
{ rawMessage: 'Tell me about my hobbies please.', prompt: '' },
|
{ rawMessage: 'Tell me about my hobbies please.', prompt: '', messages: [] },
|
||||||
{ messageProvider: 'telegram', senderId: 'U006' },
|
{ messageProvider: 'telegram', senderId: 'U006' },
|
||||||
);
|
);
|
||||||
|
|
||||||
|
|
@ -327,7 +328,7 @@ describe('before_agent_start hook', () => {
|
||||||
expect(callArgs.max_tokens).toBeGreaterThan(0);
|
expect(callArgs.max_tokens).toBeGreaterThan(0);
|
||||||
});
|
});
|
||||||
|
|
||||||
it('includes the user message in the prependContext block', async () => {
|
it('includes recalled memories in the prependContext block', async () => {
|
||||||
if (!apiReachable) return;
|
if (!apiReachable) return;
|
||||||
recallSpy.mockResolvedValue({
|
recallSpy.mockResolvedValue({
|
||||||
results: [makeMemoryResult('User loves hiking')],
|
results: [makeMemoryResult('User loves hiking')],
|
||||||
|
|
@ -337,12 +338,13 @@ describe('before_agent_start hook', () => {
|
||||||
});
|
});
|
||||||
|
|
||||||
const result = (await triggerHook(
|
const result = (await triggerHook(
|
||||||
'before_agent_start',
|
'before_prompt_build',
|
||||||
{ rawMessage: 'What outdoor activities do I enjoy?', prompt: '' },
|
{ rawMessage: 'What outdoor activities do I enjoy?', prompt: '', messages: [] },
|
||||||
{ messageProvider: 'telegram', senderId: 'U007' },
|
{ messageProvider: 'telegram', senderId: 'U007' },
|
||||||
)) as { prependContext: string };
|
)) as { prependContext: string };
|
||||||
|
|
||||||
expect(result.prependContext).toContain('What outdoor activities do I enjoy?');
|
expect(result.prependContext).toContain('User loves hiking');
|
||||||
|
expect(result.prependContext).toContain('<hindsight_memories>');
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue