feat(openclaw): add resilient startup and richer retain metadata (#942)
* feat(openclaw): enrich retain metadata and ignore heartbeat by default * docs(openclaw): move retain metadata note out of config table * fix(openclaw): make hook registration runtime-idempotent * fix(openclaw): lazily initialize when service start is skipped --------- Co-authored-by: Aldous <aldous@agentmail.to> Co-authored-by: Josh <joshdwells@gmail.com> Co-authored-by: Aldous the Orchestrator <Aldoustheorchestrator@users.noreply.github.com>
This commit is contained in:
Aldous
GitHub
parent
61a8014f9d
commit
1f1716bdb0
4 changed files with 137 additions and 22 deletions
|
|
@ -51,7 +51,7 @@ Optional settings in `~/.openclaw/openclaw.json` under `plugins.entries.hindsigh
|
|||
| `retainTags` | `[]` | Tags applied to every retained document, useful for cross-agent/source labeling (e.g. `source_system:openclaw`, `agent:agentname`) |
|
||||
| `retainSource` | `"openclaw"` | `source` value written into retained document metadata |
|
||||
| `dynamicBankGranularity` | `["agent", "channel", "user"]` | Fields used to derive bank ID. Options: `agent`, `channel`, `user`, `provider` |
|
||||
| `excludeProviders` | `[]` | Message providers to skip for recall/retain (e.g. `slack`, `telegram`, `discord`) |
|
||||
| `excludeProviders` | `["heartbeat"]` | Message providers to skip for recall/retain (e.g. `heartbeat`, `slack`, `telegram`, `discord`) |
|
||||
| `autoRecall` | `true` | Auto-inject memories before each turn. Set to `false` when the agent has its own recall tool. |
|
||||
| `autoRetain` | `true` | Auto-retain conversations after each turn |
|
||||
| `retainRoles` | `["user", "assistant"]` | Which message roles to retain. Options: `user`, `assistant`, `system`, `tool` |
|
||||
|
|
@ -68,6 +68,10 @@ Optional settings in `~/.openclaw/openclaw.json` under `plugins.entries.hindsigh
|
|||
| `hindsightApiUrl` | — | External Hindsight API URL (skips local daemon) |
|
||||
| `hindsightApiToken` | — | Auth token for external API |
|
||||
|
||||
## Retention details
|
||||
|
||||
Retained documents use stable session-scoped IDs like `openclaw:agent:agentname:discord:channel:123:turn:000001` (or `...:window:000002` for chunked retention), and include richer metadata such as `session_key`, `agent_id`, `provider`, `channel_id`, `thread_id`, `sender_id`, `turn_index`, and `retention_scope`.
|
||||
|
||||
## Documentation
|
||||
|
||||
For full documentation, configuration options, troubleshooting, and development guide, see:
|
||||
|
|
|
|||
|
|
@ -98,7 +98,8 @@
|
|||
"items": {
|
||||
"type": "string"
|
||||
},
|
||||
"description": "Message providers to exclude from recall and retain (e.g. ['telegram', 'discord'])"
|
||||
"description": "Message providers to exclude from recall and retain (e.g. ['heartbeat', 'telegram', 'discord'])",
|
||||
"default": ["heartbeat"]
|
||||
},
|
||||
"dynamicBankGranularity": {
|
||||
"type": "array",
|
||||
|
|
@ -320,7 +321,7 @@
|
|||
},
|
||||
"excludeProviders": {
|
||||
"label": "Excluded Providers",
|
||||
"placeholder": "e.g. telegram, discord"
|
||||
"placeholder": "e.g. heartbeat, telegram, discord"
|
||||
},
|
||||
"dynamicBankGranularity": {
|
||||
"label": "Bank Granularity",
|
||||
|
|
|
|||
|
|
@ -229,6 +229,7 @@ describe('formatMemories', () => {
|
|||
describe('buildRetainRequest', () => {
|
||||
it('adds configured source metadata and retain tags', () => {
|
||||
const request = buildRetainRequest('hello world', 2, {
|
||||
agentId: 'main',
|
||||
sessionKey: 'agent:main:main',
|
||||
messageProvider: 'discord',
|
||||
channelId: 'channel:123',
|
||||
|
|
@ -240,21 +241,56 @@ describe('buildRetainRequest', () => {
|
|||
|
||||
expect(request).toEqual({
|
||||
content: 'hello world',
|
||||
document_id: 'agent:main:main-1700000000000',
|
||||
document_id: 'openclaw:agent:main:main:turn:000001',
|
||||
metadata: {
|
||||
retained_at: expect.any(String),
|
||||
message_count: '2',
|
||||
source: 'openclaw',
|
||||
retention_scope: 'turn',
|
||||
turn_index: '1',
|
||||
session_key: 'agent:main:main',
|
||||
agent_id: 'main',
|
||||
provider: 'discord',
|
||||
channel_type: 'discord',
|
||||
channel_id: 'channel:123',
|
||||
thread_id: undefined,
|
||||
sender_id: 'user:456',
|
||||
},
|
||||
tags: ['source_system:openclaw', 'agent:agentname'],
|
||||
});
|
||||
});
|
||||
|
||||
it('uses window ids and metadata for chunked retention', () => {
|
||||
const request = buildRetainRequest('hello world', 4, {
|
||||
agentId: 'agentname',
|
||||
sessionKey: 'agent:agentname:discord:group:123:topic:456',
|
||||
messageProvider: 'discord',
|
||||
senderId: 'user:456',
|
||||
}, {
|
||||
retainSource: 'openclaw',
|
||||
}, 1700000000000, {
|
||||
retentionScope: 'window',
|
||||
turnIndex: 2,
|
||||
windowTurns: 2,
|
||||
});
|
||||
|
||||
expect(request.document_id).toBe('openclaw:agent:agentname:discord:group:123:topic:456:window:000002');
|
||||
expect(request.metadata).toMatchObject({
|
||||
source: 'openclaw',
|
||||
retention_scope: 'window',
|
||||
turn_index: '2',
|
||||
agent_id: 'agentname',
|
||||
provider: 'discord',
|
||||
channel_type: 'discord',
|
||||
channel_id: 'group:123:topic:456',
|
||||
thread_id: '456',
|
||||
sender_id: 'user:456',
|
||||
window_turns: '2',
|
||||
});
|
||||
});
|
||||
|
||||
it('defaults source metadata to openclaw when unset', () => {
|
||||
const request = buildRetainRequest('hello world', 1, {}, {}, 1700000000000);
|
||||
const request = buildRetainRequest('hello world', 1, {}, {}, 1700000000000, { turnIndex: 1 });
|
||||
expect(request.metadata?.source).toBe('openclaw');
|
||||
expect(request.tags).toBeUndefined();
|
||||
});
|
||||
|
|
|
|||
|
|
@ -43,10 +43,11 @@ const DEFAULT_RECALL_TIMEOUT_MS = 10_000;
|
|||
// Cache sender IDs discovered in before_prompt_build (where event.prompt has the metadata
|
||||
// blocks) so agent_end can look them up — event.messages in agent_end is clean history.
|
||||
const senderIdBySession = new Map<string, string>();
|
||||
const documentSequenceBySession = new Map<string, number>();
|
||||
|
||||
// Guard against double hook registration on the same api instance
|
||||
// Uses a WeakSet so each api instance can only register hooks once
|
||||
const registeredApis = new WeakSet<object>();
|
||||
// Guard against duplicate hook registration within a single runtime load.
|
||||
// Do not tie this to api instance identity, which can be brittle across loader phases.
|
||||
let hooksRegistered = false;
|
||||
|
||||
// Cooldown + guard to prevent concurrent reinit attempts
|
||||
let lastReinitAttempt = 0;
|
||||
|
|
@ -134,23 +135,23 @@ function formatCurrentTimeForRecall(date = new Date()): string {
|
|||
* Throttled to one attempt per 30s to avoid hammering a down service.
|
||||
* Only works if initialization was attempted at least once (isInitialized guard).
|
||||
*/
|
||||
async function lazyReinit(): Promise<void> {
|
||||
async function lazyReinit(configOverride?: PluginConfig): Promise<void> {
|
||||
const now = Date.now();
|
||||
if (now - lastReinitAttempt < REINIT_COOLDOWN_MS || isReinitInProgress) {
|
||||
return;
|
||||
}
|
||||
|
||||
// Only attempt lazy reinit if we've already done initial setup
|
||||
// (i.e., service.start() was called at least once)
|
||||
if (!currentPluginConfig) {
|
||||
debug('[Hindsight] lazyReinit skipped - no plugin config (service.start() not called yet)');
|
||||
const config = configOverride ?? currentPluginConfig;
|
||||
if (!config) {
|
||||
debug('[Hindsight] lazyReinit skipped - no plugin config available');
|
||||
return;
|
||||
}
|
||||
|
||||
// Persist config if we only have it from the live hook registration path.
|
||||
currentPluginConfig = config;
|
||||
|
||||
isReinitInProgress = true;
|
||||
lastReinitAttempt = now;
|
||||
|
||||
const config = currentPluginConfig;
|
||||
const externalApi = detectExternalApi(config);
|
||||
if (!externalApi.apiUrl) {
|
||||
isReinitInProgress = false;
|
||||
|
|
@ -200,6 +201,11 @@ if (typeof global !== 'undefined') {
|
|||
// If initPromise is null, it means service.start() hasn't been called yet
|
||||
// (CLI mode, not gateway mode). Hooks should gracefully no-op.
|
||||
if (!initPromise) {
|
||||
if (currentPluginConfig) {
|
||||
log.warn('waitForReady called before service.start() — attempting lazy initialization fallback');
|
||||
await lazyReinit(currentPluginConfig);
|
||||
return;
|
||||
}
|
||||
log.warn('waitForReady called before service.start() — hooks will no-op (expected in CLI mode)');
|
||||
return;
|
||||
}
|
||||
|
|
@ -813,7 +819,9 @@ function getPluginConfig(api: MoltbotPluginAPI): PluginConfig {
|
|||
bankIdPrefix: config.bankIdPrefix,
|
||||
retainTags: Array.isArray(config.retainTags) ? config.retainTags.filter((tag): tag is string => typeof tag === 'string') : undefined,
|
||||
retainSource: typeof config.retainSource === 'string' && config.retainSource.trim().length > 0 ? config.retainSource.trim() : undefined,
|
||||
excludeProviders: Array.isArray(config.excludeProviders) ? config.excludeProviders : [],
|
||||
excludeProviders: Array.isArray(config.excludeProviders)
|
||||
? Array.from(new Set(['heartbeat', ...config.excludeProviders.filter((provider): provider is string => typeof provider === 'string')]))
|
||||
: ['heartbeat'],
|
||||
autoRecall: config.autoRecall !== false, // Default: true (on) — backward compatible
|
||||
dynamicBankGranularity: Array.isArray(config.dynamicBankGranularity) ? config.dynamicBankGranularity : undefined,
|
||||
autoRetain: config.autoRetain !== false, // Default: true
|
||||
|
|
@ -839,6 +847,7 @@ function getPluginConfig(api: MoltbotPluginAPI): PluginConfig {
|
|||
|
||||
export default function (api: MoltbotPluginAPI) {
|
||||
try {
|
||||
log.info('plugin entry invoked');
|
||||
debug('[Hindsight] Plugin loading...');
|
||||
|
||||
// Get plugin config first (needed for debug flag and service registration)
|
||||
|
|
@ -863,9 +872,11 @@ export default function (api: MoltbotPluginAPI) {
|
|||
// happens in service.start() which is ONLY called on gateway start,
|
||||
// not on every CLI command.
|
||||
debug('[Hindsight] Registering service...');
|
||||
log.info('registering plugin service');
|
||||
api.registerService({
|
||||
id: 'hindsight-memory',
|
||||
async start() {
|
||||
log.info('service.start invoked');
|
||||
debug('[Hindsight] Service start called - beginning heavy initialization...');
|
||||
|
||||
// Detect LLM configuration (env vars > plugin config > auto-detect)
|
||||
|
|
@ -1180,12 +1191,13 @@ export default function (api: MoltbotPluginAPI) {
|
|||
debug('[Hindsight] Plugin loaded successfully');
|
||||
|
||||
// Register agent hooks for auto-recall and auto-retention
|
||||
if (registeredApis.has(api)) {
|
||||
debug('[Hindsight] Hooks already registered for this api instance, skipping duplicate registration');
|
||||
if (hooksRegistered) {
|
||||
debug('[Hindsight] Hooks already registered in this runtime, skipping duplicate hook registration');
|
||||
return;
|
||||
}
|
||||
registeredApis.add(api);
|
||||
hooksRegistered = true;
|
||||
debug('[Hindsight] Registering agent hooks...');
|
||||
log.info('registering agent hooks');
|
||||
|
||||
// Auto-recall: Inject relevant memories before agent processes the message
|
||||
// Hook signature: (event, ctx) where event has {prompt, messages?} and ctx has agent context
|
||||
|
|
@ -1433,7 +1445,17 @@ ${memoriesFormatted}
|
|||
|
||||
|
||||
const retainNow = Date.now();
|
||||
const retainRequest = buildRetainRequest(transcript, messageCount, effectiveCtx, pluginConfig, retainNow);
|
||||
const retainRequest = buildRetainRequest(
|
||||
transcript,
|
||||
messageCount,
|
||||
effectiveCtxForRetain,
|
||||
pluginConfig,
|
||||
retainNow,
|
||||
{
|
||||
retentionScope: retainFullWindow ? 'window' : 'turn',
|
||||
windowTurns: retainFullWindow ? (pluginConfig.retainEveryNTurns ?? 1) + (pluginConfig.retainOverlapTurns ?? 0) : undefined,
|
||||
},
|
||||
);
|
||||
|
||||
// Retain to Hindsight
|
||||
debug(`[Hindsight] Retaining to bank ${bankId}, document: ${retainRequest.document_id}, chars: ${transcript.length}\n---\n${transcript.substring(0, 500)}${transcript.length > 500 ? '\n...(truncated)' : ''}\n---`);
|
||||
|
|
@ -1462,6 +1484,7 @@ ${memoriesFormatted}
|
|||
}
|
||||
});
|
||||
debug('[Hindsight] Hooks registered');
|
||||
log.info('agent hooks registered');
|
||||
} catch (error) {
|
||||
log.error('plugin loading error', error);
|
||||
if (error instanceof Error) {
|
||||
|
|
@ -1473,23 +1496,74 @@ ${memoriesFormatted}
|
|||
|
||||
// Export client getter for tools
|
||||
|
||||
function sanitizeDocumentIdPart(value: string | undefined, fallback: string): string {
|
||||
const normalized = (value || '').trim();
|
||||
if (!normalized) return fallback;
|
||||
return normalized
|
||||
.replace(/[^a-zA-Z0-9:_-]+/g, '_')
|
||||
.replace(/_+/g, '_')
|
||||
.replace(/^_+|_+$/g, '') || fallback;
|
||||
}
|
||||
|
||||
function getSessionDocumentBase(effectiveCtx: PluginHookAgentContext | undefined): string {
|
||||
const sessionKeyPart = sanitizeDocumentIdPart(effectiveCtx?.sessionKey, 'session');
|
||||
return `openclaw:${sessionKeyPart}`;
|
||||
}
|
||||
|
||||
function nextDocumentSequence(effectiveCtx: PluginHookAgentContext | undefined): number {
|
||||
const sequenceKey = effectiveCtx?.sessionKey || 'session';
|
||||
const next = (documentSequenceBySession.get(sequenceKey) || 0) + 1;
|
||||
documentSequenceBySession.set(sequenceKey, next);
|
||||
if (documentSequenceBySession.size > MAX_TRACKED_SESSIONS) {
|
||||
const oldestKey = documentSequenceBySession.keys().next().value;
|
||||
if (oldestKey) {
|
||||
documentSequenceBySession.delete(oldestKey);
|
||||
}
|
||||
}
|
||||
return next;
|
||||
}
|
||||
|
||||
function extractThreadId(channelId: string | undefined): string | undefined {
|
||||
if (!channelId) return undefined;
|
||||
const match = channelId.match(/(?:^|:)topic:([^:]+)$/);
|
||||
return match?.[1];
|
||||
}
|
||||
|
||||
export function buildRetainRequest(
|
||||
transcript: string,
|
||||
messageCount: number,
|
||||
effectiveCtx: PluginHookAgentContext | undefined,
|
||||
pluginConfig: PluginConfig,
|
||||
now = Date.now(),
|
||||
options?: { retentionScope?: 'turn' | 'window' | 'manual'; windowTurns?: number; turnIndex?: number },
|
||||
): RetainRequest {
|
||||
const parsedSession = effectiveCtx?.sessionKey ? parseSessionKey(effectiveCtx.sessionKey) : {};
|
||||
const turnIndex = options?.turnIndex ?? nextDocumentSequence(effectiveCtx);
|
||||
const retentionScope = options?.retentionScope || 'turn';
|
||||
const documentBase = getSessionDocumentBase(effectiveCtx);
|
||||
const documentKind = retentionScope === 'window' ? 'window' : 'turn';
|
||||
const documentId = `${documentBase}:${documentKind}:${String(turnIndex).padStart(6, '0')}`;
|
||||
const channelId = effectiveCtx?.channelId || parsedSession.channel;
|
||||
const provider = effectiveCtx?.messageProvider || parsedSession.provider;
|
||||
const threadId = extractThreadId(channelId);
|
||||
|
||||
return {
|
||||
content: transcript,
|
||||
document_id: `${effectiveCtx?.sessionKey || 'session'}-${now}`,
|
||||
document_id: documentId,
|
||||
metadata: {
|
||||
retained_at: new Date(now).toISOString(),
|
||||
message_count: String(messageCount),
|
||||
source: pluginConfig.retainSource || 'openclaw',
|
||||
retention_scope: retentionScope,
|
||||
turn_index: String(turnIndex),
|
||||
session_key: effectiveCtx?.sessionKey,
|
||||
agent_id: effectiveCtx?.agentId || parsedSession.agentId,
|
||||
provider,
|
||||
channel_type: effectiveCtx?.messageProvider,
|
||||
channel_id: effectiveCtx?.channelId,
|
||||
channel_id: channelId,
|
||||
thread_id: threadId,
|
||||
sender_id: effectiveCtx?.senderId,
|
||||
...(options?.windowTurns !== undefined ? { window_turns: String(options.windowTurns) } : {}),
|
||||
},
|
||||
tags: pluginConfig.retainTags && pluginConfig.retainTags.length > 0 ? pluginConfig.retainTags : undefined,
|
||||
};
|
||||
|
|
|
|||
Loading…
Reference in a new issue