diff --git a/hindsight-integrations/openclaw/README.md b/hindsight-integrations/openclaw/README.md index 59a08a3e..4ce28410 100644 --- a/hindsight-integrations/openclaw/README.md +++ b/hindsight-integrations/openclaw/README.md @@ -51,7 +51,7 @@ Optional settings in `~/.openclaw/openclaw.json` under `plugins.entries.hindsigh | `retainTags` | `[]` | Tags applied to every retained document, useful for cross-agent/source labeling (e.g. `source_system:openclaw`, `agent:agentname`) | | `retainSource` | `"openclaw"` | `source` value written into retained document metadata | | `dynamicBankGranularity` | `["agent", "channel", "user"]` | Fields used to derive bank ID. Options: `agent`, `channel`, `user`, `provider` | -| `excludeProviders` | `[]` | Message providers to skip for recall/retain (e.g. `slack`, `telegram`, `discord`) | +| `excludeProviders` | `["heartbeat"]` | Message providers to skip for recall/retain (e.g. `heartbeat`, `slack`, `telegram`, `discord`) | | `autoRecall` | `true` | Auto-inject memories before each turn. Set to `false` when the agent has its own recall tool. | | `autoRetain` | `true` | Auto-retain conversations after each turn | | `retainRoles` | `["user", "assistant"]` | Which message roles to retain. Options: `user`, `assistant`, `system`, `tool` | @@ -68,6 +68,10 @@ Optional settings in `~/.openclaw/openclaw.json` under `plugins.entries.hindsigh | `hindsightApiUrl` | — | External Hindsight API URL (skips local daemon) | | `hindsightApiToken` | — | Auth token for external API | +## Retention details + +Retained documents use stable session-scoped IDs like `openclaw:agent:agentname:discord:channel:123:turn:000001` (or `...:window:000002` for chunked retention), and include richer metadata such as `session_key`, `agent_id`, `provider`, `channel_id`, `thread_id`, `sender_id`, `turn_index`, and `retention_scope`. + ## Documentation For full documentation, configuration options, troubleshooting, and development guide, see: diff --git a/hindsight-integrations/openclaw/openclaw.plugin.json b/hindsight-integrations/openclaw/openclaw.plugin.json index 1e5259bc..d1bd6003 100644 --- a/hindsight-integrations/openclaw/openclaw.plugin.json +++ b/hindsight-integrations/openclaw/openclaw.plugin.json @@ -98,7 +98,8 @@ "items": { "type": "string" }, - "description": "Message providers to exclude from recall and retain (e.g. ['telegram', 'discord'])" + "description": "Message providers to exclude from recall and retain (e.g. ['heartbeat', 'telegram', 'discord'])", + "default": ["heartbeat"] }, "dynamicBankGranularity": { "type": "array", @@ -320,7 +321,7 @@ }, "excludeProviders": { "label": "Excluded Providers", - "placeholder": "e.g. telegram, discord" + "placeholder": "e.g. heartbeat, telegram, discord" }, "dynamicBankGranularity": { "label": "Bank Granularity", diff --git a/hindsight-integrations/openclaw/src/index.test.ts b/hindsight-integrations/openclaw/src/index.test.ts index 3ff959c1..c1da6fde 100644 --- a/hindsight-integrations/openclaw/src/index.test.ts +++ b/hindsight-integrations/openclaw/src/index.test.ts @@ -229,6 +229,7 @@ describe('formatMemories', () => { describe('buildRetainRequest', () => { it('adds configured source metadata and retain tags', () => { const request = buildRetainRequest('hello world', 2, { + agentId: 'main', sessionKey: 'agent:main:main', messageProvider: 'discord', channelId: 'channel:123', @@ -240,21 +241,56 @@ describe('buildRetainRequest', () => { expect(request).toEqual({ content: 'hello world', - document_id: 'agent:main:main-1700000000000', + document_id: 'openclaw:agent:main:main:turn:000001', metadata: { retained_at: expect.any(String), message_count: '2', source: 'openclaw', + retention_scope: 'turn', + turn_index: '1', + session_key: 'agent:main:main', + agent_id: 'main', + provider: 'discord', channel_type: 'discord', channel_id: 'channel:123', + thread_id: undefined, sender_id: 'user:456', }, tags: ['source_system:openclaw', 'agent:agentname'], }); }); + it('uses window ids and metadata for chunked retention', () => { + const request = buildRetainRequest('hello world', 4, { + agentId: 'agentname', + sessionKey: 'agent:agentname:discord:group:123:topic:456', + messageProvider: 'discord', + senderId: 'user:456', + }, { + retainSource: 'openclaw', + }, 1700000000000, { + retentionScope: 'window', + turnIndex: 2, + windowTurns: 2, + }); + + expect(request.document_id).toBe('openclaw:agent:agentname:discord:group:123:topic:456:window:000002'); + expect(request.metadata).toMatchObject({ + source: 'openclaw', + retention_scope: 'window', + turn_index: '2', + agent_id: 'agentname', + provider: 'discord', + channel_type: 'discord', + channel_id: 'group:123:topic:456', + thread_id: '456', + sender_id: 'user:456', + window_turns: '2', + }); + }); + it('defaults source metadata to openclaw when unset', () => { - const request = buildRetainRequest('hello world', 1, {}, {}, 1700000000000); + const request = buildRetainRequest('hello world', 1, {}, {}, 1700000000000, { turnIndex: 1 }); expect(request.metadata?.source).toBe('openclaw'); expect(request.tags).toBeUndefined(); }); diff --git a/hindsight-integrations/openclaw/src/index.ts b/hindsight-integrations/openclaw/src/index.ts index 8e8d3443..66f57f39 100644 --- a/hindsight-integrations/openclaw/src/index.ts +++ b/hindsight-integrations/openclaw/src/index.ts @@ -43,10 +43,11 @@ const DEFAULT_RECALL_TIMEOUT_MS = 10_000; // Cache sender IDs discovered in before_prompt_build (where event.prompt has the metadata // blocks) so agent_end can look them up — event.messages in agent_end is clean history. const senderIdBySession = new Map(); +const documentSequenceBySession = new Map(); -// Guard against double hook registration on the same api instance -// Uses a WeakSet so each api instance can only register hooks once -const registeredApis = new WeakSet(); +// Guard against duplicate hook registration within a single runtime load. +// Do not tie this to api instance identity, which can be brittle across loader phases. +let hooksRegistered = false; // Cooldown + guard to prevent concurrent reinit attempts let lastReinitAttempt = 0; @@ -134,23 +135,23 @@ function formatCurrentTimeForRecall(date = new Date()): string { * Throttled to one attempt per 30s to avoid hammering a down service. * Only works if initialization was attempted at least once (isInitialized guard). */ -async function lazyReinit(): Promise { +async function lazyReinit(configOverride?: PluginConfig): Promise { const now = Date.now(); if (now - lastReinitAttempt < REINIT_COOLDOWN_MS || isReinitInProgress) { return; } - // Only attempt lazy reinit if we've already done initial setup - // (i.e., service.start() was called at least once) - if (!currentPluginConfig) { - debug('[Hindsight] lazyReinit skipped - no plugin config (service.start() not called yet)'); + const config = configOverride ?? currentPluginConfig; + if (!config) { + debug('[Hindsight] lazyReinit skipped - no plugin config available'); return; } + // Persist config if we only have it from the live hook registration path. + currentPluginConfig = config; + isReinitInProgress = true; lastReinitAttempt = now; - - const config = currentPluginConfig; const externalApi = detectExternalApi(config); if (!externalApi.apiUrl) { isReinitInProgress = false; @@ -200,6 +201,11 @@ if (typeof global !== 'undefined') { // If initPromise is null, it means service.start() hasn't been called yet // (CLI mode, not gateway mode). Hooks should gracefully no-op. if (!initPromise) { + if (currentPluginConfig) { + log.warn('waitForReady called before service.start() — attempting lazy initialization fallback'); + await lazyReinit(currentPluginConfig); + return; + } log.warn('waitForReady called before service.start() — hooks will no-op (expected in CLI mode)'); return; } @@ -813,7 +819,9 @@ function getPluginConfig(api: MoltbotPluginAPI): PluginConfig { bankIdPrefix: config.bankIdPrefix, retainTags: Array.isArray(config.retainTags) ? config.retainTags.filter((tag): tag is string => typeof tag === 'string') : undefined, retainSource: typeof config.retainSource === 'string' && config.retainSource.trim().length > 0 ? config.retainSource.trim() : undefined, - excludeProviders: Array.isArray(config.excludeProviders) ? config.excludeProviders : [], + excludeProviders: Array.isArray(config.excludeProviders) + ? Array.from(new Set(['heartbeat', ...config.excludeProviders.filter((provider): provider is string => typeof provider === 'string')])) + : ['heartbeat'], autoRecall: config.autoRecall !== false, // Default: true (on) — backward compatible dynamicBankGranularity: Array.isArray(config.dynamicBankGranularity) ? config.dynamicBankGranularity : undefined, autoRetain: config.autoRetain !== false, // Default: true @@ -839,6 +847,7 @@ function getPluginConfig(api: MoltbotPluginAPI): PluginConfig { export default function (api: MoltbotPluginAPI) { try { + log.info('plugin entry invoked'); debug('[Hindsight] Plugin loading...'); // Get plugin config first (needed for debug flag and service registration) @@ -863,9 +872,11 @@ export default function (api: MoltbotPluginAPI) { // happens in service.start() which is ONLY called on gateway start, // not on every CLI command. debug('[Hindsight] Registering service...'); + log.info('registering plugin service'); api.registerService({ id: 'hindsight-memory', async start() { + log.info('service.start invoked'); debug('[Hindsight] Service start called - beginning heavy initialization...'); // Detect LLM configuration (env vars > plugin config > auto-detect) @@ -1180,12 +1191,13 @@ export default function (api: MoltbotPluginAPI) { debug('[Hindsight] Plugin loaded successfully'); // Register agent hooks for auto-recall and auto-retention - if (registeredApis.has(api)) { - debug('[Hindsight] Hooks already registered for this api instance, skipping duplicate registration'); + if (hooksRegistered) { + debug('[Hindsight] Hooks already registered in this runtime, skipping duplicate hook registration'); return; } - registeredApis.add(api); + hooksRegistered = true; debug('[Hindsight] Registering agent hooks...'); + log.info('registering agent hooks'); // Auto-recall: Inject relevant memories before agent processes the message // Hook signature: (event, ctx) where event has {prompt, messages?} and ctx has agent context @@ -1433,7 +1445,17 @@ ${memoriesFormatted} const retainNow = Date.now(); - const retainRequest = buildRetainRequest(transcript, messageCount, effectiveCtx, pluginConfig, retainNow); + const retainRequest = buildRetainRequest( + transcript, + messageCount, + effectiveCtxForRetain, + pluginConfig, + retainNow, + { + retentionScope: retainFullWindow ? 'window' : 'turn', + windowTurns: retainFullWindow ? (pluginConfig.retainEveryNTurns ?? 1) + (pluginConfig.retainOverlapTurns ?? 0) : undefined, + }, + ); // Retain to Hindsight debug(`[Hindsight] Retaining to bank ${bankId}, document: ${retainRequest.document_id}, chars: ${transcript.length}\n---\n${transcript.substring(0, 500)}${transcript.length > 500 ? '\n...(truncated)' : ''}\n---`); @@ -1462,6 +1484,7 @@ ${memoriesFormatted} } }); debug('[Hindsight] Hooks registered'); + log.info('agent hooks registered'); } catch (error) { log.error('plugin loading error', error); if (error instanceof Error) { @@ -1473,23 +1496,74 @@ ${memoriesFormatted} // Export client getter for tools +function sanitizeDocumentIdPart(value: string | undefined, fallback: string): string { + const normalized = (value || '').trim(); + if (!normalized) return fallback; + return normalized + .replace(/[^a-zA-Z0-9:_-]+/g, '_') + .replace(/_+/g, '_') + .replace(/^_+|_+$/g, '') || fallback; +} + +function getSessionDocumentBase(effectiveCtx: PluginHookAgentContext | undefined): string { + const sessionKeyPart = sanitizeDocumentIdPart(effectiveCtx?.sessionKey, 'session'); + return `openclaw:${sessionKeyPart}`; +} + +function nextDocumentSequence(effectiveCtx: PluginHookAgentContext | undefined): number { + const sequenceKey = effectiveCtx?.sessionKey || 'session'; + const next = (documentSequenceBySession.get(sequenceKey) || 0) + 1; + documentSequenceBySession.set(sequenceKey, next); + if (documentSequenceBySession.size > MAX_TRACKED_SESSIONS) { + const oldestKey = documentSequenceBySession.keys().next().value; + if (oldestKey) { + documentSequenceBySession.delete(oldestKey); + } + } + return next; +} + +function extractThreadId(channelId: string | undefined): string | undefined { + if (!channelId) return undefined; + const match = channelId.match(/(?:^|:)topic:([^:]+)$/); + return match?.[1]; +} + export function buildRetainRequest( transcript: string, messageCount: number, effectiveCtx: PluginHookAgentContext | undefined, pluginConfig: PluginConfig, now = Date.now(), + options?: { retentionScope?: 'turn' | 'window' | 'manual'; windowTurns?: number; turnIndex?: number }, ): RetainRequest { + const parsedSession = effectiveCtx?.sessionKey ? parseSessionKey(effectiveCtx.sessionKey) : {}; + const turnIndex = options?.turnIndex ?? nextDocumentSequence(effectiveCtx); + const retentionScope = options?.retentionScope || 'turn'; + const documentBase = getSessionDocumentBase(effectiveCtx); + const documentKind = retentionScope === 'window' ? 'window' : 'turn'; + const documentId = `${documentBase}:${documentKind}:${String(turnIndex).padStart(6, '0')}`; + const channelId = effectiveCtx?.channelId || parsedSession.channel; + const provider = effectiveCtx?.messageProvider || parsedSession.provider; + const threadId = extractThreadId(channelId); + return { content: transcript, - document_id: `${effectiveCtx?.sessionKey || 'session'}-${now}`, + document_id: documentId, metadata: { retained_at: new Date(now).toISOString(), message_count: String(messageCount), source: pluginConfig.retainSource || 'openclaw', + retention_scope: retentionScope, + turn_index: String(turnIndex), + session_key: effectiveCtx?.sessionKey, + agent_id: effectiveCtx?.agentId || parsedSession.agentId, + provider, channel_type: effectiveCtx?.messageProvider, - channel_id: effectiveCtx?.channelId, + channel_id: channelId, + thread_id: threadId, sender_id: effectiveCtx?.senderId, + ...(options?.windowTurns !== undefined ? { window_turns: String(options.windowTurns) } : {}), }, tags: pluginConfig.retainTags && pluginConfig.retainTags.length > 0 ? pluginConfig.retainTags : undefined, };