feat(openclaw): add resilient startup and richer retain metadata (#942)

* feat(openclaw): enrich retain metadata and ignore heartbeat by default

* docs(openclaw): move retain metadata note out of config table

* fix(openclaw): make hook registration runtime-idempotent

* fix(openclaw): lazily initialize when service start is skipped

---------

Co-authored-by: Aldous <aldous@agentmail.to>
Co-authored-by: Josh <joshdwells@gmail.com>
Co-authored-by: Aldous the Orchestrator <Aldoustheorchestrator@users.noreply.github.com>

hindsight-integrations/openclaw/README.md

@@ -51,7 +51,7 @@ Optional settings in `~/.openclaw/openclaw.json` under `plugins.entries.hindsigh
 | `retainTags` | `[]` | Tags applied to every retained document, useful for cross-agent/source labeling (e.g. `source_system:openclaw`, `agent:agentname`) |
 | `retainSource` | `"openclaw"` | `source` value written into retained document metadata |
 | `dynamicBankGranularity` | `["agent", "channel", "user"]` | Fields used to derive bank ID. Options: `agent`, `channel`, `user`, `provider` |
-| `excludeProviders` | `[]` | Message providers to skip for recall/retain (e.g. `slack`, `telegram`, `discord`) |
+| `excludeProviders` | `["heartbeat"]` | Message providers to skip for recall/retain (e.g. `heartbeat`, `slack`, `telegram`, `discord`) |
 | `autoRecall` | `true` | Auto-inject memories before each turn. Set to `false` when the agent has its own recall tool. |
 | `autoRetain` | `true` | Auto-retain conversations after each turn |
 | `retainRoles` | `["user", "assistant"]` | Which message roles to retain. Options: `user`, `assistant`, `system`, `tool` |
@@ -68,6 +68,10 @@ Optional settings in `~/.openclaw/openclaw.json` under `plugins.entries.hindsigh
 | `hindsightApiUrl` | — | External Hindsight API URL (skips local daemon) |
 | `hindsightApiToken` | — | Auth token for external API |
 
+## Retention details
+
+Retained documents use stable session-scoped IDs like `openclaw:agent:agentname:discord:channel:123:turn:000001` (or `...:window:000002` for chunked retention), and include richer metadata such as `session_key`, `agent_id`, `provider`, `channel_id`, `thread_id`, `sender_id`, `turn_index`, and `retention_scope`.
+
 ## Documentation
 
 For full documentation, configuration options, troubleshooting, and development guide, see:

hindsight-integrations/openclaw/openclaw.plugin.json

@@ -98,7 +98,8 @@
         "items": {
           "type": "string"
         },
-        "description": "Message providers to exclude from recall and retain (e.g. ['telegram', 'discord'])"
+        "description": "Message providers to exclude from recall and retain (e.g. ['heartbeat', 'telegram', 'discord'])",
+        "default": ["heartbeat"]
       },
       "dynamicBankGranularity": {
         "type": "array",
@@ -320,7 +321,7 @@
     },
     "excludeProviders": {
       "label": "Excluded Providers",
-      "placeholder": "e.g. telegram, discord"
+      "placeholder": "e.g. heartbeat, telegram, discord"
     },
     "dynamicBankGranularity": {
       "label": "Bank Granularity",

hindsight-integrations/openclaw/src/index.test.ts

@@ -229,6 +229,7 @@ describe('formatMemories', () => {
 describe('buildRetainRequest', () => {
   it('adds configured source metadata and retain tags', () => {
     const request = buildRetainRequest('hello world', 2, {
+      agentId: 'main',
       sessionKey: 'agent:main:main',
       messageProvider: 'discord',
       channelId: 'channel:123',
@@ -240,21 +241,56 @@ describe('buildRetainRequest', () => {
 
     expect(request).toEqual({
       content: 'hello world',
-      document_id: 'agent:main:main-1700000000000',
+      document_id: 'openclaw:agent:main:main:turn:000001',
       metadata: {
         retained_at: expect.any(String),
         message_count: '2',
         source: 'openclaw',
+        retention_scope: 'turn',
+        turn_index: '1',
+        session_key: 'agent:main:main',
+        agent_id: 'main',
+        provider: 'discord',
         channel_type: 'discord',
         channel_id: 'channel:123',
+        thread_id: undefined,
         sender_id: 'user:456',
       },
       tags: ['source_system:openclaw', 'agent:agentname'],
     });
   });
 
+  it('uses window ids and metadata for chunked retention', () => {
+    const request = buildRetainRequest('hello world', 4, {
+      agentId: 'agentname',
+      sessionKey: 'agent:agentname:discord:group:123:topic:456',
+      messageProvider: 'discord',
+      senderId: 'user:456',
+    }, {
+      retainSource: 'openclaw',
+    }, 1700000000000, {
+      retentionScope: 'window',
+      turnIndex: 2,
+      windowTurns: 2,
+    });
+
+    expect(request.document_id).toBe('openclaw:agent:agentname:discord:group:123:topic:456:window:000002');
+    expect(request.metadata).toMatchObject({
+      source: 'openclaw',
+      retention_scope: 'window',
+      turn_index: '2',
+      agent_id: 'agentname',
+      provider: 'discord',
+      channel_type: 'discord',
+      channel_id: 'group:123:topic:456',
+      thread_id: '456',
+      sender_id: 'user:456',
+      window_turns: '2',
+    });
+  });
+
   it('defaults source metadata to openclaw when unset', () => {
-    const request = buildRetainRequest('hello world', 1, {}, {}, 1700000000000);
+    const request = buildRetainRequest('hello world', 1, {}, {}, 1700000000000, { turnIndex: 1 });
     expect(request.metadata?.source).toBe('openclaw');
     expect(request.tags).toBeUndefined();
   });

hindsight-integrations/openclaw/src/index.ts

@@ -43,10 +43,11 @@ const DEFAULT_RECALL_TIMEOUT_MS = 10_000;
 // Cache sender IDs discovered in before_prompt_build (where event.prompt has the metadata
 // blocks) so agent_end can look them up — event.messages in agent_end is clean history.
 const senderIdBySession = new Map<string, string>();
+const documentSequenceBySession = new Map<string, number>();
 
-// Guard against double hook registration on the same api instance
+// Guard against duplicate hook registration within a single runtime load.
-// Uses a WeakSet so each api instance can only register hooks once
+// Do not tie this to api instance identity, which can be brittle across loader phases.
-const registeredApis = new WeakSet<object>();
+let hooksRegistered = false;
 
 // Cooldown + guard to prevent concurrent reinit attempts
 let lastReinitAttempt = 0;
@@ -134,23 +135,23 @@ function formatCurrentTimeForRecall(date = new Date()): string {
  * Throttled to one attempt per 30s to avoid hammering a down service.
  * Only works if initialization was attempted at least once (isInitialized guard).
  */
-async function lazyReinit(): Promise<void> {
+async function lazyReinit(configOverride?: PluginConfig): Promise<void> {
   const now = Date.now();
   if (now - lastReinitAttempt < REINIT_COOLDOWN_MS || isReinitInProgress) {
     return;
   }
 
-  // Only attempt lazy reinit if we've already done initial setup
+  const config = configOverride ?? currentPluginConfig;
-  // (i.e., service.start() was called at least once)
+  if (!config) {
-  if (!currentPluginConfig) {
+    debug('[Hindsight] lazyReinit skipped - no plugin config available');
-    debug('[Hindsight] lazyReinit skipped - no plugin config (service.start() not called yet)');
     return;
   }
 
+  // Persist config if we only have it from the live hook registration path.
+  currentPluginConfig = config;
+
   isReinitInProgress = true;
   lastReinitAttempt = now;
-
-  const config = currentPluginConfig;
   const externalApi = detectExternalApi(config);
   if (!externalApi.apiUrl) {
     isReinitInProgress = false;
@@ -200,6 +201,11 @@ if (typeof global !== 'undefined') {
       // If initPromise is null, it means service.start() hasn't been called yet
       // (CLI mode, not gateway mode). Hooks should gracefully no-op.
       if (!initPromise) {
+        if (currentPluginConfig) {
+          log.warn('waitForReady called before service.start() — attempting lazy initialization fallback');
+          await lazyReinit(currentPluginConfig);
+          return;
+        }
         log.warn('waitForReady called before service.start() — hooks will no-op (expected in CLI mode)');
         return;
       }
@@ -813,7 +819,9 @@ function getPluginConfig(api: MoltbotPluginAPI): PluginConfig {
     bankIdPrefix: config.bankIdPrefix,
     retainTags: Array.isArray(config.retainTags) ? config.retainTags.filter((tag): tag is string => typeof tag === 'string') : undefined,
     retainSource: typeof config.retainSource === 'string' && config.retainSource.trim().length > 0 ? config.retainSource.trim() : undefined,
-    excludeProviders: Array.isArray(config.excludeProviders) ? config.excludeProviders : [],
+    excludeProviders: Array.isArray(config.excludeProviders)
+      ? Array.from(new Set(['heartbeat', ...config.excludeProviders.filter((provider): provider is string => typeof provider === 'string')]))
+      : ['heartbeat'],
     autoRecall: config.autoRecall !== false, // Default: true (on) — backward compatible
     dynamicBankGranularity: Array.isArray(config.dynamicBankGranularity) ? config.dynamicBankGranularity : undefined,
     autoRetain: config.autoRetain !== false, // Default: true
@@ -839,6 +847,7 @@ function getPluginConfig(api: MoltbotPluginAPI): PluginConfig {
 
 export default function (api: MoltbotPluginAPI) {
   try {
+    log.info('plugin entry invoked');
     debug('[Hindsight] Plugin loading...');
 
     // Get plugin config first (needed for debug flag and service registration)
@@ -863,9 +872,11 @@ export default function (api: MoltbotPluginAPI) {
     // happens in service.start() which is ONLY called on gateway start,
     // not on every CLI command.
     debug('[Hindsight] Registering service...');
+    log.info('registering plugin service');
     api.registerService({
       id: 'hindsight-memory',
       async start() {
+        log.info('service.start invoked');
         debug('[Hindsight] Service start called - beginning heavy initialization...');
 
         // Detect LLM configuration (env vars > plugin config > auto-detect)
@@ -1180,12 +1191,13 @@ export default function (api: MoltbotPluginAPI) {
     debug('[Hindsight] Plugin loaded successfully');
 
     // Register agent hooks for auto-recall and auto-retention
-    if (registeredApis.has(api)) {
+    if (hooksRegistered) {
-      debug('[Hindsight] Hooks already registered for this api instance, skipping duplicate registration');
+      debug('[Hindsight] Hooks already registered in this runtime, skipping duplicate hook registration');
       return;
     }
-    registeredApis.add(api);
+    hooksRegistered = true;
     debug('[Hindsight] Registering agent hooks...');
+    log.info('registering agent hooks');
 
     // Auto-recall: Inject relevant memories before agent processes the message
     // Hook signature: (event, ctx) where event has {prompt, messages?} and ctx has agent context
@@ -1433,7 +1445,17 @@ ${memoriesFormatted}
 
 
         const retainNow = Date.now();
-        const retainRequest = buildRetainRequest(transcript, messageCount, effectiveCtx, pluginConfig, retainNow);
+        const retainRequest = buildRetainRequest(
+          transcript,
+          messageCount,
+          effectiveCtxForRetain,
+          pluginConfig,
+          retainNow,
+          {
+            retentionScope: retainFullWindow ? 'window' : 'turn',
+            windowTurns: retainFullWindow ? (pluginConfig.retainEveryNTurns ?? 1) + (pluginConfig.retainOverlapTurns ?? 0) : undefined,
+          },
+        );
 
         // Retain to Hindsight
         debug(`[Hindsight] Retaining to bank ${bankId}, document: ${retainRequest.document_id}, chars: ${transcript.length}\n---\n${transcript.substring(0, 500)}${transcript.length > 500 ? '\n...(truncated)' : ''}\n---`);
@@ -1462,6 +1484,7 @@ ${memoriesFormatted}
       }
     });
     debug('[Hindsight] Hooks registered');
+    log.info('agent hooks registered');
   } catch (error) {
     log.error('plugin loading error', error);
     if (error instanceof Error) {
@@ -1473,23 +1496,74 @@ ${memoriesFormatted}
 
 // Export client getter for tools
 
+function sanitizeDocumentIdPart(value: string | undefined, fallback: string): string {
+  const normalized = (value || '').trim();
+  if (!normalized) return fallback;
+  return normalized
+    .replace(/[^a-zA-Z0-9:_-]+/g, '_')
+    .replace(/_+/g, '_')
+    .replace(/^_+|_+$/g, '') || fallback;
+}
+
+function getSessionDocumentBase(effectiveCtx: PluginHookAgentContext | undefined): string {
+  const sessionKeyPart = sanitizeDocumentIdPart(effectiveCtx?.sessionKey, 'session');
+  return `openclaw:${sessionKeyPart}`;
+}
+
+function nextDocumentSequence(effectiveCtx: PluginHookAgentContext | undefined): number {
+  const sequenceKey = effectiveCtx?.sessionKey || 'session';
+  const next = (documentSequenceBySession.get(sequenceKey) || 0) + 1;
+  documentSequenceBySession.set(sequenceKey, next);
+  if (documentSequenceBySession.size > MAX_TRACKED_SESSIONS) {
+    const oldestKey = documentSequenceBySession.keys().next().value;
+    if (oldestKey) {
+      documentSequenceBySession.delete(oldestKey);
+    }
+  }
+  return next;
+}
+
+function extractThreadId(channelId: string | undefined): string | undefined {
+  if (!channelId) return undefined;
+  const match = channelId.match(/(?:^|:)topic:([^:]+)$/);
+  return match?.[1];
+}
+
 export function buildRetainRequest(
   transcript: string,
   messageCount: number,
   effectiveCtx: PluginHookAgentContext | undefined,
   pluginConfig: PluginConfig,
   now = Date.now(),
+  options?: { retentionScope?: 'turn' | 'window' | 'manual'; windowTurns?: number; turnIndex?: number },
 ): RetainRequest {
+  const parsedSession = effectiveCtx?.sessionKey ? parseSessionKey(effectiveCtx.sessionKey) : {};
+  const turnIndex = options?.turnIndex ?? nextDocumentSequence(effectiveCtx);
+  const retentionScope = options?.retentionScope || 'turn';
+  const documentBase = getSessionDocumentBase(effectiveCtx);
+  const documentKind = retentionScope === 'window' ? 'window' : 'turn';
+  const documentId = `${documentBase}:${documentKind}:${String(turnIndex).padStart(6, '0')}`;
+  const channelId = effectiveCtx?.channelId || parsedSession.channel;
+  const provider = effectiveCtx?.messageProvider || parsedSession.provider;
+  const threadId = extractThreadId(channelId);
+
   return {
     content: transcript,
-    document_id: `${effectiveCtx?.sessionKey || 'session'}-${now}`,
+    document_id: documentId,
     metadata: {
       retained_at: new Date(now).toISOString(),
       message_count: String(messageCount),
       source: pluginConfig.retainSource || 'openclaw',
+      retention_scope: retentionScope,
+      turn_index: String(turnIndex),
+      session_key: effectiveCtx?.sessionKey,
+      agent_id: effectiveCtx?.agentId || parsedSession.agentId,
+      provider,
       channel_type: effectiveCtx?.messageProvider,
-      channel_id: effectiveCtx?.channelId,
+      channel_id: channelId,
+      thread_id: threadId,
       sender_id: effectiveCtx?.senderId,
+      ...(options?.windowTurns !== undefined ? { window_turns: String(options.windowTurns) } : {}),
     },
     tags: pluginConfig.retainTags && pluginConfig.retainTags.length > 0 ? pluginConfig.retainTags : undefined,
   };