From 0b17a67c70b33dfbb0f859a7047c1be07a652f22 Mon Sep 17 00:00:00 2001 From: Ben <62250174+benfrank241@users.noreply.github.com> Date: Mon, 30 Mar 2026 04:51:53 -0400 Subject: [PATCH] feat: add Hindsight memory integration for OpenAI Codex CLI (#730) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat(codex): add Hindsight memory integration for OpenAI Codex CLI Hooks-based integration that gives Codex CLI long-term memory via Hindsight. Three hooks keep memory in sync: SessionStart (daemon pre-warm), UserPromptSubmit (recall + context injection), Stop (retain conversation to memory). Key differences from the Claude Code integration: - Codex transcript format: JSONL with {msg: {type, message}} (user_message/agent_message) - No CODEX_PLUGIN_ROOT env var — install.sh writes hooks.json with absolute paths - State stored in ~/.hindsight/codex/state/ (not CLAUDE_PLUGIN_DATA) - No async: true in hooks (not supported by Codex) - No SessionEnd event - hooks.json written to ~/.codex/hooks.json with codex_hooks = true in config.toml * fix(codex): fix transcript parser for actual Codex disk format Codex stores sessions as rollout-*.jsonl with response_item entries: User: {type:response_item, payload:{type:message, role:user, content:[{type:input_text, text:...}]}} Assistant: {type:response_item, payload:{type:message, role:assistant, phase:final_answer, content:[{type:output_text, text:...}]}} Previous parser expected an undocumented {msg:{type:user_message}} format from the Rust protocol spec that does not match the actual on-disk storage format. * feat(codex): add reflect mode to UserPromptSubmit hook Add recallMode config option (default: 'recall') that switches the UserPromptSubmit hook between: - 'recall': existing behavior, fast raw facts list - 'reflect': agentic synthesis loop, returns coherent prose answer Also adds reflect() method to HindsightClient and HINDSIGHT_RECALL_MODE env var override. Reflect uses a 25s timeout (vs 10s for recall). * feat(codex): auto mode for recall/reflect selection Add recallMode: 'auto' (new default) that picks the operation per-query: - Synthesis patterns (what do you know, what's my, summarize, etc.) → reflect - All other prompts → recall (fast, raw facts, better for code tasks) * feat(codex): add automated test suite and finalize recall-only mode * docs(codex): add docs page and sidebar entry for Codex CLI integration --- .../docs/sdks/integrations/codex.md | 189 +++++++++ hindsight-docs/sidebars.ts | 6 + hindsight-integrations/codex/README.md | 127 ++++++ hindsight-integrations/codex/hooks/hooks.json | 37 ++ hindsight-integrations/codex/install.sh | 149 +++++++ .../codex/scripts/lib/__init__.py | 0 .../codex/scripts/lib/bank.py | 91 +++++ .../codex/scripts/lib/client.py | 163 ++++++++ .../codex/scripts/lib/config.py | 143 +++++++ .../codex/scripts/lib/content.py | 318 +++++++++++++++ .../codex/scripts/lib/daemon.py | 275 +++++++++++++ .../codex/scripts/lib/llm.py | 146 +++++++ .../codex/scripts/lib/state.py | 113 ++++++ .../codex/scripts/recall.py | 165 ++++++++ .../codex/scripts/retain.py | 186 +++++++++ .../codex/scripts/session_start.py | 53 +++ hindsight-integrations/codex/settings.json | 35 ++ .../codex/tests/conftest.py | 90 +++++ .../codex/tests/test_content.py | 364 ++++++++++++++++++ .../codex/tests/test_hooks.py | 319 +++++++++++++++ 20 files changed, 2969 insertions(+) create mode 100644 hindsight-docs/docs/sdks/integrations/codex.md create mode 100644 hindsight-integrations/codex/README.md create mode 100644 hindsight-integrations/codex/hooks/hooks.json create mode 100755 hindsight-integrations/codex/install.sh create mode 100644 hindsight-integrations/codex/scripts/lib/__init__.py create mode 100644 hindsight-integrations/codex/scripts/lib/bank.py create mode 100644 hindsight-integrations/codex/scripts/lib/client.py create mode 100644 hindsight-integrations/codex/scripts/lib/config.py create mode 100644 hindsight-integrations/codex/scripts/lib/content.py create mode 100644 hindsight-integrations/codex/scripts/lib/daemon.py create mode 100644 hindsight-integrations/codex/scripts/lib/llm.py create mode 100644 hindsight-integrations/codex/scripts/lib/state.py create mode 100755 hindsight-integrations/codex/scripts/recall.py create mode 100755 hindsight-integrations/codex/scripts/retain.py create mode 100755 hindsight-integrations/codex/scripts/session_start.py create mode 100644 hindsight-integrations/codex/settings.json create mode 100644 hindsight-integrations/codex/tests/conftest.py create mode 100644 hindsight-integrations/codex/tests/test_content.py create mode 100644 hindsight-integrations/codex/tests/test_hooks.py diff --git a/hindsight-docs/docs/sdks/integrations/codex.md b/hindsight-docs/docs/sdks/integrations/codex.md new file mode 100644 index 00000000..edb3b697 --- /dev/null +++ b/hindsight-docs/docs/sdks/integrations/codex.md @@ -0,0 +1,189 @@ +--- +sidebar_position: 6 +--- + +# OpenAI Codex CLI + +Persistent memory for [OpenAI Codex CLI](https://github.com/openai/codex) using [Hindsight](https://vectorize.io/hindsight). Three Python hook scripts automatically recall relevant context before each prompt and retain conversations after each turn — no changes to your Codex workflow required. + +## Quick Start + +```bash +# 1. Clone the Hindsight repo and install the plugin +git clone https://github.com/vectorize-io/hindsight.git +cd hindsight/hindsight-integrations/codex +./install.sh + +# 2. Configure your Hindsight connection +cat > ~/.hindsight/codex.json << 'EOF' +{ + "hindsightApiUrl": "https://api.hindsight.vectorize.io", + "hindsightApiToken": "hsk_your_token_here", + "bankId": "codex" +} +EOF + +# 3. Start Codex — memory is live +codex +``` + +For a local Hindsight instance, set `hindsightApiUrl` to `http://localhost:9077` and omit `hindsightApiToken`. + +## Features + +- **Auto-recall** — on every user prompt, queries Hindsight for relevant memories and injects them as `additionalContext` (invisible to the transcript, visible to Codex) +- **Auto-retain** — after each Codex response, stores the conversation transcript to Hindsight for future recall +- **Dynamic bank IDs** — supports per-project memory isolation based on the working directory +- **Session-level upsert** — uses the session ID as the document ID so re-running the same session updates rather than duplicates stored content +- **Zero dependencies** — pure Python stdlib, no pip install required + +## Architecture + +The plugin uses three Codex hook events: + +| Hook | Event | Purpose | +|------|-------|---------| +| `session_start.py` | `SessionStart` | Warm up — verify Hindsight is reachable | +| `recall.py` | `UserPromptSubmit` | **Auto-recall** — query memories, inject as `additionalContext` | +| `retain.py` | `Stop` | **Auto-retain** — extract transcript, POST to Hindsight (async) | + +On `UserPromptSubmit`, the hook reads the prompt, queries Hindsight for the most relevant memories, and outputs a `hookSpecificOutput.additionalContext` block. Codex prepends this to the conversation before sending it to the model: + +``` + +Relevant memories from past conversations... +Current time - 2026-03-27 09:14 + +- Project uses FastAPI with asyncpg — not SQLAlchemy [world] (2026-03-26) +- Preferred testing framework: pytest with pytest-asyncio [experience] (2026-03-26) + +``` + +On `Stop`, the hook reads the session transcript, strips previously injected memory tags (to prevent feedback loops), and POSTs the conversation to Hindsight asynchronously. + +## Connection Modes + +### 1. External API (recommended) + +Connect to a running Hindsight server (cloud or self-hosted): + +```json +{ + "hindsightApiUrl": "https://api.hindsight.vectorize.io", + "hindsightApiToken": "hsk_your_token" +} +``` + +### 2. Local Daemon + +Run `hindsight-embed` locally. The `session_start.py` hook will detect it on `apiPort` (default `9077`). The daemon is not auto-started by the Codex plugin — start it separately: + +```bash +uvx hindsight-embed +``` + +Then leave `hindsightApiUrl` empty in your config and the plugin will connect to `http://localhost:9077`. + +## Configuration + +Settings are loaded from `~/.hindsight/codex.json`. Every setting can also be overridden via environment variable. + +**Loading order** (later entries win): + +1. Built-in defaults +2. Plugin `settings.json` (at `~/.hindsight/codex/settings.json`) +3. User config (`~/.hindsight/codex.json`) +4. Environment variables + +--- + +### Connection + +| Setting | Env Var | Default | Description | +|---------|---------|---------|-------------| +| `hindsightApiUrl` | `HINDSIGHT_API_URL` | `""` | URL of the Hindsight API server. Required. | +| `hindsightApiToken` | `HINDSIGHT_API_TOKEN` | `null` | API token for authentication. Required for Hindsight Cloud. | +| `apiPort` | `HINDSIGHT_API_PORT` | `9077` | Port for the local `hindsight-embed` daemon. | + +--- + +### Memory Bank + +| Setting | Env Var | Default | Description | +|---------|---------|---------|-------------| +| `bankId` | `HINDSIGHT_BANK_ID` | `"codex"` | The bank to read from and write to. All sessions share this bank unless `dynamicBankId` is enabled. | +| `bankMission` | `HINDSIGHT_BANK_MISSION` | coding assistant prompt | Describes the agent's purpose. Sent when creating or updating the bank. | +| `retainMission` | — | extraction prompt | Instructions for Hindsight's fact extraction — what to extract from coding conversations. | +| `dynamicBankId` | `HINDSIGHT_DYNAMIC_BANK_ID` | `false` | When `true`, derives a unique bank ID from `dynamicBankGranularity` fields — useful for per-project isolation. | +| `dynamicBankGranularity` | — | `["agent", "project"]` | Which fields to combine for dynamic bank IDs. `"project"` = working directory, `"agent"` = agent name. | +| `bankIdPrefix` | — | `""` | Prefix prepended to all bank IDs. | +| `agentName` | `HINDSIGHT_AGENT_NAME` | `"codex"` | Agent name used in dynamic bank ID derivation. | + +--- + +### Auto-Recall + +| Setting | Env Var | Default | Description | +|---------|---------|---------|-------------| +| `autoRecall` | `HINDSIGHT_AUTO_RECALL` | `true` | Master switch for auto-recall. | +| `recallBudget` | `HINDSIGHT_RECALL_BUDGET` | `"mid"` | Search depth: `"low"` (fast), `"mid"` (balanced), `"high"` (thorough). | +| `recallMaxTokens` | `HINDSIGHT_RECALL_MAX_TOKENS` | `1024` | Max tokens in the recalled memory block. | +| `recallTypes` | — | `["world", "experience"]` | Memory types to retrieve. | +| `recallContextTurns` | `HINDSIGHT_RECALL_CONTEXT_TURNS` | `1` | Prior turns to include when building the recall query. `1` = latest prompt only. | +| `recallMaxQueryChars` | `HINDSIGHT_RECALL_MAX_QUERY_CHARS` | `800` | Max characters in the query sent to Hindsight. | +| `recallRoles` | — | `["user", "assistant"]` | Which roles to include when building a multi-turn query. | +| `recallPromptPreamble` | — | built-in | Text placed above the recalled memories in the injected context block. | + +--- + +### Auto-Retain + +| Setting | Env Var | Default | Description | +|---------|---------|---------|-------------| +| `autoRetain` | `HINDSIGHT_AUTO_RETAIN` | `true` | Master switch for auto-retain. | +| `retainMode` | `HINDSIGHT_RETAIN_MODE` | `"full-session"` | `"full-session"` sends the full transcript per session (upserted by session ID). `"chunked"` sends sliding windows every N turns. | +| `retainEveryNTurns` | — | `10` | Retain fires every N turns. `1` = every turn. Higher values reduce API calls. | +| `retainOverlapTurns` | — | `2` | Extra turns included from the previous chunk (chunked mode only). | +| `retainRoles` | — | `["user", "assistant"]` | Which roles to include in the retained transcript. | +| `retainTags` | — | `["{session_id}"]` | Tags attached to the stored document. `{session_id}` is replaced at runtime. | +| `retainMetadata` | — | `{}` | Arbitrary key-value metadata attached to the stored document. | +| `retainContext` | — | `"codex"` | Label identifying the source integration. Useful when multiple integrations write to the same bank. | + +--- + +### Debug + +| Setting | Env Var | Default | Description | +|---------|---------|---------|-------------| +| `debug` | `HINDSIGHT_DEBUG` | `false` | Enable verbose logging to stderr. All log lines are prefixed with `[Hindsight]`. | + +## Per-Project Memory + +To give each project its own isolated memory bank, enable dynamic bank IDs: + +```json +{ + "dynamicBankId": true, + "dynamicBankGranularity": ["agent", "project"] +} +``` + +With this config, running Codex in `~/projects/api` and `~/projects/frontend` stores and recalls memories separately. Bank IDs are derived from the working directory path. + +## Troubleshooting + +**Hooks not firing**: Check that `~/.codex/config.toml` contains `codex_hooks = true` under `[features]`. Re-run `install.sh` to write this automatically. + +**No memories recalled**: Recall returns results only after something has been retained. Either complete one Codex session first, or seed your bank manually using the [cookbook example](https://github.com/vectorize-io/hindsight-cookbook/tree/main/applications/codex-memory). + +**Memory not being stored**: `retainEveryNTurns` defaults to `10` — retain only fires every 10 turns. While testing, add `"retainEveryNTurns": 1` to `~/.hindsight/codex.json`. + +**Debug mode**: Add `"debug": true` to `~/.hindsight/codex.json` to see what Hindsight is doing on each turn: + +``` +[Hindsight] Recalling from bank 'codex', query length: 42 +[Hindsight] Injecting 3 memories +[Hindsight] Retaining to bank 'codex', doc 'sess-abc123', 2 messages, 847 chars +``` + +**High latency on recall**: Use `"recallBudget": "low"` or reduce `recallMaxTokens` to speed up recall queries. diff --git a/hindsight-docs/sidebars.ts b/hindsight-docs/sidebars.ts index 07ca4441..2423c824 100644 --- a/hindsight-docs/sidebars.ts +++ b/hindsight-docs/sidebars.ts @@ -190,6 +190,12 @@ const sidebars: SidebarsConfig = { label: 'Claude Code', customProps: { icon: '/img/icons/claudecode.svg' }, }, + { + type: 'doc', + id: 'sdks/integrations/codex', + label: 'OpenAI Codex CLI', + customProps: { icon: '/img/icons/terminal.svg' }, + }, { type: 'doc', id: 'sdks/integrations/openclaw', diff --git a/hindsight-integrations/codex/README.md b/hindsight-integrations/codex/README.md new file mode 100644 index 00000000..2efe8578 --- /dev/null +++ b/hindsight-integrations/codex/README.md @@ -0,0 +1,127 @@ +# Hindsight for OpenAI Codex CLI + +Long-term memory for [OpenAI Codex CLI](https://github.com/openai/codex) — remembers your projects, preferences, and past sessions across every conversation. + +## How it works + +Three Codex hooks keep memory in sync automatically: + +| Hook | Action | +|------|--------| +| `SessionStart` | Warms up the Hindsight server in the background | +| `UserPromptSubmit` | Recalls relevant memories and injects them into context | +| `Stop` | Retains the conversation to long-term memory | + +## Requirements + +- **OpenAI Codex CLI** v0.116.0 or later (hooks support) +- **Python 3.9+** (for hook scripts) +- **Hindsight**: [Hindsight Cloud](https://hindsight.vectorize.io) or local `hindsight-embed` + +## Installation + +```bash +git clone https://github.com/vectorize-io/hindsight +cd hindsight/hindsight-integrations/codex +./install.sh +``` + +The installer: +1. Copies scripts to `~/.hindsight/codex/scripts/` +2. Writes `~/.codex/hooks.json` with absolute paths to the scripts +3. Adds `codex_hooks = true` to `~/.codex/config.toml` + +### Uninstall + +```bash +./install.sh --uninstall +``` + +## Configuration + +The default config is written to `~/.hindsight/codex/settings.json` on first install. + +For personal overrides (stable across updates), create `~/.hindsight/codex.json`: + +```json +{ + "hindsightApiUrl": "https://api.hindsight.vectorize.io", + "hindsightApiToken": "your-api-key", + "bankId": "my-codex-memory" +} +``` + +### Hindsight Cloud + +```json +{ + "hindsightApiUrl": "https://api.hindsight.vectorize.io", + "hindsightApiToken": "your-api-key" +} +``` + +### Local daemon (hindsight-embed) + +Set an LLM API key and Hindsight will start the local server automatically: + +```bash +export OPENAI_API_KEY=sk-your-key +# or +export ANTHROPIC_API_KEY=your-key +``` + +### Configuration options + +| Key | Default | Description | +|-----|---------|-------------| +| `hindsightApiUrl` | `""` | External API URL (empty = local daemon) | +| `hindsightApiToken` | `null` | API token for Hindsight Cloud | +| `bankId` | `"codex"` | Memory bank identifier | +| `bankMission` | (set) | Guides what facts Hindsight retains | +| `autoRecall` | `true` | Inject memories before each prompt | +| `autoRetain` | `true` | Store conversations after each turn | +| `retainMode` | `"full-session"` | `"full-session"` or `"chunked"` | +| `retainEveryNTurns` | `10` | Retain every N turns (1 = every turn) | +| `recallBudget` | `"mid"` | Recall depth: `"low"`, `"mid"`, `"high"` | +| `recallMaxTokens` | `1024` | Max tokens for injected memories | +| `dynamicBankId` | `false` | Separate bank per project/session | +| `dynamicBankGranularity` | `["agent", "project"]` | Fields for dynamic bank ID | +| `debug` | `false` | Log debug info to stderr | + +### Environment variable overrides + +All settings can also be set via environment variables: + +```bash +export HINDSIGHT_API_URL=https://api.hindsight.vectorize.io +export HINDSIGHT_API_TOKEN=your-api-key +export HINDSIGHT_BANK_ID=my-project +export HINDSIGHT_DEBUG=true +``` + +## How memory works + +**Recall** — before each prompt, Hindsight searches your memory bank for facts relevant to what you're about to ask. Found memories are injected as context so Codex has continuity across sessions. + +**Retain** — after each turn, Codex's conversation is stored to Hindsight. The memory engine extracts facts, relationships, and experiences — so you don't need to re-explain your stack, preferences, or past decisions. + +## Dynamic bank IDs + +To keep separate memory per project: + +```json +{ + "dynamicBankId": true, + "dynamicBankGranularity": ["agent", "project"] +} +``` + +This creates banks like `codex::my-project` automatically, using the working directory name. + +## Troubleshooting + +**Memory not appearing**: Enable debug mode (`"debug": true`) and check stderr output. + +**Server not starting**: Set `hindsightApiUrl` to use an external server, or ensure `uvx` is on PATH for local daemon mode. + +**Hooks not firing**: Check that `~/.codex/config.toml` contains `codex_hooks = true` under `[features]`, and that your Codex CLI version supports hooks (v0.116.0+). diff --git a/hindsight-integrations/codex/hooks/hooks.json b/hindsight-integrations/codex/hooks/hooks.json new file mode 100644 index 00000000..48768cbd --- /dev/null +++ b/hindsight-integrations/codex/hooks/hooks.json @@ -0,0 +1,37 @@ +{ + "hooks": { + "SessionStart": [ + { + "hooks": [ + { + "type": "command", + "command": "python3 \"__SCRIPTS_DIR__/session_start.py\"", + "timeout": 5 + } + ] + } + ], + "UserPromptSubmit": [ + { + "hooks": [ + { + "type": "command", + "command": "python3 \"__SCRIPTS_DIR__/recall.py\"", + "timeout": 12 + } + ] + } + ], + "Stop": [ + { + "hooks": [ + { + "type": "command", + "command": "python3 \"__SCRIPTS_DIR__/retain.py\"", + "timeout": 30 + } + ] + } + ] + } +} diff --git a/hindsight-integrations/codex/install.sh b/hindsight-integrations/codex/install.sh new file mode 100755 index 00000000..4c3e09b8 --- /dev/null +++ b/hindsight-integrations/codex/install.sh @@ -0,0 +1,149 @@ +#!/usr/bin/env bash +# Hindsight memory integration for OpenAI Codex CLI +# +# This script installs the Hindsight hooks into ~/.codex/ and copies +# the hook scripts to ~/.hindsight/codex/scripts/. +# +# Usage: +# ./install.sh # Install (or update) +# ./install.sh --uninstall # Remove Hindsight hooks + +set -euo pipefail + +INTEGRATION_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +INSTALL_DIR="${HOME}/.hindsight/codex" +SCRIPTS_DIR="${INSTALL_DIR}/scripts" +CODEX_DIR="${HOME}/.codex" +HOOKS_FILE="${CODEX_DIR}/hooks.json" +CONFIG_FILE="${CODEX_DIR}/config.toml" + +# ────────────────────────────────────────────────────────────────────────────── +# Uninstall +# ────────────────────────────────────────────────────────────────────────────── +if [[ "${1:-}" == "--uninstall" ]]; then + echo "Uninstalling Hindsight Codex integration..." + + # Remove scripts directory + if [[ -d "${SCRIPTS_DIR}" ]]; then + rm -rf "${SCRIPTS_DIR}" + echo " Removed ${SCRIPTS_DIR}" + fi + + # Remove hooks.json + if [[ -f "${HOOKS_FILE}" ]]; then + rm -f "${HOOKS_FILE}" + echo " Removed ${HOOKS_FILE}" + fi + + # Remove codex_hooks feature flag from config.toml (if present) + if [[ -f "${CONFIG_FILE}" ]]; then + # Remove the [features] block line for codex_hooks + sed -i.bak '/^codex_hooks *= *true/d' "${CONFIG_FILE}" && rm -f "${CONFIG_FILE}.bak" + echo " Removed codex_hooks from ${CONFIG_FILE}" + fi + + echo "Uninstall complete." + exit 0 +fi + +# ────────────────────────────────────────────────────────────────────────────── +# Install +# ────────────────────────────────────────────────────────────────────────────── +echo "Installing Hindsight Codex integration..." + +# 1. Copy scripts to ~/.hindsight/codex/scripts/ +mkdir -p "${SCRIPTS_DIR}" +cp -r "${INTEGRATION_DIR}/scripts/." "${SCRIPTS_DIR}/" +chmod +x "${SCRIPTS_DIR}/session_start.py" +chmod +x "${SCRIPTS_DIR}/recall.py" +chmod +x "${SCRIPTS_DIR}/retain.py" +echo " Scripts installed to ${SCRIPTS_DIR}" + +# 2. Copy default settings (don't overwrite user's existing settings) +SETTINGS_DST="${INSTALL_DIR}/settings.json" +if [[ ! -f "${SETTINGS_DST}" ]]; then + cp "${INTEGRATION_DIR}/settings.json" "${SETTINGS_DST}" + echo " Default settings written to ${SETTINGS_DST}" +else + echo " Keeping existing settings at ${SETTINGS_DST}" +fi + +# 3. Write ~/.codex/hooks.json with absolute paths +mkdir -p "${CODEX_DIR}" +cat > "${HOOKS_FILE}" </dev/null; then + # Section exists — add codex_hooks under it if not already present + if ! grep -q '^codex_hooks' "${CONFIG_FILE}"; then + # Insert codex_hooks after [features] + sed -i.bak '/^\[features\]/a codex_hooks = true' "${CONFIG_FILE}" && rm -f "${CONFIG_FILE}.bak" + echo " Added codex_hooks = true to [features] in ${CONFIG_FILE}" + else + echo " codex_hooks already enabled in ${CONFIG_FILE}" + fi +else + # No [features] section — append it + printf '\n[features]\ncodex_hooks = true\n' >> "${CONFIG_FILE}" + echo " Added [features] codex_hooks = true to ${CONFIG_FILE}" +fi + +echo "" +echo "Hindsight is installed for Codex." +echo "" +echo "Configuration:" +echo " Edit ${SETTINGS_DST} to customize settings." +echo " Or create ~/.hindsight/codex.json for personal overrides." +echo "" +echo "For Hindsight Cloud, set:" +echo " \"hindsightApiUrl\": \"https://api.hindsight.vectorize.io\"" +echo " \"hindsightApiToken\": \"your-api-key\"" +echo "" +echo "For local daemon mode, set an LLM API key:" +echo " export OPENAI_API_KEY=sk-your-key" +echo "" +echo "Start a new Codex session to activate." diff --git a/hindsight-integrations/codex/scripts/lib/__init__.py b/hindsight-integrations/codex/scripts/lib/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/hindsight-integrations/codex/scripts/lib/bank.py b/hindsight-integrations/codex/scripts/lib/bank.py new file mode 100644 index 00000000..21d81c4c --- /dev/null +++ b/hindsight-integrations/codex/scripts/lib/bank.py @@ -0,0 +1,91 @@ +"""Bank ID derivation and mission management for Codex. + +Codex context dimensions: + - agent → configured name or "codex" (HINDSIGHT_AGENT_NAME) + - project → derived from cwd (working directory basename) + - session → session_id from hook input + - user → from env var HINDSIGHT_USER_ID + +The channel dimension is omitted — Codex is a CLI tool without multi-channel +routing like Telegram/Discord agents. +""" + +import os +import sys +import urllib.parse + +from .state import read_state, write_state + +DEFAULT_BANK_NAME = "codex" + +# Valid granularity fields for Codex +VALID_FIELDS = {"agent", "project", "session", "user"} + + +def derive_bank_id(hook_input: dict, config: dict) -> str: + """Derive a bank ID from hook context and config. + + When dynamicBankId is false, returns the static bank. + When true, composes from granularity fields joined by '::'. + """ + prefix = config.get("bankIdPrefix", "") + + if not config.get("dynamicBankId", False): + base = config.get("bankId") or DEFAULT_BANK_NAME + return f"{prefix}-{base}" if prefix else base + + # Dynamic mode — compose from granularity fields + fields = config.get("dynamicBankGranularity") + if not fields or not isinstance(fields, list): + fields = ["agent", "project"] + + for f in fields: + if f not in VALID_FIELDS: + print( + f'[Hindsight] Unknown dynamicBankGranularity field "{f}" — ' + f"valid for Codex: {', '.join(sorted(VALID_FIELDS))}", + file=sys.stderr, + ) + + cwd = hook_input.get("cwd", "") + session_id = hook_input.get("session_id", "") + agent_name = config.get("agentName", "codex") + user_id = os.environ.get("HINDSIGHT_USER_ID", "") + + field_map = { + "agent": agent_name, + "project": os.path.basename(cwd) if cwd else "unknown", + "session": session_id or "unknown", + "user": user_id or "anonymous", + } + + segments = [urllib.parse.quote(field_map.get(f, "unknown"), safe="") for f in fields] + base_bank_id = "::".join(segments) + + return f"{prefix}-{base_bank_id}" if prefix else base_bank_id + + +def ensure_bank_mission(client, bank_id: str, config: dict, debug_fn=None): + """Set bank mission on first use, skip if already set.""" + mission = config.get("bankMission", "") + if not mission or not mission.strip(): + return + + missions_set = read_state("bank_missions.json", {}) + if bank_id in missions_set: + return + + try: + retain_mission = config.get("retainMission") + client.set_bank_mission(bank_id, mission, retain_mission=retain_mission, timeout=10) + missions_set[bank_id] = True + if len(missions_set) > 10000: + keys = sorted(missions_set.keys()) + for k in keys[: len(keys) // 2]: + del missions_set[k] + write_state("bank_missions.json", missions_set) + if debug_fn: + debug_fn(f"Set mission for bank: {bank_id}") + except Exception as e: + if debug_fn: + debug_fn(f"Could not set bank mission for {bank_id}: {e}") diff --git a/hindsight-integrations/codex/scripts/lib/client.py b/hindsight-integrations/codex/scripts/lib/client.py new file mode 100644 index 00000000..277cbafb --- /dev/null +++ b/hindsight-integrations/codex/scripts/lib/client.py @@ -0,0 +1,163 @@ +"""Hindsight REST API client. + +Communicates with a Hindsight server via HTTP. Mirrors the HTTP mode of the +Openclaw HindsightClient (client.js), adapted for Python stdlib. +""" + +import json +import urllib.error +import urllib.parse +import urllib.request +from typing import Optional + +DEFAULT_TIMEOUT = 15 # seconds +HEALTH_CHECK_RETRIES = 3 +HEALTH_CHECK_DELAY = 2 # seconds + + +def _validate_api_url(url: str) -> str: + """Validate and normalize the API URL. Reject non-HTTP schemes.""" + parsed = urllib.parse.urlparse(url) + if parsed.scheme not in ("http", "https"): + raise ValueError(f"Hindsight API URL must use http or https, got: {parsed.scheme!r}") + if not parsed.hostname: + raise ValueError(f"Hindsight API URL has no hostname: {url!r}") + return url.rstrip("/") + + +class HindsightClient: + """HTTP client for the Hindsight API.""" + + def __init__(self, api_url: str, api_token: Optional[str] = None): + self.api_url = _validate_api_url(api_url) + self.api_token = api_token + + def _headers(self) -> dict: + headers = {"Content-Type": "application/json"} + if self.api_token: + headers["Authorization"] = f"Bearer {self.api_token}" + return headers + + def _request(self, method: str, path: str, body: Optional[dict] = None, timeout: int = DEFAULT_TIMEOUT) -> dict: + url = f"{self.api_url}{path}" + data = json.dumps(body).encode() if body else None + req = urllib.request.Request(url, data=data, headers=self._headers(), method=method) + try: + with urllib.request.urlopen(req, timeout=timeout) as resp: + return json.loads(resp.read().decode()) + except urllib.error.HTTPError as e: + body_text = "" + try: + body_text = e.read().decode() + except Exception: + pass + raise RuntimeError(f"HTTP {e.code} from {url}: {body_text}") from e + + def health_check(self, timeout: int = 5) -> bool: + """Check if the Hindsight server is reachable. + + Mirrors Openclaw's checkExternalApiHealth: retries up to 3 times + with 2s delay between attempts. + """ + import time + + for attempt in range(1, HEALTH_CHECK_RETRIES + 1): + try: + url = f"{self.api_url}/health" + req = urllib.request.Request(url, headers=self._headers(), method="GET") + with urllib.request.urlopen(req, timeout=timeout) as resp: + if resp.status == 200: + return True + except Exception: + pass + if attempt < HEALTH_CHECK_RETRIES: + time.sleep(HEALTH_CHECK_DELAY) + return False + + def recall( + self, + bank_id: str, + query: str, + max_tokens: int = 1024, + budget: str = "mid", + types: Optional[list] = None, + timeout: int = 10, + ) -> dict: + """Recall memories from a bank. + + Returns the raw API response dict with 'results' list. + """ + path = f"/v1/default/banks/{urllib.parse.quote(bank_id, safe='')}/memories/recall" + body = { + "query": query, + "max_tokens": max_tokens, + } + if budget: + body["budget"] = budget + if types: + body["types"] = types + return self._request("POST", path, body, timeout=timeout) + + def retain( + self, + bank_id: str, + content: str, + document_id: str = "conversation", + context: Optional[str] = None, + metadata: Optional[dict] = None, + tags: Optional[list] = None, + timeout: int = 15, + ) -> dict: + """Retain content into a bank's memory. + + Posts with async=true so the server processes in the background. + The context field helps Hindsight cluster memories by provenance + (e.g. "claude-code" vs manual retains). + """ + path = f"/v1/default/banks/{urllib.parse.quote(bank_id, safe='')}/memories" + item = { + "content": content, + "document_id": document_id, + "metadata": metadata or {}, + } + if context: + item["context"] = context + if tags: + item["tags"] = tags + body = { + "items": [item], + "async": True, + } + return self._request("POST", path, body, timeout=timeout) + + def reflect( + self, + bank_id: str, + query: str, + budget: str = "mid", + max_tokens: int = 1024, + timeout: int = 30, + ) -> dict: + """Reflect on memories and return a synthesized answer. + + Runs an agentic loop that retrieves facts, mental models, and experiences, + then uses the LLM to formulate a coherent response. Slower than recall + but produces a synthesized prose answer rather than raw facts. + """ + path = f"/v1/default/banks/{urllib.parse.quote(bank_id, safe='')}/reflect" + body = {"query": query, "budget": budget, "max_tokens": max_tokens} + return self._request("POST", path, body, timeout=timeout) + + def set_bank_mission( + self, bank_id: str, mission: str, retain_mission: Optional[str] = None, timeout: int = 15 + ) -> dict: + """Set the mission/persona for a bank. + + Uses PATCH /banks/{id}/config with reflect_mission and retain_mission. + The old PUT /banks/{id} with 'mission' field is deprecated in v0.4.19. + """ + path = f"/v1/default/banks/{urllib.parse.quote(bank_id, safe='')}/config" + updates = {"reflect_mission": mission} + if retain_mission: + updates["retain_mission"] = retain_mission + return self._request("PATCH", path, {"updates": updates}, timeout=timeout) diff --git a/hindsight-integrations/codex/scripts/lib/config.py b/hindsight-integrations/codex/scripts/lib/config.py new file mode 100644 index 00000000..be63cda2 --- /dev/null +++ b/hindsight-integrations/codex/scripts/lib/config.py @@ -0,0 +1,143 @@ +"""Configuration management for Hindsight Codex plugin. + +Loads settings from settings.json (plugin defaults) merged with environment +variable overrides. Full config schema matching Openclaw's 30+ options. +""" + +import json +import os +import sys + +DEFAULTS = { + # Recall + "autoRecall": True, + "recallBudget": "mid", + "recallMaxTokens": 1024, + "recallTypes": ["world", "experience"], + "recallContextTurns": 1, + "recallMaxQueryChars": 800, + "recallRoles": ["user", "assistant"], + "recallPromptPreamble": ( + "Relevant memories from past conversations (prioritize recent when " + "conflicting). Only use memories that are directly useful to continue " + "this conversation; ignore the rest:" + ), + # Retain + "autoRetain": True, + "retainMode": "full-session", + "retainRoles": ["user", "assistant"], + "retainEveryNTurns": 10, + "retainOverlapTurns": 2, + "retainToolCalls": True, + "retainContext": "codex", + "retainTags": [], + "retainMetadata": {}, + # Connection + "hindsightApiUrl": None, + "hindsightApiToken": None, + "apiPort": 9077, + "daemonIdleTimeout": 0, + "embedVersion": "latest", + "embedPackagePath": None, + # Bank + "bankId": None, + "bankIdPrefix": "", + "dynamicBankId": False, + "dynamicBankGranularity": ["agent", "project"], + "bankMission": "", + "retainMission": None, + "agentName": "codex", + # LLM (for daemon mode) + "llmProvider": None, + "llmModel": None, + "llmApiKeyEnv": None, + # Misc + "debug": False, +} + +# Map env var names to config keys and their types +ENV_OVERRIDES = { + "HINDSIGHT_API_URL": ("hindsightApiUrl", str), + "HINDSIGHT_API_TOKEN": ("hindsightApiToken", str), + "HINDSIGHT_BANK_ID": ("bankId", str), + "HINDSIGHT_AGENT_NAME": ("agentName", str), + "HINDSIGHT_AUTO_RECALL": ("autoRecall", bool), + "HINDSIGHT_AUTO_RETAIN": ("autoRetain", bool), + "HINDSIGHT_RETAIN_MODE": ("retainMode", str), + "HINDSIGHT_RECALL_BUDGET": ("recallBudget", str), + "HINDSIGHT_RECALL_MAX_TOKENS": ("recallMaxTokens", int), + "HINDSIGHT_RECALL_MAX_QUERY_CHARS": ("recallMaxQueryChars", int), + "HINDSIGHT_RECALL_CONTEXT_TURNS": ("recallContextTurns", int), + "HINDSIGHT_API_PORT": ("apiPort", int), + "HINDSIGHT_DAEMON_IDLE_TIMEOUT": ("daemonIdleTimeout", int), + "HINDSIGHT_EMBED_VERSION": ("embedVersion", str), + "HINDSIGHT_EMBED_PACKAGE_PATH": ("embedPackagePath", str), + "HINDSIGHT_DYNAMIC_BANK_ID": ("dynamicBankId", bool), + "HINDSIGHT_BANK_MISSION": ("bankMission", str), + "HINDSIGHT_LLM_PROVIDER": ("llmProvider", str), + "HINDSIGHT_LLM_MODEL": ("llmModel", str), + "HINDSIGHT_DEBUG": ("debug", bool), +} + + +def _cast_env(value: str, typ): + """Cast environment variable string to target type. Returns None on failure.""" + try: + if typ is bool: + return value.lower() in ("true", "1", "yes") + if typ is int: + return int(value) + return value + except (ValueError, AttributeError): + return None + + +def _load_settings_file(path: str, config: dict) -> None: + """Merge a settings.json file into config in-place. Silently skips if missing.""" + if not os.path.exists(path): + return + try: + with open(path) as f: + file_config = json.load(f) + config.update({k: v for k, v in file_config.items() if v is not None}) + except (json.JSONDecodeError, OSError) as e: + debug_log(config, f"Failed to load {path}: {e}") + + +def load_config() -> dict: + """Load plugin configuration from settings.json + env overrides. + + Loading order (later entries win): + 1. Built-in defaults + 2. Plugin install settings.json (~/.hindsight/codex/settings.json) + 3. User config (~/.hindsight/codex.json) + 4. Environment variable overrides + + ~/.hindsight/codex.json is the recommended place to configure the + plugin — stable across updates. + """ + config = dict(DEFAULTS) + + # 1. Plugin install settings.json (written by install.sh) + install_root = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + _load_settings_file(os.path.join(install_root, "settings.json"), config) + + # 2. User config — stable, version-independent + user_config_path = os.path.join(os.path.expanduser("~"), ".hindsight", "codex.json") + _load_settings_file(user_config_path, config) + + # Apply environment variable overrides + for env_name, (key, typ) in ENV_OVERRIDES.items(): + val = os.environ.get(env_name) + if val is not None: + cast_val = _cast_env(val, typ) + if cast_val is not None: + config[key] = cast_val + + return config + + +def debug_log(config: dict, *args): + """Log to stderr if debug mode is enabled.""" + if config.get("debug"): + print("[Hindsight]", *args, file=sys.stderr) diff --git a/hindsight-integrations/codex/scripts/lib/content.py b/hindsight-integrations/codex/scripts/lib/content.py new file mode 100644 index 00000000..502a973a --- /dev/null +++ b/hindsight-integrations/codex/scripts/lib/content.py @@ -0,0 +1,318 @@ +"""Content processing utilities for Codex. + +Adapts Openclaw/Claude Code content processing for Codex's transcript format. + +Codex transcript format (JSONL): + {"session_id": "...", "ts": 1234567890, "msg": {"type": "user_message", "message": "..."}} + +EventMsg types (from codex-rs/protocol/src/protocol.rs, serde snake_case): + - user_message → role: user + - agent_message → role: assistant + - task_started, task_complete, exec, etc. → skipped +""" + +import json +import os +import re +from datetime import datetime, timezone + + +# --------------------------------------------------------------------------- +# Memory tag stripping (anti-feedback-loop) +# --------------------------------------------------------------------------- + + +def strip_memory_tags(content: str) -> str: + """Remove and blocks. + + Prevents retain feedback loop — these were injected during recall and + should not be re-stored. + """ + content = re.sub(r"[\s\S]*?", "", content) + content = re.sub(r"[\s\S]*?", "", content) + return content + + +# --------------------------------------------------------------------------- +# Transcript reading +# --------------------------------------------------------------------------- + + +def read_transcript(transcript_path: str) -> list: + """Read a Codex JSONL transcript and return list of {role, content} dicts. + + Codex disk format (rollout-*.jsonl): + User: {"type":"response_item","payload":{"type":"message","role":"user", + "content":[{"type":"input_text","text":"..."}]}} + Assistant: {"type":"response_item","payload":{"type":"message","role":"assistant", + "content":[{"type":"output_text","text":"..."}],"phase":"final_answer"}} + + Flat format for testing: + {"role": "user", "content": "..."} + """ + if not transcript_path or not os.path.isfile(transcript_path): + return [] + messages = [] + try: + with open(transcript_path) as f: + for line in f: + line = line.strip() + if not line: + continue + try: + entry = json.loads(line) + # Codex response_item format + if entry.get("type") == "response_item": + payload = entry.get("payload", {}) + if payload.get("type") == "message": + role = payload.get("role", "") + if role not in ("user", "assistant"): + continue + # Only include final_answer for assistant (not reasoning/intermediary) + if role == "assistant" and payload.get("phase") != "final_answer": + continue + content_blocks = payload.get("content", []) + text_parts = [] + for block in content_blocks: + if isinstance(block, dict) and block.get("type") in ("input_text", "output_text"): + t = block.get("text", "").strip() + if t: + text_parts.append(t) + text = "\n".join(text_parts).strip() + if text: + messages.append({"role": role, "content": text}) + # Flat format (testing / future compatibility) + elif "role" in entry and "content" in entry: + messages.append({"role": entry["role"], "content": entry["content"]}) + except json.JSONDecodeError: + continue + except OSError: + pass + return messages + + +# --------------------------------------------------------------------------- +# Recall: query composition and truncation +# --------------------------------------------------------------------------- + + +def compose_recall_query( + latest_query: str, + messages: list, + recall_context_turns: int, + recall_roles: list = None, +) -> str: + """Compose a multi-turn recall query from conversation history. + + When recallContextTurns > 1, includes prior context above the latest + user query. Format: + + Prior context: + + user: ... + assistant: ... + + + """ + latest = latest_query.strip() + if recall_context_turns <= 1 or not isinstance(messages, list) or not messages: + return latest + + allowed_roles = set(recall_roles or ["user", "assistant"]) + contextual_messages = slice_last_turns_by_user_boundary(messages, recall_context_turns) + + context_lines = [] + for msg in contextual_messages: + role = msg.get("role") + if role not in allowed_roles: + continue + + content = msg.get("content", "") + if not isinstance(content, str): + content = str(content) + content = strip_memory_tags(content).strip() + if not content: + continue + + if role == "user" and content == latest: + continue + + context_lines.append(f"{role}: {content}") + + if not context_lines: + return latest + + return "\n\n".join( + [ + "Prior context:", + "\n".join(context_lines), + latest, + ] + ) + + +def truncate_recall_query(query: str, latest_query: str, max_chars: int) -> str: + """Truncate a composed recall query to max_chars. + + Preserves the latest user message. Drops oldest context lines first. + """ + if max_chars <= 0: + return query + + latest = latest_query.strip() + if len(query) <= max_chars: + return query + + latest_only = latest[:max_chars] if len(latest) > max_chars else latest + + if "Prior context:" not in query: + return latest_only + + context_marker = "Prior context:\n\n" + marker_index = query.find(context_marker) + if marker_index == -1: + return latest_only + + suffix_marker = "\n\n" + latest + suffix_index = query.rfind(suffix_marker) + if suffix_index == -1: + return latest_only + + suffix = query[suffix_index:] + if len(suffix) >= max_chars: + return latest_only + + context_body = query[marker_index + len(context_marker) : suffix_index] + context_lines = [line for line in context_body.split("\n") if line] + + kept = [] + for i in range(len(context_lines) - 1, -1, -1): + kept.insert(0, context_lines[i]) + candidate = f"{context_marker}{chr(10).join(kept)}{suffix}" + if len(candidate) > max_chars: + kept.pop(0) + break + + if kept: + return f"{context_marker}{chr(10).join(kept)}{suffix}" + return latest_only + + +# --------------------------------------------------------------------------- +# Turn slicing +# --------------------------------------------------------------------------- + + +def slice_last_turns_by_user_boundary(messages: list, turns: int) -> list: + """Slice messages to the last N turns, where a turn starts at a user message.""" + if not isinstance(messages, list) or not messages or turns <= 0: + return [] + + user_turns_seen = 0 + start_index = -1 + + for i in range(len(messages) - 1, -1, -1): + if messages[i].get("role") == "user": + user_turns_seen += 1 + if user_turns_seen >= turns: + start_index = i + break + + if start_index == -1: + return list(messages) + + return messages[start_index:] + + +# --------------------------------------------------------------------------- +# Memory formatting (recall results → context string) +# --------------------------------------------------------------------------- + + +def format_memories(results: list) -> str: + """Format recall results into human-readable text.""" + if not results: + return "" + lines = [] + for r in results: + text = r.get("text", "") + mem_type = r.get("type", "") + mentioned_at = r.get("mentioned_at", "") + type_str = f" [{mem_type}]" if mem_type else "" + date_str = f" ({mentioned_at})" if mentioned_at else "" + lines.append(f"- {text}{type_str}{date_str}") + return "\n\n".join(lines) + + +def format_current_time() -> str: + """Format current UTC time for recall context.""" + now = datetime.now(timezone.utc) + return now.strftime("%Y-%m-%d %H:%M") + + +# --------------------------------------------------------------------------- +# Retention transcript formatting +# --------------------------------------------------------------------------- + + +def prepare_retention_transcript( + messages: list, + retain_roles: list = None, + retain_full_window: bool = False, +) -> tuple: + """Format messages into a retention transcript. + + Outputs plain text with [role: ...]...[role:end] markers. + Codex doesn't have tool calls to retain (it's a coding agent with + shell/patch commands, not MCP tools), so we use the text format only. + + Args: + messages: List of {role, content} dicts. + retain_roles: Roles to include (default: ['user', 'assistant']). + retain_full_window: If True, retain all messages. If False, retain + only the last turn (last user msg + responses). + + Returns: + (transcript_text, message_count) or (None, 0) if nothing to retain. + """ + if not messages: + return None, 0 + + if retain_full_window: + target_messages = messages + else: + last_user_idx = -1 + for i in range(len(messages) - 1, -1, -1): + if messages[i].get("role") == "user": + last_user_idx = i + break + if last_user_idx == -1: + return None, 0 + target_messages = messages[last_user_idx:] + + allowed_roles = set(retain_roles or ["user", "assistant"]) + parts = [] + + for msg in target_messages: + role = msg.get("role", "unknown") + if role not in allowed_roles: + continue + + content = msg.get("content", "") + if not isinstance(content, str): + content = str(content) + content = strip_memory_tags(content).strip() + + if not content: + continue + + parts.append(f"[role: {role}]\n{content}\n[{role}:end]") + + if not parts: + return None, 0 + + transcript = "\n\n".join(parts) + if len(transcript.strip()) < 10: + return None, 0 + + return transcript, len(parts) diff --git a/hindsight-integrations/codex/scripts/lib/daemon.py b/hindsight-integrations/codex/scripts/lib/daemon.py new file mode 100644 index 00000000..d9ca19e6 --- /dev/null +++ b/hindsight-integrations/codex/scripts/lib/daemon.py @@ -0,0 +1,275 @@ +"""Hindsight-embed daemon lifecycle management for Codex. + +Manages three connection modes: + 1. External API — user provides hindsightApiUrl (skip daemon entirely) + 2. Existing local server — user already has hindsight running + 3. Auto-managed daemon — plugin starts/stops hindsight-embed + +Daemon state is tracked via files in ~/.hindsight/codex/state/. +""" + +import os +import platform +import subprocess +import time +import urllib.error +import urllib.request + +from .llm import detect_llm_config, get_llm_env_vars +from .state import read_state, write_state + +DAEMON_STATE_FILE = "daemon.json" +PROFILE_NAME = "codex" + + +def _get_embed_command(config: dict) -> list: + """Get the command to run hindsight-embed.""" + embed_path = config.get("embedPackagePath") + if embed_path: + return ["uv", "run", "--directory", embed_path, "hindsight-embed"] + + version = config.get("embedVersion", "latest") + package = f"hindsight-embed@{version}" if version else "hindsight-embed@latest" + return ["uvx", package] + + +def _run_embed(config: dict, args: list, env: dict = None, timeout: int = 10) -> subprocess.CompletedProcess: + """Run a hindsight-embed command and return the result.""" + cmd = _get_embed_command(config) + args + run_env = dict(os.environ) + if env: + run_env.update(env) + return subprocess.run( + cmd, + capture_output=True, + text=True, + timeout=timeout, + env=run_env, + ) + + +def _is_embed_available(config: dict) -> bool: + """Quick check if hindsight-embed is available on PATH.""" + import shutil + + embed_path = config.get("embedPackagePath") + if embed_path: + return os.path.isdir(embed_path) + return shutil.which("uvx") is not None or shutil.which("hindsight-embed") is not None + + +def _check_health(base_url: str, timeout: int = 2) -> bool: + """Quick health check against a Hindsight server.""" + try: + url = f"{base_url.rstrip('/')}/health" + req = urllib.request.Request(url, method="GET") + with urllib.request.urlopen(req, timeout=timeout) as resp: + return resp.status == 200 + except Exception: + return False + + +def get_api_url(config: dict, debug_fn=None, allow_daemon_start: bool = False) -> str: + """Determine the API URL, optionally starting daemon if needed. + + Connection mode priority: + 1. External API (hindsightApiUrl configured) + 2. Existing local server (check port health) + 3. Auto-managed daemon (only if allow_daemon_start=True) + """ + # Mode 1: External API + external_url = config.get("hindsightApiUrl") + if external_url: + if debug_fn: + debug_fn(f"Using external API: {external_url}") + return external_url + + # Mode 2 & 3: Local server + port = config.get("apiPort", 9077) + base_url = f"http://127.0.0.1:{port}" + + if _check_health(base_url): + if debug_fn: + debug_fn(f"Existing server healthy on port {port}") + return base_url + + # Mode 3: Auto-start daemon (only when allowed) + if not allow_daemon_start: + raise RuntimeError( + f"No Hindsight server on port {port}. Set hindsightApiUrl for external " + f"API, start hindsight-embed manually, or wait for the retain hook to " + f"auto-start the daemon." + ) + + if debug_fn: + debug_fn(f"No server on port {port}, attempting daemon start") + + try: + _ensure_daemon_running(config, port, debug_fn) + except Exception as e: + if debug_fn: + debug_fn(f"Daemon start failed: {e}") + raise RuntimeError( + "No Hindsight server available. Set hindsightApiUrl for external API, " + "or ensure hindsight-embed is installed for local daemon mode." + ) from e + + return base_url + + +def _ensure_daemon_running(config: dict, port: int, debug_fn=None): + """Start the hindsight-embed daemon if not already running.""" + if not _is_embed_available(config): + raise RuntimeError( + "hindsight-embed not found (uvx not on PATH). " + "Install with: pip install hindsight-embed, or set hindsightApiUrl." + ) + + base_url = f"http://127.0.0.1:{port}" + + try: + llm_config = detect_llm_config(config) + except RuntimeError as e: + raise RuntimeError(f"Cannot start daemon: {e}") from e + + llm_env = get_llm_env_vars(llm_config) + + daemon_env = dict(llm_env) + idle_timeout = config.get("daemonIdleTimeout", 300) + daemon_env["HINDSIGHT_EMBED_DAEMON_IDLE_TIMEOUT"] = str(idle_timeout) + + if platform.system() == "Darwin": + daemon_env["HINDSIGHT_API_EMBEDDINGS_LOCAL_FORCE_CPU"] = "1" + daemon_env["HINDSIGHT_API_RERANKER_LOCAL_FORCE_CPU"] = "1" + + # Step 1: Configure profile + if debug_fn: + debug_fn(f'Configuring "{PROFILE_NAME}" profile...') + + profile_args = [ + "profile", + "create", + PROFILE_NAME, + "--merge", + "--port", + str(port), + ] + for env_name, env_val in daemon_env.items(): + if env_val: + profile_args.extend(["--env", f"{env_name}={env_val}"]) + + try: + result = _run_embed(config, profile_args, daemon_env, timeout=10) + if result.returncode != 0: + if debug_fn: + debug_fn(f"Profile create stderr: {result.stderr.strip()}") + raise RuntimeError(f"Profile create failed (exit {result.returncode}): {result.stderr}") + if debug_fn: + debug_fn("Profile configured") + except subprocess.TimeoutExpired: + raise RuntimeError("Profile create timed out") + except FileNotFoundError: + raise RuntimeError( + "hindsight-embed not found. Install with: pip install hindsight-embed " + "or set hindsightApiUrl for external API mode." + ) + + # Step 2: Start daemon + if debug_fn: + debug_fn("Starting daemon...") + + try: + result = _run_embed( + config, + ["daemon", "--profile", PROFILE_NAME, "start"], + daemon_env, + timeout=30, + ) + if debug_fn: + debug_fn(f"Daemon start exit={result.returncode} stdout={result.stdout.strip()}") + if result.returncode != 0 and "already running" not in result.stderr.lower(): + raise RuntimeError(f"Daemon start failed (exit {result.returncode}): {result.stderr}") + except subprocess.TimeoutExpired: + raise RuntimeError("Daemon start timed out") + + # Step 3: Wait for ready + if debug_fn: + debug_fn("Waiting for daemon to be ready...") + + for attempt in range(30): + if _check_health(base_url): + if debug_fn: + debug_fn(f"Daemon ready after {attempt + 1} attempts") + write_state( + DAEMON_STATE_FILE, + { + "port": port, + "started_by_plugin": True, + "started_at": time.time(), + "pid": os.getpid(), + }, + ) + return + time.sleep(1) + + raise RuntimeError("Daemon failed to become ready within 30 seconds") + + +def prestart_daemon_background(config: dict, debug_fn=None): + """Fire off daemon startup in the background — non-blocking. + + Called from SessionStart hook to warm up the daemon before the first + recall or retain hook fires. + """ + if config.get("hindsightApiUrl"): + return # External API mode — no local daemon needed + + port = config.get("apiPort", 9077) + if _check_health(f"http://127.0.0.1:{port}"): + if debug_fn: + debug_fn(f"Daemon already running on port {port}, skipping pre-start") + return + + if not _is_embed_available(config): + if debug_fn: + debug_fn("hindsight-embed not available, skipping pre-start") + return + + try: + llm_config = detect_llm_config(config) + except RuntimeError as e: + if debug_fn: + debug_fn(f"No LLM configured, skipping daemon pre-start: {e}") + return + + llm_env = get_llm_env_vars(llm_config) + daemon_env = dict(os.environ) + daemon_env.update(llm_env) + idle_timeout = config.get("daemonIdleTimeout", 300) + daemon_env["HINDSIGHT_EMBED_DAEMON_IDLE_TIMEOUT"] = str(idle_timeout) + if platform.system() == "Darwin": + daemon_env["HINDSIGHT_API_EMBEDDINGS_LOCAL_FORCE_CPU"] = "1" + daemon_env["HINDSIGHT_API_RERANKER_LOCAL_FORCE_CPU"] = "1" + + embed_cmd = _get_embed_command(config) + + profile_args = ["profile", "create", PROFILE_NAME, "--merge", "--port", str(port)] + for env_name, env_val in llm_env.items(): + if env_val: + profile_args.extend(["--env", f"{env_name}={env_val}"]) + + import shlex + profile_str = shlex.join(embed_cmd + profile_args) + daemon_str = shlex.join(embed_cmd + ["daemon", "--profile", PROFILE_NAME, "start"]) + + import subprocess as _sp + _sp.Popen( + f"{profile_str} && {daemon_str}", + shell=True, + env=daemon_env, + stdout=_sp.DEVNULL, + stderr=_sp.DEVNULL, + start_new_session=True, + ) + if debug_fn: + debug_fn(f"Daemon pre-start initiated in background (port {port})") diff --git a/hindsight-integrations/codex/scripts/lib/llm.py b/hindsight-integrations/codex/scripts/lib/llm.py new file mode 100644 index 00000000..c007955d --- /dev/null +++ b/hindsight-integrations/codex/scripts/lib/llm.py @@ -0,0 +1,146 @@ +"""LLM provider detection for Hindsight's fact extraction. + +Port of: detectLLMConfig() in index.js + +When running hindsight-embed locally (daemon mode), it needs an LLM to +extract facts from retained conversations. This module detects the LLM +config using the same priority chain as Openclaw: + + 1. HINDSIGHT_API_LLM_* environment variables (highest priority) + 2. Plugin config (llmProvider, llmModel, llmApiKeyEnv) + 3. Auto-detect from standard provider env vars + 4. External API mode (server-side LLM, no local config needed) +""" + +import os + +# Provider detection table — same order as Openclaw +PROVIDER_DETECTION = [ + {"name": "openai", "key_env": "OPENAI_API_KEY"}, + {"name": "anthropic", "key_env": "ANTHROPIC_API_KEY"}, + {"name": "gemini", "key_env": "GEMINI_API_KEY"}, + {"name": "groq", "key_env": "GROQ_API_KEY"}, + {"name": "ollama", "key_env": ""}, + {"name": "openai-codex", "key_env": ""}, + {"name": "claude-code", "key_env": ""}, +] + +# Providers that don't require an API key +NO_KEY_REQUIRED = {"ollama", "openai-codex", "claude-code"} + + +def _find_provider(name): + """Find a provider entry by name.""" + for p in PROVIDER_DETECTION: + if p["name"] == name: + return p + return None + + +def detect_llm_config(config: dict) -> dict: + """Detect LLM configuration. + + Returns dict with: provider, api_key, model, base_url, source. + Returns None values for external API mode (server handles LLM). + Raises RuntimeError if no configuration found and not in external API mode. + """ + override_provider = os.environ.get("HINDSIGHT_API_LLM_PROVIDER") + override_model = os.environ.get("HINDSIGHT_API_LLM_MODEL") + override_key = os.environ.get("HINDSIGHT_API_LLM_API_KEY") + override_base_url = os.environ.get("HINDSIGHT_API_LLM_BASE_URL") + + # Priority 1: HINDSIGHT_API_LLM_PROVIDER env var + if override_provider: + if not override_key and override_provider not in NO_KEY_REQUIRED: + raise RuntimeError( + f'HINDSIGHT_API_LLM_PROVIDER is set to "{override_provider}" but HINDSIGHT_API_LLM_API_KEY is not set.' + ) + pinfo = _find_provider(override_provider) + return { + "provider": override_provider, + "api_key": override_key or "", + "model": override_model, + "base_url": override_base_url, + "source": "HINDSIGHT_API_LLM_PROVIDER override", + } + + # Priority 2: Plugin config llmProvider/llmModel + cfg_provider = config.get("llmProvider") + if cfg_provider: + pinfo = _find_provider(cfg_provider) + api_key = "" + key_env_name = config.get("llmApiKeyEnv") + if key_env_name: + api_key = os.environ.get(key_env_name, "") + elif pinfo and pinfo["key_env"]: + api_key = os.environ.get(pinfo["key_env"], "") + + if not api_key and cfg_provider not in NO_KEY_REQUIRED: + key_source = key_env_name or (pinfo["key_env"] if pinfo else "unknown") + raise RuntimeError( + f'Plugin config llmProvider is "{cfg_provider}" but no API key found. Expected env var: {key_source}' + ) + return { + "provider": cfg_provider, + "api_key": api_key, + "model": config.get("llmModel") or override_model, + "base_url": override_base_url, + "source": "plugin config", + } + + # Priority 3: Auto-detect from standard provider env vars + for pinfo in PROVIDER_DETECTION: + if pinfo["name"] in NO_KEY_REQUIRED: + continue # Must be explicitly requested + if not pinfo["key_env"]: + continue + api_key = os.environ.get(pinfo["key_env"], "") + if api_key: + return { + "provider": pinfo["name"], + "api_key": api_key, + "model": override_model, + "base_url": override_base_url, + "source": f"auto-detected from {pinfo['key_env']}", + } + + # Priority 4: External API mode — server handles LLM + if config.get("hindsightApiUrl"): + return { + "provider": None, + "api_key": None, + "model": None, + "base_url": None, + "source": "external-api-mode-no-llm", + } + + raise RuntimeError( + "No LLM configuration found for Hindsight.\n\n" + "Option 1: Set a standard provider API key (auto-detect):\n" + " export OPENAI_API_KEY=sk-your-key\n" + " export ANTHROPIC_API_KEY=your-key\n\n" + "Option 2: Override with Hindsight-specific env vars:\n" + " export HINDSIGHT_API_LLM_PROVIDER=openai\n" + " export HINDSIGHT_API_LLM_API_KEY=sk-your-key\n\n" + "Option 3: Use an external Hindsight API (server-side LLM):\n" + " Set hindsightApiUrl in settings.json or HINDSIGHT_API_URL env var\n\n" + "The model will be selected automatically by Hindsight. To override: export HINDSIGHT_API_LLM_MODEL=your-model" + ) + + +def get_llm_env_vars(llm_config: dict) -> dict: + """Build environment variables for hindsight-embed daemon from LLM config. + + These are passed to the daemon subprocess so it knows which LLM to use + for fact extraction. + """ + env = {} + if llm_config.get("provider"): + env["HINDSIGHT_API_LLM_PROVIDER"] = llm_config["provider"] + if llm_config.get("api_key"): + env["HINDSIGHT_API_LLM_API_KEY"] = llm_config["api_key"] + if llm_config.get("model"): + env["HINDSIGHT_API_LLM_MODEL"] = llm_config["model"] + if llm_config.get("base_url"): + env["HINDSIGHT_API_LLM_BASE_URL"] = llm_config["base_url"] + return env diff --git a/hindsight-integrations/codex/scripts/lib/state.py b/hindsight-integrations/codex/scripts/lib/state.py new file mode 100644 index 00000000..9689d612 --- /dev/null +++ b/hindsight-integrations/codex/scripts/lib/state.py @@ -0,0 +1,113 @@ +"""File-based state persistence. + +Codex hooks are ephemeral processes — state must be persisted to files. +Uses ~/.hindsight/codex/state/ as the storage directory. +""" + +import json +import os +import re +import sys + +# fcntl is Unix-only; import conditionally so the module loads on Windows +if sys.platform != "win32": + import fcntl +else: + fcntl = None + + +def _state_dir() -> str: + """Get the state directory, creating it if needed.""" + state_dir = os.path.join(os.path.expanduser("~"), ".hindsight", "codex", "state") + os.makedirs(state_dir, exist_ok=True) + return state_dir + + +def _safe_filename(name: str) -> str: + """Sanitize a filename to prevent path traversal.""" + name = re.sub(r'[\\/:*?"<>|\x00-\x1f]', "_", name) + name = name.replace("..", "_") + name = name[:200] + return name or "state" + + +def _state_file(name: str) -> str: + """Get path for a state file. Name is sanitized to prevent traversal.""" + safe = _safe_filename(name) + path = os.path.join(_state_dir(), safe) + # Final guard: resolved path must be inside state_dir + resolved = os.path.realpath(path) + expected_dir = os.path.realpath(_state_dir()) + if not resolved.startswith(expected_dir + os.sep) and resolved != expected_dir: + raise ValueError(f"State file path escapes state directory: {name!r}") + return path + + +def read_state(name: str, default=None): + """Read a JSON state file. Returns default if not found.""" + path = _state_file(name) + if not os.path.exists(path): + return default + try: + with open(path) as f: + return json.load(f) + except (json.JSONDecodeError, OSError): + return default + + +def write_state(name: str, data): + """Write data to a JSON state file atomically.""" + path = _state_file(name) + tmp_path = path + ".tmp" + try: + with open(tmp_path, "w") as f: + json.dump(data, f) + os.replace(tmp_path, path) + except OSError: + try: + os.unlink(tmp_path) + except OSError: + pass + + +def get_turn_count(session_id: str) -> int: + """Get the current turn count for a session.""" + turns = read_state("turns.json", {}) + return turns.get(session_id, 0) + + +def increment_turn_count(session_id: str) -> int: + """Increment and return the turn count for a session. + + Uses flock on Unix to prevent race conditions. On Windows, proceeds + without a lock — minor races here are harmless. + """ + lock_path = _state_file("turns.lock") + if fcntl is not None: + try: + lock_fd = open(lock_path, "w") + fcntl.flock(lock_fd, fcntl.LOCK_EX) + try: + turns = read_state("turns.json", {}) + turns[session_id] = turns.get(session_id, 0) + 1 + if len(turns) > 10000: + sorted_keys = sorted(turns.keys()) + for k in sorted_keys[: len(sorted_keys) // 2]: + del turns[k] + write_state("turns.json", turns) + return turns[session_id] + finally: + fcntl.flock(lock_fd, fcntl.LOCK_UN) + lock_fd.close() + except OSError: + pass + + # Fallback: proceed without lock + turns = read_state("turns.json", {}) + turns[session_id] = turns.get(session_id, 0) + 1 + if len(turns) > 10000: + sorted_keys = sorted(turns.keys()) + for k in sorted_keys[: len(sorted_keys) // 2]: + del turns[k] + write_state("turns.json", turns) + return turns[session_id] diff --git a/hindsight-integrations/codex/scripts/recall.py b/hindsight-integrations/codex/scripts/recall.py new file mode 100755 index 00000000..a7a11e5d --- /dev/null +++ b/hindsight-integrations/codex/scripts/recall.py @@ -0,0 +1,165 @@ +#!/usr/bin/env python3 +"""Auto-recall hook for UserPromptSubmit. + +Fires before each user prompt. Retrieves relevant memories from Hindsight +and injects them into the Codex context via hookSpecificOutput.additionalContext. + +Flow: + 1. Read hook input from stdin (session_id, transcript_path, prompt/user_prompt) + 2. Resolve API URL + 3. Derive bank ID and ensure mission + 4. Compose multi-turn query if recallContextTurns > 1 + 5. Truncate to recallMaxQueryChars + 6. Call Hindsight recall API + 7. Format memories and output hookSpecificOutput.additionalContext + +Exit codes: + 0 — always (graceful degradation on any error) +""" + +import json +import os +import sys +import time + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) + +from lib.bank import derive_bank_id, ensure_bank_mission +from lib.client import HindsightClient +from lib.config import debug_log, load_config +from lib.content import ( + compose_recall_query, + format_current_time, + format_memories, + read_transcript, + truncate_recall_query, +) +from lib.daemon import get_api_url +from lib.state import write_state + +LAST_RECALL_STATE = "last_recall.json" + + +def main(): + config = load_config() + + if not config.get("autoRecall"): + debug_log(config, "Auto-recall disabled, exiting") + return + + # Read hook input from stdin + try: + hook_input = json.load(sys.stdin) + except (json.JSONDecodeError, EOFError): + print("[Hindsight] Failed to read hook input", file=sys.stderr) + return + + debug_log(config, f"Hook input keys: {list(hook_input.keys())}") + + # Extract user query — accept both "prompt" and "user_prompt" defensively + prompt = (hook_input.get("prompt") or hook_input.get("user_prompt") or "").strip() + if not prompt or len(prompt) < 5: + debug_log(config, "Prompt too short for recall, skipping") + return + + def _dbg(*a): + debug_log(config, *a) + + try: + api_url = get_api_url(config, debug_fn=_dbg, allow_daemon_start=False) + except RuntimeError as e: + print(f"[Hindsight] {e}", file=sys.stderr) + return + + api_token = config.get("hindsightApiToken") + try: + client = HindsightClient(api_url, api_token) + except ValueError as e: + print(f"[Hindsight] Invalid API URL: {e}", file=sys.stderr) + return + + bank_id = derive_bank_id(hook_input, config) + ensure_bank_mission(client, bank_id, config, debug_fn=_dbg) + + # Multi-turn query composition + recall_context_turns = config.get("recallContextTurns", 1) + recall_max_query_chars = config.get("recallMaxQueryChars", 800) + recall_roles = config.get("recallRoles", ["user", "assistant"]) + + if recall_context_turns > 1: + transcript_path = hook_input.get("transcript_path", "") + messages = read_transcript(transcript_path) + debug_log(config, f"Multi-turn context: {recall_context_turns} turns, {len(messages)} messages") + query = compose_recall_query(prompt, messages, recall_context_turns, recall_roles) + else: + query = prompt + + query = truncate_recall_query(query, prompt, recall_max_query_chars) + if len(query) > recall_max_query_chars: + query = query[:recall_max_query_chars] + + current_time = format_current_time() + preamble = config.get("recallPromptPreamble", "") + + debug_log(config, f"Recalling from bank '{bank_id}', query length: {len(query)}") + try: + response = client.recall( + bank_id=bank_id, + query=query, + max_tokens=config.get("recallMaxTokens", 1024), + budget=config.get("recallBudget", "mid"), + types=config.get("recallTypes"), + timeout=10, + ) + except Exception as e: + print(f"[Hindsight] Recall failed: {e}", file=sys.stderr) + return + + results = response.get("results", []) + if not results: + debug_log(config, "No memories found") + return + + debug_log(config, f"Injecting {len(results)} memories") + + memories_formatted = format_memories(results) + + context_message = ( + f"\n" + f"{preamble}\n" + f"Current time - {current_time}\n\n" + f"{memories_formatted}\n" + f"" + ) + + write_state( + LAST_RECALL_STATE, + { + "context": context_message, + "saved_at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), + "bank_id": bank_id, + "result_count": len(results), + }, + ) + + # Output JSON for Codex hook system + output = { + "hookSpecificOutput": { + "hookEventName": "UserPromptSubmit", + "additionalContext": context_message, + } + } + json.dump(output, sys.stdout) + + +if __name__ == "__main__": + try: + main() + except Exception as e: + print(f"[Hindsight] Unexpected error in recall: {e}", file=sys.stderr) + try: + from lib.config import load_config + + sys.exit(2 if load_config().get("debug") else 0) + except Exception: + sys.exit(0) diff --git a/hindsight-integrations/codex/scripts/retain.py b/hindsight-integrations/codex/scripts/retain.py new file mode 100755 index 00000000..d3c27e58 --- /dev/null +++ b/hindsight-integrations/codex/scripts/retain.py @@ -0,0 +1,186 @@ +#!/usr/bin/env python3 +"""Auto-retain hook for Stop event. + +Fires after each agent turn. Reads the Codex session transcript and stores +the conversation into Hindsight memory for future recall. + +Flow: + 1. Read hook input from stdin (session_id, transcript_path, cwd) + 2. Read conversation transcript from transcript_path + 3. Apply chunked retention logic (retainEveryNTurns + overlap window) + 4. Resolve API URL (external, existing local, or auto-start daemon) + 5. Derive bank ID and ensure mission + 6. Format transcript (strip memory tags, filter roles) + 7. POST to Hindsight retain API + +Exit codes: + 0 — always (graceful degradation on any error) +""" + +import json +import os +import sys +import time + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) + +from lib.bank import derive_bank_id, ensure_bank_mission +from lib.client import HindsightClient +from lib.config import debug_log, load_config +from lib.content import ( + prepare_retention_transcript, + read_transcript, + slice_last_turns_by_user_boundary, +) +from lib.daemon import get_api_url +from lib.state import increment_turn_count + + +def main(): + config = load_config() + + if not config.get("autoRetain"): + debug_log(config, "Auto-retain disabled, exiting") + return + + # Read hook input from stdin + try: + hook_input = json.load(sys.stdin) + except (json.JSONDecodeError, EOFError): + print("[Hindsight] Failed to read hook input", file=sys.stderr) + return + + debug_log(config, f"Stop hook input keys: {list(hook_input.keys())}") + + session_id = hook_input.get("session_id", "unknown") + transcript_path = hook_input.get("transcript_path", "") + + # Read full transcript + all_messages = read_transcript(transcript_path) + if not all_messages: + debug_log(config, "No messages in transcript, skipping retain") + return + + debug_log(config, f"Read {len(all_messages)} messages from transcript") + + # Retention mode: full session (default) or chunked (legacy) + retain_mode = config.get("retainMode", "full-session") + retain_every_n = max(1, config.get("retainEveryNTurns", 1)) + retain_full_window = False + messages_to_retain = all_messages + + # Respect retainEveryNTurns in both modes + if retain_every_n > 1: + turn_count = increment_turn_count(session_id) + if turn_count % retain_every_n != 0: + next_at = ((turn_count // retain_every_n) + 1) * retain_every_n + debug_log(config, f"Turn {turn_count}/{retain_every_n}, skipping retain (next at turn {next_at})") + return + + if retain_mode == "chunked" and retain_every_n > 1: + overlap_turns = config.get("retainOverlapTurns", 0) + window_turns = retain_every_n + overlap_turns + messages_to_retain = slice_last_turns_by_user_boundary(all_messages, window_turns) + retain_full_window = True + debug_log( + config, + f"Chunked retain firing (window: {window_turns} turns, {len(messages_to_retain)} messages)", + ) + else: + retain_full_window = True + debug_log(config, f"Full session retain: {len(all_messages)} messages") + + # Format transcript + retain_roles = config.get("retainRoles", ["user", "assistant"]) + transcript, message_count = prepare_retention_transcript( + messages_to_retain, retain_roles, retain_full_window + ) + + if not transcript: + debug_log(config, "Empty transcript after formatting, skipping retain") + return + + # Resolve API URL + def _dbg(*a): + debug_log(config, *a) + + try: + api_url = get_api_url(config, debug_fn=_dbg, allow_daemon_start=True) + except RuntimeError as e: + print(f"[Hindsight] {e}", file=sys.stderr) + return + + api_token = config.get("hindsightApiToken") + try: + client = HindsightClient(api_url, api_token) + except ValueError as e: + print(f"[Hindsight] Invalid API URL: {e}", file=sys.stderr) + return + + # Derive bank ID and ensure mission + bank_id = derive_bank_id(hook_input, config) + ensure_bank_mission(client, bank_id, config, debug_fn=_dbg) + + # Document ID: use session_id so the same session always upserts. + # In chunked mode, append timestamp to create distinct documents per chunk. + if retain_mode == "chunked" and retain_every_n > 1: + document_id = f"{session_id}-{int(time.time() * 1000)}" + else: + document_id = session_id + + # Resolve template variables in tags and metadata + template_vars = { + "session_id": session_id, + "bank_id": bank_id, + "timestamp": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), + } + + def _resolve_template(value: str) -> str: + for k, v in template_vars.items(): + value = value.replace(f"{{{k}}}", v) + return value + + raw_tags = config.get("retainTags", []) + tags = [_resolve_template(t) for t in raw_tags] if raw_tags else None + + metadata = { + "retained_at": template_vars["timestamp"], + "message_count": str(message_count), + "session_id": session_id, + } + for k, v in config.get("retainMetadata", {}).items(): + metadata[k] = _resolve_template(str(v)) + + debug_log( + config, f"Retaining to bank '{bank_id}', doc '{document_id}', {message_count} messages, {len(transcript)} chars" + ) + if tags: + debug_log(config, f"Tags: {tags}") + + # POST to Hindsight retain API + try: + response = client.retain( + bank_id=bank_id, + content=transcript, + document_id=document_id, + context=config.get("retainContext", "codex"), + metadata=metadata, + tags=tags, + timeout=15, + ) + debug_log(config, f"Retain response: {json.dumps(response)[:200]}") + except Exception as e: + print(f"[Hindsight] Retain failed: {e}", file=sys.stderr) + + +if __name__ == "__main__": + try: + main() + except Exception as e: + print(f"[Hindsight] Unexpected error in retain: {e}", file=sys.stderr) + try: + from lib.config import load_config + + sys.exit(2 if load_config().get("debug") else 0) + except Exception: + sys.exit(0) diff --git a/hindsight-integrations/codex/scripts/session_start.py b/hindsight-integrations/codex/scripts/session_start.py new file mode 100755 index 00000000..34b0fa47 --- /dev/null +++ b/hindsight-integrations/codex/scripts/session_start.py @@ -0,0 +1,53 @@ +#!/usr/bin/env python3 +"""SessionStart hook: health check and daemon pre-start. + +Fires once when a Codex session begins. Verifies the Hindsight server is +reachable, and kicks off a background daemon pre-start if not — so it's +ready by the first recall or retain hook. +""" + +import json +import os +import sys + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) + +from lib.client import HindsightClient +from lib.config import debug_log, load_config +from lib.daemon import get_api_url, prestart_daemon_background + + +def main(): + config = load_config() + + if not config.get("autoRecall") and not config.get("autoRetain"): + debug_log(config, "Both autoRecall and autoRetain disabled, skipping session start") + return + + # Consume stdin + try: + hook_input = json.load(sys.stdin) + except (json.JSONDecodeError, EOFError): + hook_input = {} + + debug_log(config, f"SessionStart hook, session: {hook_input.get('session_id', 'unknown')}") + + def _dbg(*a): + debug_log(config, *a) + + try: + api_url = get_api_url(config, debug_fn=_dbg, allow_daemon_start=False) + HindsightClient(api_url, config.get("hindsightApiToken")) + debug_log(config, f"Hindsight server reachable at {api_url}") + except (RuntimeError, ValueError) as e: + debug_log(config, f"Hindsight not running, initiating background pre-start: {e}") + prestart_daemon_background(config, debug_fn=_dbg) + return + + +if __name__ == "__main__": + try: + main() + except Exception as e: + print(f"[Hindsight] SessionStart error: {e}", file=sys.stderr) + sys.exit(0) diff --git a/hindsight-integrations/codex/settings.json b/hindsight-integrations/codex/settings.json new file mode 100644 index 00000000..3b092e42 --- /dev/null +++ b/hindsight-integrations/codex/settings.json @@ -0,0 +1,35 @@ +{ + "hindsightApiUrl": "", + "bankId": "codex", + "bankMission": "You are a Codex AI coding assistant. Focus on technical decisions, code changes, debugging sessions, and project context relevant to the user's work.", + "retainMission": "Extract technical decisions, code patterns, debugging solutions, user preferences, project context, and architectural choices. Ignore routine greetings and transient operational details.", + "autoRecall": true, + "autoRetain": true, + "retainMode": "full-session", + "recallBudget": "mid", + "recallMaxTokens": 1024, + "recallTypes": ["world", "experience"], + "recallContextTurns": 1, + "recallMaxQueryChars": 800, + "recallRoles": ["user", "assistant"], + "recallPromptPreamble": "Relevant memories from past conversations (prioritize recent when conflicting). Only use memories that are directly useful to continue this conversation; ignore the rest:", + "retainRoles": ["user", "assistant"], + "retainEveryNTurns": 10, + "retainOverlapTurns": 2, + "retainTags": ["{session_id}"], + "retainMetadata": {}, + "retainContext": "codex", + "hindsightApiToken": null, + "apiPort": 9077, + "daemonIdleTimeout": 0, + "embedVersion": "latest", + "embedPackagePath": null, + "bankIdPrefix": "", + "dynamicBankId": false, + "dynamicBankGranularity": ["agent", "project"], + "agentName": "codex", + "llmProvider": null, + "llmModel": null, + "llmApiKeyEnv": null, + "debug": false +} diff --git a/hindsight-integrations/codex/tests/conftest.py b/hindsight-integrations/codex/tests/conftest.py new file mode 100644 index 00000000..af096cc4 --- /dev/null +++ b/hindsight-integrations/codex/tests/conftest.py @@ -0,0 +1,90 @@ +"""Shared fixtures for Hindsight Codex plugin tests.""" + +import io +import json +import os +import sys + +import pytest + +# Make scripts/ importable as the root — the hook scripts do: +# sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +# so lib.* imports resolve relative to scripts/ +SCRIPTS_DIR = os.path.join(os.path.dirname(__file__), "..", "scripts") +if SCRIPTS_DIR not in sys.path: + sys.path.insert(0, os.path.abspath(SCRIPTS_DIR)) + + +def make_hook_input( + prompt="What is the capital of France?", + session_id="sess-abc123", + cwd="/home/user/myproject", + transcript_path="", +): + return { + "prompt": prompt, + "session_id": session_id, + "cwd": cwd, + "transcript_path": transcript_path, + } + + +def make_transcript_file(tmp_path, messages, codex_format=False): + """Write messages as a JSONL transcript file. + + By default writes flat format {role, content} which read_transcript() accepts. + Set codex_format=True to write actual Codex response_item format. + """ + f = tmp_path / "rollout-test.jsonl" + lines = [] + for msg in messages: + if codex_format: + role = msg["role"] + text = msg["content"] + content_type = "input_text" if role == "user" else "output_text" + entry = { + "type": "response_item", + "payload": { + "type": "message", + "role": role, + "content": [{"type": content_type, "text": text}], + }, + } + if role == "assistant": + entry["payload"]["phase"] = "final_answer" + lines.append(json.dumps(entry)) + else: + lines.append(json.dumps(msg)) + f.write_text("\n".join(lines)) + return str(f) + + +def make_memory(text, mem_type="experience", mentioned_at="2024-01-15"): + return {"text": text, "type": mem_type, "mentioned_at": mentioned_at} + + +def make_user_config(tmp_path, overrides=None): + """Write a ~/.hindsight/codex.json in tmp_path with test defaults.""" + hindsight_dir = tmp_path / ".hindsight" + hindsight_dir.mkdir(exist_ok=True) + config = {"retainEveryNTurns": 1} + if overrides: + config.update(overrides) + (hindsight_dir / "codex.json").write_text(json.dumps(config)) + + +class FakeHTTPResponse: + """Minimal urllib response mock.""" + + def __init__(self, data: dict, status: int = 200): + self.status = status + self._data = json.dumps(data).encode() + + def read(self): + return self._data + + def __enter__(self): + return self + + def __exit__(self, *_): + pass diff --git a/hindsight-integrations/codex/tests/test_content.py b/hindsight-integrations/codex/tests/test_content.py new file mode 100644 index 00000000..4406edf8 --- /dev/null +++ b/hindsight-integrations/codex/tests/test_content.py @@ -0,0 +1,364 @@ +"""Tests for lib/content.py — pure content-processing functions.""" + +import json +import os +import sys + +import pytest + +from lib.content import ( + compose_recall_query, + format_memories, + prepare_retention_transcript, + read_transcript, + slice_last_turns_by_user_boundary, + strip_memory_tags, + truncate_recall_query, +) + + +# --------------------------------------------------------------------------- +# strip_memory_tags +# --------------------------------------------------------------------------- + + +class TestStripMemoryTags: + def test_strips_hindsight_memories_block(self): + raw = "before\nsecret\nafter" + result = strip_memory_tags(raw) + assert "hindsight_memories" not in result + assert "before" in result + assert "after" in result + + def test_strips_relevant_memories_block(self): + raw = "text old stuff text" + result = strip_memory_tags(raw) + assert "relevant_memories" not in result + assert "old stuff" not in result + + def test_passthrough_clean_text(self): + raw = "no memory tags here" + assert strip_memory_tags(raw) == raw + + def test_strips_multiline_block(self): + raw = "\n- mem1\n- mem2\n" + assert strip_memory_tags(raw).strip() == "" + + +# --------------------------------------------------------------------------- +# read_transcript — flat format +# --------------------------------------------------------------------------- + + +def _write_jsonl(tmp_path, entries): + f = tmp_path / "transcript.jsonl" + f.write_text("\n".join(json.dumps(e) for e in entries)) + return str(f) + + +class TestReadTranscriptFlat: + def test_reads_flat_format(self, tmp_path): + path = _write_jsonl(tmp_path, [ + {"role": "user", "content": "hello"}, + {"role": "assistant", "content": "hi there"}, + ]) + msgs = read_transcript(path) + assert len(msgs) == 2 + assert msgs[0] == {"role": "user", "content": "hello"} + + def test_returns_empty_for_missing_file(self): + assert read_transcript("/nonexistent/path.jsonl") == [] + + def test_returns_empty_for_empty_string(self): + assert read_transcript("") == [] + + +class TestReadTranscriptCodexFormat: + def test_reads_codex_response_item_format(self, tmp_path): + entries = [ + { + "type": "response_item", + "payload": { + "type": "message", + "role": "user", + "content": [{"type": "input_text", "text": "What is Python?"}], + }, + }, + { + "type": "response_item", + "payload": { + "type": "message", + "role": "assistant", + "content": [{"type": "output_text", "text": "A programming language."}], + "phase": "final_answer", + }, + }, + ] + path = _write_jsonl(tmp_path, entries) + msgs = read_transcript(path) + assert len(msgs) == 2 + assert msgs[0]["role"] == "user" + assert msgs[0]["content"] == "What is Python?" + assert msgs[1]["role"] == "assistant" + assert msgs[1]["content"] == "A programming language." + + def test_skips_non_final_answer_assistant_messages(self, tmp_path): + entries = [ + { + "type": "response_item", + "payload": { + "type": "message", + "role": "assistant", + "content": [{"type": "output_text", "text": "thinking..."}], + "phase": "reasoning", + }, + }, + { + "type": "response_item", + "payload": { + "type": "message", + "role": "assistant", + "content": [{"type": "output_text", "text": "The answer is 42."}], + "phase": "final_answer", + }, + }, + ] + path = _write_jsonl(tmp_path, entries) + msgs = read_transcript(path) + assert len(msgs) == 1 + assert msgs[0]["content"] == "The answer is 42." + + def test_skips_non_message_response_items(self, tmp_path): + entries = [ + {"type": "response_item", "payload": {"type": "tool_call", "name": "Bash"}}, + { + "type": "response_item", + "payload": { + "type": "message", + "role": "user", + "content": [{"type": "input_text", "text": "hello"}], + }, + }, + ] + path = _write_jsonl(tmp_path, entries) + msgs = read_transcript(path) + assert len(msgs) == 1 + assert msgs[0]["role"] == "user" + + def test_skips_invalid_roles(self, tmp_path): + entries = [ + { + "type": "response_item", + "payload": { + "type": "message", + "role": "system", + "content": [{"type": "input_text", "text": "system message"}], + }, + }, + ] + path = _write_jsonl(tmp_path, entries) + msgs = read_transcript(path) + assert len(msgs) == 0 + + def test_skips_blank_lines_gracefully(self, tmp_path): + f = tmp_path / "transcript.jsonl" + f.write_text('\n{"role": "user", "content": "hi"}\n\n{"role": "assistant", "content": "hey"}\n') + msgs = read_transcript(str(f)) + assert len(msgs) == 2 + + +# --------------------------------------------------------------------------- +# slice_last_turns_by_user_boundary +# --------------------------------------------------------------------------- + + +def _msgs(*pairs): + return [{"role": r, "content": c} for r, c in pairs] + + +class TestSliceLastTurnsByUserBoundary: + def test_returns_all_when_fewer_turns_than_requested(self): + msgs = _msgs(("user", "hi"), ("assistant", "hello")) + assert slice_last_turns_by_user_boundary(msgs, 5) == msgs + + def test_slices_to_last_one_turn(self): + msgs = _msgs( + ("user", "first"), ("assistant", "a1"), + ("user", "second"), ("assistant", "a2"), + ) + result = slice_last_turns_by_user_boundary(msgs, 1) + assert result[0]["content"] == "second" + assert len(result) == 2 + + def test_slices_to_last_two_turns(self): + msgs = _msgs( + ("user", "u1"), ("assistant", "a1"), + ("user", "u2"), ("assistant", "a2"), + ("user", "u3"), ("assistant", "a3"), + ) + result = slice_last_turns_by_user_boundary(msgs, 2) + assert result[0]["content"] == "u2" + assert len(result) == 4 + + def test_empty_list_returns_empty(self): + assert slice_last_turns_by_user_boundary([], 3) == [] + + def test_zero_turns_returns_empty(self): + assert slice_last_turns_by_user_boundary(_msgs(("user", "hi")), 0) == [] + + def test_non_list_returns_empty(self): + assert slice_last_turns_by_user_boundary(None, 1) == [] + + +# --------------------------------------------------------------------------- +# compose_recall_query +# --------------------------------------------------------------------------- + + +class TestComposeRecallQuery: + def test_single_turn_returns_latest_only(self): + msgs = _msgs(("user", "previous"), ("assistant", "reply")) + result = compose_recall_query("new query", msgs, recall_context_turns=1) + assert result == "new query" + + def test_multi_turn_includes_prior_context(self): + msgs = _msgs(("user", "prior question"), ("assistant", "prior answer")) + result = compose_recall_query("current question", msgs, recall_context_turns=2) + assert "Prior context:" in result + assert "prior question" in result + assert "current question" in result + + def test_skips_duplicate_of_latest_query(self): + msgs = _msgs(("user", "same question"), ("assistant", "answer")) + result = compose_recall_query("same question", msgs, recall_context_turns=2) + assert result.count("same question") == 1 + + def test_empty_messages_returns_latest(self): + result = compose_recall_query("query", [], recall_context_turns=3) + assert result == "query" + + def test_strips_memory_tags_from_context(self): + msgs = _msgs(("user", "secret actual question")) + result = compose_recall_query("now", msgs, recall_context_turns=2) + assert "hindsight_memories" not in result + assert "secret" not in result + + def test_filters_by_recall_roles(self): + msgs = _msgs(("user", "user msg"), ("assistant", "assistant msg")) + result = compose_recall_query("query", msgs, recall_context_turns=2, recall_roles=["user"]) + assert "user msg" in result + assert "assistant msg" not in result + + +# --------------------------------------------------------------------------- +# truncate_recall_query +# --------------------------------------------------------------------------- + + +class TestTruncateRecallQuery: + def test_short_query_unchanged(self): + q = "short" + assert truncate_recall_query(q, q, max_chars=100) == q + + def test_plain_query_truncated_to_max(self): + q = "x" * 50 + result = truncate_recall_query(q, q, max_chars=20) + assert len(result) <= 20 + + def test_preserves_latest_when_context_dropped(self): + latest = "final question" + query = f"Prior context:\n\nuser: old stuff\nassistant: old reply\n\n{latest}" + result = truncate_recall_query(query, latest, max_chars=30) + assert latest in result + + def test_drops_oldest_context_lines_first(self): + latest = "latest" + query = f"Prior context:\n\nuser: oldest\nassistant: old\nuser: newer\n\n{latest}" + max_chars = len(f"Prior context:\n\nnewer\n\n{latest}") + 5 + result = truncate_recall_query(query, latest, max_chars=max_chars) + if "Prior context:" in result: + assert "oldest" not in result + + def test_zero_max_returns_query_unchanged(self): + q = "anything" + assert truncate_recall_query(q, q, max_chars=0) == q + + +# --------------------------------------------------------------------------- +# format_memories +# --------------------------------------------------------------------------- + + +class TestFormatMemories: + def test_formats_single_memory(self): + mems = [{"text": "Paris is the capital", "type": "world", "mentioned_at": "2024-01-01"}] + result = format_memories(mems) + assert "Paris is the capital" in result + assert "[world]" in result + assert "(2024-01-01)" in result + + def test_formats_multiple_memories(self): + mems = [ + {"text": "mem1", "type": "experience", "mentioned_at": "2024-01-01"}, + {"text": "mem2", "type": "world", "mentioned_at": "2024-02-01"}, + ] + result = format_memories(mems) + assert "mem1" in result + assert "mem2" in result + + def test_empty_list_returns_empty_string(self): + assert format_memories([]) == "" + + def test_missing_optional_fields_graceful(self): + mems = [{"text": "bare memory"}] + result = format_memories(mems) + assert "bare memory" in result + + +# --------------------------------------------------------------------------- +# prepare_retention_transcript +# --------------------------------------------------------------------------- + + +class TestPrepareRetentionTranscript: + def test_formats_last_turn_by_default(self): + msgs = _msgs(("user", "old"), ("assistant", "old reply"), ("user", "new"), ("assistant", "new reply")) + transcript, count = prepare_retention_transcript(msgs, retain_full_window=False) + assert "new" in transcript + assert "new reply" in transcript + assert count == 2 + + def test_full_window_retains_all(self): + msgs = _msgs(("user", "msg1"), ("assistant", "reply1"), ("user", "msg2"), ("assistant", "reply2")) + transcript, count = prepare_retention_transcript(msgs, retain_full_window=True) + assert "msg1" in transcript + assert "msg2" in transcript + assert count == 4 + + def test_strips_memory_tags(self): + msgs = _msgs(("user", "leaked actual question")) + transcript, _ = prepare_retention_transcript(msgs, retain_full_window=True) + assert "leaked" not in transcript + assert "actual question" in transcript + + def test_filters_by_retain_roles(self): + msgs = _msgs(("user", "user msg"), ("assistant", "assistant msg")) + transcript, _ = prepare_retention_transcript(msgs, retain_roles=["user"], retain_full_window=True) + assert "user msg" in transcript + assert "assistant msg" not in transcript + + def test_empty_messages_returns_none(self): + result, count = prepare_retention_transcript([]) + assert result is None + assert count == 0 + + def test_role_markers_present(self): + msgs = _msgs(("user", "hello")) + transcript, _ = prepare_retention_transcript(msgs, retain_full_window=True) + assert "[role: user]" in transcript + assert "[user:end]" in transcript + + def test_no_user_message_returns_none(self): + msgs = [{"role": "assistant", "content": "only assistant"}] + result, _ = prepare_retention_transcript(msgs, retain_full_window=False) + assert result is None diff --git a/hindsight-integrations/codex/tests/test_hooks.py b/hindsight-integrations/codex/tests/test_hooks.py new file mode 100644 index 00000000..0dda21b9 --- /dev/null +++ b/hindsight-integrations/codex/tests/test_hooks.py @@ -0,0 +1,319 @@ +"""End-to-end tests for recall.py and retain.py hook scripts. + +Mocks the Codex hook runtime: + - stdin → io.StringIO(json.dumps(hook_input)) + - stdout → io.StringIO() captured for assertions + - urllib.request.urlopen → fake HTTP responses + - HOME → tmp_path (isolates ~/.hindsight/codex.json and state) +""" + +import importlib +import io +import json +import os +import sys +from unittest.mock import patch + +import pytest + +from conftest import FakeHTTPResponse, make_hook_input, make_memory, make_transcript_file, make_user_config + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + + +def _run_hook(module_name, hook_input, monkeypatch, tmp_path, urlopen_side_effect=None, user_config=None): + """Import and run a hook script's main() with mocked stdin/stdout/HTTP.""" + # Isolate HOME so ~/.hindsight/codex.json and state land in tmp_path + monkeypatch.setenv("HOME", str(tmp_path)) + + # Strip real HINDSIGHT_* env vars + for k in list(os.environ): + if k.startswith("HINDSIGHT_"): + monkeypatch.delenv(k, raising=False) + + # Set required API URL via env var + monkeypatch.setenv("HINDSIGHT_API_URL", "http://fake:9077") + + # Write user config (enables retain on every turn + any overrides) + cfg = {"retainEveryNTurns": 1, "autoRecall": True, "autoRetain": True} + if user_config: + cfg.update(user_config) + make_user_config(tmp_path, cfg) + + stdin_data = io.StringIO(json.dumps(hook_input)) + stdout_capture = io.StringIO() + + # Force reimport so the module picks up patched env + scripts_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "scripts")) + spec = importlib.util.spec_from_file_location( + module_name + "_fresh", os.path.join(scripts_dir, f"{module_name}.py") + ) + mod = importlib.util.module_from_spec(spec) + + default_response = FakeHTTPResponse({"results": []}) + side_effect = urlopen_side_effect or (lambda *a, **kw: default_response) + + with ( + patch("sys.stdin", stdin_data), + patch("sys.stdout", stdout_capture), + patch("urllib.request.urlopen", side_effect=side_effect), + ): + spec.loader.exec_module(mod) + mod.main() + + return stdout_capture.getvalue() + + +# --------------------------------------------------------------------------- +# recall hook +# --------------------------------------------------------------------------- + + +class TestRecallHook: + def test_outputs_additional_context_when_memories_found(self, monkeypatch, tmp_path): + memory = make_memory("Paris is the capital of France", "world") + response = FakeHTTPResponse({"results": [memory]}) + + hook_input = make_hook_input(prompt="What is the capital of France?") + output = _run_hook("recall", hook_input, monkeypatch, tmp_path, + urlopen_side_effect=lambda *a, **kw: response) + + data = json.loads(output) + context = data["hookSpecificOutput"]["additionalContext"] + assert "Paris is the capital of France" in context + assert "" in context + + def test_no_output_when_no_memories(self, monkeypatch, tmp_path): + hook_input = make_hook_input(prompt="hello there world") + output = _run_hook("recall", hook_input, monkeypatch, tmp_path) + assert output.strip() == "" + + def test_no_output_for_short_prompt(self, monkeypatch, tmp_path): + hook_input = make_hook_input(prompt="hi") + output = _run_hook("recall", hook_input, monkeypatch, tmp_path) + assert output.strip() == "" + + def test_graceful_on_api_error(self, monkeypatch, tmp_path): + def raise_error(*a, **kw): + raise OSError("connection refused") + + hook_input = make_hook_input(prompt="What is my project about?") + output = _run_hook("recall", hook_input, monkeypatch, tmp_path, urlopen_side_effect=raise_error) + assert output.strip() == "" + + def test_output_format_matches_codex_spec(self, monkeypatch, tmp_path): + memory = make_memory("User prefers Python") + response = FakeHTTPResponse({"results": [memory]}) + + hook_input = make_hook_input(prompt="What language should I use?") + output = _run_hook("recall", hook_input, monkeypatch, tmp_path, + urlopen_side_effect=lambda *a, **kw: response) + + data = json.loads(output) + assert data["hookSpecificOutput"]["hookEventName"] == "UserPromptSubmit" + assert "additionalContext" in data["hookSpecificOutput"] + + def test_multi_turn_context_from_transcript(self, monkeypatch, tmp_path): + """When recallContextTurns > 1, prior transcript is included in query.""" + messages = [ + {"role": "user", "content": "I use Python for all my scripts"}, + {"role": "assistant", "content": "Noted!"}, + ] + transcript = make_transcript_file(tmp_path, messages) + + captured_body = {} + + def capture_and_respond(req, timeout=None): + if "/recall" in req.full_url: + captured_body["body"] = json.loads(req.data.decode()) + return FakeHTTPResponse({"results": []}) + + hook_input = make_hook_input(prompt="What language should I use?", transcript_path=transcript) + _run_hook("recall", hook_input, monkeypatch, tmp_path, + urlopen_side_effect=capture_and_respond, + user_config={"recallContextTurns": 2}) + + if "body" in captured_body: + assert "Python" in captured_body["body"].get("query", "") + + def test_disabled_auto_recall_produces_no_output(self, monkeypatch, tmp_path): + hook_input = make_hook_input(prompt="What is the capital of France?") + output = _run_hook("recall", hook_input, monkeypatch, tmp_path, + user_config={"autoRecall": False}) + assert output.strip() == "" + + +# --------------------------------------------------------------------------- +# retain hook +# --------------------------------------------------------------------------- + + +class TestRetainHook: + def test_posts_transcript_to_hindsight(self, monkeypatch, tmp_path): + messages = [{"role": "user", "content": "hello"}, {"role": "assistant", "content": "world"}] + transcript = make_transcript_file(tmp_path, messages) + + captured = {} + + def capture(req, timeout=None): + if "/memories" in req.full_url and "/recall" not in req.full_url: + captured["body"] = json.loads(req.data.decode()) + return FakeHTTPResponse({"status": "accepted"}) + + hook_input = make_hook_input(transcript_path=transcript) + _run_hook("retain", hook_input, monkeypatch, tmp_path, urlopen_side_effect=capture) + + assert "body" in captured, "retain API was not called" + assert "hello" in captured["body"]["items"][0]["content"] + + def test_no_retain_on_empty_transcript(self, monkeypatch, tmp_path): + hook_input = make_hook_input(transcript_path="/nonexistent/transcript.jsonl") + captured = {} + + def capture(req, timeout=None): + if "/memories" in req.full_url: + captured["called"] = True + return FakeHTTPResponse({}) + + _run_hook("retain", hook_input, monkeypatch, tmp_path, urlopen_side_effect=capture) + assert "called" not in captured + + def test_strips_memory_tags_before_retaining(self, monkeypatch, tmp_path): + messages = [ + {"role": "user", "content": "old memories actual question"}, + {"role": "assistant", "content": "sure!"}, + ] + transcript = make_transcript_file(tmp_path, messages) + captured = {} + + def capture(req, timeout=None): + if "/memories" in req.full_url and "/recall" not in req.full_url: + captured["body"] = json.loads(req.data.decode()) + return FakeHTTPResponse({}) + + hook_input = make_hook_input(transcript_path=transcript) + _run_hook("retain", hook_input, monkeypatch, tmp_path, urlopen_side_effect=capture) + + if "body" in captured: + content = captured["body"]["items"][0]["content"] + assert "old memories" not in content + assert "actual question" in content + + def test_retain_posts_async_true(self, monkeypatch, tmp_path): + messages = [{"role": "user", "content": "hello"}, {"role": "assistant", "content": "world"}] + transcript = make_transcript_file(tmp_path, messages) + captured = {} + + def capture(req, timeout=None): + if "/memories" in req.full_url and "/recall" not in req.full_url: + captured["body"] = json.loads(req.data.decode()) + return FakeHTTPResponse({}) + + hook_input = make_hook_input(transcript_path=transcript) + _run_hook("retain", hook_input, monkeypatch, tmp_path, urlopen_side_effect=capture) + + if "body" in captured: + assert captured["body"].get("async") is True + + def test_retain_includes_codex_context_label(self, monkeypatch, tmp_path): + messages = [{"role": "user", "content": "hello"}, {"role": "assistant", "content": "world"}] + transcript = make_transcript_file(tmp_path, messages) + captured = {} + + def capture(req, timeout=None): + if "/memories" in req.full_url and "/recall" not in req.full_url: + captured["body"] = json.loads(req.data.decode()) + return FakeHTTPResponse({}) + + hook_input = make_hook_input(transcript_path=transcript) + _run_hook("retain", hook_input, monkeypatch, tmp_path, urlopen_side_effect=capture) + + if "body" in captured: + assert captured["body"]["items"][0]["context"] == "codex" + + def test_retain_skips_below_every_n_turns_threshold(self, monkeypatch, tmp_path): + messages = [{"role": "user", "content": "hello"}, {"role": "assistant", "content": "world"}] + transcript = make_transcript_file(tmp_path, messages) + captured = {} + + def capture(req, timeout=None): + if "/memories" in req.full_url and "/recall" not in req.full_url: + captured["called"] = True + return FakeHTTPResponse({}) + + hook_input = make_hook_input(transcript_path=transcript) + # retainEveryNTurns=3 — first call should be skipped + _run_hook("retain", hook_input, monkeypatch, tmp_path, + urlopen_side_effect=capture, + user_config={"retainEveryNTurns": 3}) + assert "called" not in captured + + def test_retain_uses_session_id_as_document_id(self, monkeypatch, tmp_path): + messages = [ + {"role": "user", "content": "question"}, {"role": "assistant", "content": "answer"}, + ] + transcript = make_transcript_file(tmp_path, messages) + hook_input = make_hook_input(transcript_path=transcript, session_id="sess-doc-test") + captured = {} + + def capture(req, timeout=None): + if "/memories" in req.full_url and "/recall" not in req.full_url: + captured["body"] = json.loads(req.data.decode()) + return FakeHTTPResponse({}) + + _run_hook("retain", hook_input, monkeypatch, tmp_path, urlopen_side_effect=capture) + + assert "body" in captured + assert captured["body"]["items"][0]["document_id"] == "sess-doc-test" + + def test_graceful_on_retain_api_error(self, monkeypatch, tmp_path): + messages = [{"role": "user", "content": "test"}, {"role": "assistant", "content": "response"}] + transcript = make_transcript_file(tmp_path, messages) + hook_input = make_hook_input(transcript_path=transcript) + + def raise_error(req, timeout=None): + if "/memories" in req.full_url: + raise OSError("connection refused") + return FakeHTTPResponse({}) + + # Should not raise + _run_hook("retain", hook_input, monkeypatch, tmp_path, urlopen_side_effect=raise_error) + + def test_disabled_auto_retain_does_not_call_api(self, monkeypatch, tmp_path): + messages = [{"role": "user", "content": "hello"}] + transcript = make_transcript_file(tmp_path, messages) + hook_input = make_hook_input(transcript_path=transcript) + captured = {} + + def capture(req, timeout=None): + captured["called"] = True + return FakeHTTPResponse({}) + + _run_hook("retain", hook_input, monkeypatch, tmp_path, + urlopen_side_effect=capture, + user_config={"autoRetain": False}) + assert "called" not in captured + + def test_reads_codex_response_item_format(self, monkeypatch, tmp_path): + """Retain should correctly parse the actual Codex on-disk transcript format.""" + messages = [ + {"role": "user", "content": "I like TypeScript"}, + {"role": "assistant", "content": "Great choice!"}, + ] + transcript = make_transcript_file(tmp_path, messages, codex_format=True) + captured = {} + + def capture(req, timeout=None): + if "/memories" in req.full_url and "/recall" not in req.full_url: + captured["body"] = json.loads(req.data.decode()) + return FakeHTTPResponse({}) + + hook_input = make_hook_input(transcript_path=transcript) + _run_hook("retain", hook_input, monkeypatch, tmp_path, urlopen_side_effect=capture) + + assert "body" in captured, "retain API was not called" + content = captured["body"]["items"][0]["content"] + assert "TypeScript" in content