diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index a4352280..302c1f4e 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -150,6 +150,55 @@ jobs: path: hindsight-clients/typescript/*.tgz retention-days: 1 + release-hindsight-all-npm: + runs-on: ubuntu-latest + environment: npm + + steps: + - uses: actions/checkout@v6 + + - name: Set up Node.js + uses: actions/setup-node@v6 + with: + node-version: '22' + registry-url: 'https://registry.npmjs.org' + cache: 'npm' + cache-dependency-path: package-lock.json + + - name: Install dependencies + run: npm ci --workspace=hindsight-all-npm + + - name: Build + run: npm run build --workspace=hindsight-all-npm + + - name: Publish to npm + working-directory: ./hindsight-all-npm + run: | + set +e + OUTPUT=$(npm publish --access public 2>&1) + EXIT_CODE=$? + echo "$OUTPUT" + if [ $EXIT_CODE -ne 0 ]; then + if echo "$OUTPUT" | grep -q "cannot publish over"; then + echo "Package version already published, skipping..." + exit 0 + fi + exit $EXIT_CODE + fi + env: + NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }} + + - name: Pack for GitHub release + working-directory: ./hindsight-all-npm + run: npm pack + + - name: Upload artifacts + uses: actions/upload-artifact@v7 + with: + name: hindsight-all-npm + path: hindsight-all-npm/*.tgz + retention-days: 1 + release-control-plane: runs-on: ubuntu-latest environment: npm @@ -407,7 +456,7 @@ jobs: create-github-release: runs-on: ubuntu-latest - needs: [release-python-packages, release-typescript-client, release-control-plane, release-rust-cli, release-docker-images, release-helm-chart] + needs: [release-python-packages, release-typescript-client, release-hindsight-all-npm, release-control-plane, release-rust-cli, release-docker-images, release-helm-chart] permissions: contents: write @@ -436,6 +485,12 @@ jobs: name: control-plane path: ./artifacts/control-plane + - name: Download hindsight-embed npm wrapper + uses: actions/download-artifact@v8 + with: + name: hindsight-all-npm + path: ./artifacts/hindsight-all-npm + - name: Download Rust CLI (Linux) uses: actions/download-artifact@v8 with: @@ -472,6 +527,8 @@ jobs: cp artifacts/python-packages/hindsight-embed/dist/* release-assets/ || true # TypeScript client cp artifacts/typescript-client/*.tgz release-assets/ || true + # hindsight-embed npm wrapper + cp artifacts/hindsight-all-npm/*.tgz release-assets/ || true # Control Plane cp artifacts/control-plane/*.tgz release-assets/ || true # Rust CLI binaries diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index ac31dccf..5b0c6e45 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -32,6 +32,7 @@ jobs: helm: ${{ steps.filter.outputs.helm }} docs: ${{ steps.filter.outputs.docs }} embed: ${{ steps.filter.outputs.embed }} + all-npm: ${{ steps.filter.outputs.all-npm }} hindsight-all: ${{ steps.filter.outputs.hindsight-all }} integration-tests: ${{ steps.filter.outputs.integration-tests }} integrations-openclaw: ${{ steps.filter.outputs.integrations-openclaw }} @@ -92,6 +93,10 @@ jobs: - '*.md' embed: - 'hindsight-embed/**' + all-npm: + - 'hindsight-all-npm/**' + - 'package.json' + - 'package-lock.json' hindsight-all: - 'hindsight-all/**' integration-tests: @@ -183,12 +188,12 @@ jobs: - name: Build TypeScript client run: npm run build --workspace=hindsight-clients/typescript - build-openclaw-integration: + build-hindsight-all-npm: needs: [detect-changes] if: >- github.event_name != 'pull_request_review' && (github.event_name == 'workflow_dispatch' || - needs.detect-changes.outputs.integrations-openclaw == 'true' || + needs.detect-changes.outputs.all-npm == 'true' || needs.detect-changes.outputs.ci == 'true') runs-on: ubuntu-latest @@ -201,19 +206,71 @@ jobs: uses: actions/setup-node@v6 with: node-version: '22' + cache: 'npm' + cache-dependency-path: package-lock.json - name: Install dependencies + run: npm ci --workspace=hindsight-all-npm + + - name: Run tests + run: npm test --workspace=hindsight-all-npm + + - name: Build + run: npm run build --workspace=hindsight-all-npm + + build-openclaw-integration: + needs: [detect-changes] + if: >- + github.event_name != 'pull_request_review' && + (github.event_name == 'workflow_dispatch' || + needs.detect-changes.outputs.integrations-openclaw == 'true' || + needs.detect-changes.outputs.clients-ts == 'true' || + needs.detect-changes.outputs.all-npm == 'true' || + needs.detect-changes.outputs.ci == 'true') + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v6 + with: + ref: ${{ github.event.pull_request.head.sha || '' }} + + - name: Set up Node.js + uses: actions/setup-node@v6 + with: + node-version: '22' + cache: 'npm' + cache-dependency-path: package-lock.json + + # openclaw depends on two monorepo workspaces via `file:` deps: + # @vectorize-io/hindsight-client and @vectorize-io/hindsight-all. Their + # `dist/` directories are gitignored, so we must build them first. + # Otherwise vitest/tsc in openclaw fails with + # "Failed to resolve entry for package ..." on the value imports. + - name: Install root workspace dependencies + run: npm ci + + - name: Build hindsight-client (openclaw dep) + run: npm run build --workspace=hindsight-clients/typescript + + - name: Build hindsight-all-npm (openclaw dep) + run: npm run build --workspace=hindsight-all-npm + + - name: Install openclaw dependencies working-directory: ./hindsight-integrations/openclaw run: npm ci + # Build must run before tests: one unit test in src/backfill.test.ts + # creates a symlink to `$cwd/dist/backfill.js` and calls realpathSync on + # it via isDirectExecution(). Without a populated dist/ the realpath call + # throws, both paths stay unresolved, and the equality assertion fails. + - name: Build + working-directory: ./hindsight-integrations/openclaw + run: npm run build + - name: Run tests working-directory: ./hindsight-integrations/openclaw run: npm test - - name: Build - working-directory: ./hindsight-integrations/openclaw - run: npm run build - test-claude-code-integration: needs: [detect-changes] if: >- @@ -1525,6 +1582,18 @@ jobs: print('Models downloaded successfully') " + # openclaw depends on @vectorize-io/hindsight-client and + # @vectorize-io/hindsight-all via `file:` — their `dist/` directories are + # gitignored and must be built before openclaw's npm ci copies them. + - name: Install root workspace dependencies + run: npm ci + + - name: Build hindsight-client (openclaw dep) + run: npm run build --workspace=hindsight-clients/typescript + + - name: Build hindsight-all-npm (openclaw dep) + run: npm run build --workspace=hindsight-all-npm + - name: Install openclaw integration dependencies working-directory: ./hindsight-integrations/openclaw run: npm ci @@ -2524,8 +2593,7 @@ jobs: core.setOutput('run_url', runUrl); - name: Report status to PR - if: github.event.pull_request.head.repo.full_name == github.repository - uses: actions/github-script@v7 + uses: actions/github-script@v8 with: script: | await github.rest.repos.createCommitStatus({ @@ -2568,4 +2636,4 @@ jobs: issue_number: prNumber, body, }); - } + } \ No newline at end of file diff --git a/hindsight-all-npm/.gitignore b/hindsight-all-npm/.gitignore new file mode 100644 index 00000000..04f9a0dc --- /dev/null +++ b/hindsight-all-npm/.gitignore @@ -0,0 +1,4 @@ +node_modules +dist +*.tgz +.DS_Store diff --git a/hindsight-all-npm/README.md b/hindsight-all-npm/README.md new file mode 100644 index 00000000..fcbacfe7 --- /dev/null +++ b/hindsight-all-npm/README.md @@ -0,0 +1,80 @@ +# @vectorize-io/hindsight-all + +Node.js equivalent of the Python [`hindsight-all`](https://pypi.org/project/hindsight-all/) package — programmatic lifecycle manager for a local Hindsight daemon. Use this when you want to embed Hindsight in a Node application without hand-rolling subprocess management. + +This package deliberately does **not** ship an HTTP client. Once the daemon is running, talk to it with [`@vectorize-io/hindsight-client`](https://www.npmjs.com/package/@vectorize-io/hindsight-client) against `server.getBaseUrl()`. The two packages compose — one owns the daemon process, the other owns the HTTP API surface. + +## Requirements + +- **Node.js >= 22** — uses global `fetch` and `AbortSignal.timeout`. +- **`uv` / `uvx`** on `PATH` — used to download and run the underlying `hindsight-embed` daemon on first use. Install via . + +## Install + +```bash +npm install @vectorize-io/hindsight-all @vectorize-io/hindsight-client +``` + +## Example + +```ts +import { HindsightServer, consoleLogger } from '@vectorize-io/hindsight-all'; +import { HindsightClient } from '@vectorize-io/hindsight-client'; + +const server = new HindsightServer({ + profile: 'my-app', + port: 9077, + env: { + HINDSIGHT_API_LLM_PROVIDER: 'anthropic', + HINDSIGHT_API_LLM_API_KEY: process.env.ANTHROPIC_API_KEY, + HINDSIGHT_API_LLM_MODEL: 'claude-sonnet-4-20250514', + HINDSIGHT_EMBED_DAEMON_IDLE_TIMEOUT: '0', + }, + logger: consoleLogger, +}); + +await server.start(); + +const client = new HindsightClient({ baseUrl: server.getBaseUrl() }); + +await client.retain('user-123', 'User prefers dark mode and concise answers.', { + documentId: 'pref-2026-04-01', +}); + +const recall = await client.recall('user-123', 'what are the user preferences?'); +console.log(recall.results); + +await server.stop(); +``` + +For a remote Hindsight API, skip `HindsightServer` entirely and just point `HindsightClient` at the remote URL. + +## Open config — forward-compatible with new daemon flags + +`HindsightServerOptions` is designed so every new environment variable or CLI flag in the underlying Hindsight daemon can be used without waiting for a wrapper release: + +- **`env`** accepts an arbitrary `Record`. Every entry is exported into the daemon process and written into the profile config via `--env KEY=VALUE`. +- **`extraProfileCreateArgs`** / **`extraDaemonStartArgs`** append raw args to the respective commands. + +## Development against a local checkout + +If you're hacking on the Python `hindsight-embed` package in the same monorepo, point the server at the local path — it'll use `uv run --directory ` instead of `uvx`: + +```ts +new HindsightServer({ + embedPackagePath: '/path/to/hindsight-embed', + // ... +}); +``` + +## API surface + +- `HindsightServer` — daemon lifecycle (`start`, `stop`, `checkHealth`, `getBaseUrl`, `getProfile`). +- `Logger` interface plus `silentLogger` (default) and `consoleLogger` helpers. +- `getEmbedCommand(opts)` — low-level helper that returns the `[cmd, ...args]` tuple used to invoke the underlying Python CLI. + +For memory operations (retain, recall, reflect, bank management, stats) use [`@vectorize-io/hindsight-client`](https://www.npmjs.com/package/@vectorize-io/hindsight-client). + +## License + +MIT diff --git a/hindsight-all-npm/package.json b/hindsight-all-npm/package.json new file mode 100644 index 00000000..16729168 --- /dev/null +++ b/hindsight-all-npm/package.json @@ -0,0 +1,57 @@ +{ + "name": "@vectorize-io/hindsight-all", + "version": "0.5.0", + "description": "Node.js programmatic lifecycle manager for Hindsight — embeds a local hindsight daemon in a Node application. Pair with @vectorize-io/hindsight-client for memory operations.", + "main": "dist/index.js", + "types": "dist/index.d.ts", + "type": "module", + "exports": { + ".": { + "types": "./dist/index.d.ts", + "import": "./dist/index.js" + } + }, + "keywords": [ + "hindsight", + "hindsight-all", + "memory", + "ai", + "agent", + "long-term-memory", + "llm", + "embedded-server" + ], + "author": "Vectorize ", + "license": "MIT", + "repository": { + "type": "git", + "url": "https://github.com/vectorize-io/hindsight.git", + "directory": "hindsight-all-npm" + }, + "files": [ + "dist", + "README.md" + ], + "scripts": { + "build": "tsup", + "dev": "tsup --watch", + "clean": "rm -rf dist", + "test": "vitest run src", + "test:watch": "vitest src", + "prepublishOnly": "npm run clean && npm run build" + }, + "devDependencies": { + "@types/node": "^22.0.0", + "tsup": "^8.5.1", + "typescript": "^5.7.0", + "vitest": "^4.1.2" + }, + "engines": { + "node": ">=22" + }, + "overrides": { + "rollup": "^4.59.0", + "picomatch": ">=2.3.2 <3.0.0 || >=4.0.4", + "vite": ">=8.0.5" + } +} diff --git a/hindsight-all-npm/src/command.test.ts b/hindsight-all-npm/src/command.test.ts new file mode 100644 index 00000000..b47e45bc --- /dev/null +++ b/hindsight-all-npm/src/command.test.ts @@ -0,0 +1,32 @@ +import { describe, it, expect } from 'vitest'; +import { getEmbedCommand } from './command.js'; + +describe('getEmbedCommand', () => { + it('defaults to uvx hindsight-embed@latest', () => { + expect(getEmbedCommand()).toEqual(['uvx', 'hindsight-embed@latest']); + }); + + it('honours an explicit version', () => { + expect(getEmbedCommand({ embedVersion: '0.5.0' })).toEqual(['uvx', 'hindsight-embed@0.5.0']); + }); + + it('treats an empty version as latest', () => { + expect(getEmbedCommand({ embedVersion: '' })).toEqual(['uvx', 'hindsight-embed@latest']); + }); + + it('uses uv run --directory when a local path is given', () => { + expect(getEmbedCommand({ embedPackagePath: '/abs/path' })).toEqual([ + 'uv', + 'run', + '--directory', + '/abs/path', + 'hindsight-embed', + ]); + }); + + it('local path takes precedence over version', () => { + expect( + getEmbedCommand({ embedPackagePath: '/abs/path', embedVersion: '0.5.0' }), + ).toEqual(['uv', 'run', '--directory', '/abs/path', 'hindsight-embed']); + }); +}); diff --git a/hindsight-all-npm/src/command.ts b/hindsight-all-npm/src/command.ts new file mode 100644 index 00000000..b959076c --- /dev/null +++ b/hindsight-all-npm/src/command.ts @@ -0,0 +1,25 @@ +/** + * Resolve the command that invokes the `hindsight-embed` Python CLI. + * + * - If `embedPackagePath` is set, runs the package from a local checkout via + * `uv run --directory hindsight-embed`. Used for in-repo development. + * - Otherwise runs it via `uvx hindsight-embed@` so no global install + * is required. + * + * Returns the argv as `[command, ...baseArgs]` suitable for `spawn()` / + * `execFile()` (never shell-interpolated). + */ +export interface EmbedCommandOptions { + /** Version spec passed to uvx (e.g. "latest", "0.5.0"). Default: "latest". */ + embedVersion?: string; + /** Local checkout path. When set, overrides `embedVersion` and uses `uv run`. */ + embedPackagePath?: string; +} + +export function getEmbedCommand(opts: EmbedCommandOptions = {}): string[] { + if (opts.embedPackagePath) { + return ['uv', 'run', '--directory', opts.embedPackagePath, 'hindsight-embed']; + } + const version = opts.embedVersion && opts.embedVersion.length > 0 ? opts.embedVersion : 'latest'; + return ['uvx', `hindsight-embed@${version}`]; +} diff --git a/hindsight-all-npm/src/index.ts b/hindsight-all-npm/src/index.ts new file mode 100644 index 00000000..5e0ccda3 --- /dev/null +++ b/hindsight-all-npm/src/index.ts @@ -0,0 +1,7 @@ +export { HindsightServer } from './server.js'; +export { getEmbedCommand } from './command.js'; +export { silentLogger, consoleLogger } from './logger.js'; + +export type { Logger } from './logger.js'; +export type { EmbedCommandOptions } from './command.js'; +export type { HindsightServerOptions } from './types.js'; diff --git a/hindsight-all-npm/src/logger.ts b/hindsight-all-npm/src/logger.ts new file mode 100644 index 00000000..5ce92b46 --- /dev/null +++ b/hindsight-all-npm/src/logger.ts @@ -0,0 +1,29 @@ +/** + * Pluggable logger interface. + * + * This package does not own any logging infrastructure — consumers inject + * whatever they want (console, pino, openclaw's logger, a no-op). The default + * is silent so embedding this package never adds noise to an unrelated app. + */ +export interface Logger { + debug(msg: string): void; + info(msg: string): void; + warn(msg: string): void; + error(msg: string): void; +} + +/** Logger that drops every call. Used when no logger is passed. */ +export const silentLogger: Logger = { + debug: () => {}, + info: () => {}, + warn: () => {}, + error: () => {}, +}; + +/** Logger that writes to the standard console. Handy for CLIs and tests. */ +export const consoleLogger: Logger = { + debug: (msg) => console.debug(msg), + info: (msg) => console.log(msg), + warn: (msg) => console.warn(msg), + error: (msg) => console.error(msg), +}; diff --git a/hindsight-all-npm/src/server.test.ts b/hindsight-all-npm/src/server.test.ts new file mode 100644 index 00000000..cac50703 --- /dev/null +++ b/hindsight-all-npm/src/server.test.ts @@ -0,0 +1,35 @@ +import { describe, it, expect } from 'vitest'; +import { HindsightServer } from './server.js'; + +describe('HindsightServer construction', () => { + it('defaults base URL to http://127.0.0.1:8888', () => { + const server = new HindsightServer(); + expect(server.getBaseUrl()).toBe('http://127.0.0.1:8888'); + expect(server.getProfile()).toBe('default'); + }); + + it('honours custom profile, port, and host', () => { + const server = new HindsightServer({ profile: 'app', port: 9077, host: '0.0.0.0' }); + expect(server.getProfile()).toBe('app'); + expect(server.getBaseUrl()).toBe('http://0.0.0.0:9077'); + }); + + it('accepts open env pass-through without complaining about unknown keys', () => { + const server = new HindsightServer({ + env: { + HINDSIGHT_API_LLM_PROVIDER: 'openai', + HINDSIGHT_API_LLM_MODEL: 'gpt-4o-mini', + // A field that does not exist today — should still be accepted + HINDSIGHT_FUTURE_FLAG: 'enabled', + }, + }); + expect(server).toBeInstanceOf(HindsightServer); + }); + + it('exposes checkHealth that returns false when no daemon is running', async () => { + // Random high port that nothing is listening on. + const server = new HindsightServer({ port: 1, readyTimeoutMs: 100 }); + const healthy = await server.checkHealth(); + expect(healthy).toBe(false); + }); +}); diff --git a/hindsight-all-npm/src/server.ts b/hindsight-all-npm/src/server.ts new file mode 100644 index 00000000..c258ed39 --- /dev/null +++ b/hindsight-all-npm/src/server.ts @@ -0,0 +1,322 @@ +import { spawn } from 'child_process'; +import { getEmbedCommand } from './command.js'; +import { silentLogger } from './logger.js'; +import type { Logger } from './logger.js'; +import type { HindsightServerOptions } from './types.js'; + +const DEFAULT_PORT = 8888; +const DEFAULT_HOST = '127.0.0.1'; +const DEFAULT_PROFILE = 'default'; +const DEFAULT_READY_TIMEOUT_MS = 30_000; +const DEFAULT_READY_POLL_INTERVAL_MS = 1_000; + +/** + * Manages the lifecycle of a local Hindsight daemon from a Node.js process. + * + * On {@link start}, this class: + * 1. Resolves the `hindsight-embed` command (via `uvx` or a local `uv run`). + * 2. Runs `profile create --merge --port [--env K=V ...]` + * with every entry in {@link HindsightServerOptions.env} forwarded as + * an `--env` flag. + * 3. Runs `daemon --profile start` and waits for the start command + * to exit. + * 4. Polls `http://host:port/health` until it returns `200` or the + * `readyTimeoutMs` budget is exhausted. + * + * On {@link stop}, it runs `daemon --profile stop` and returns once + * the command exits (or after a short grace period). + * + * This is the Node.js equivalent of the Python `hindsight-all` package's + * `HindsightServer`: a thin programmatic lifecycle wrapper around the + * Hindsight daemon. It does NOT ship an HTTP client — once `start()` + * resolves, use `@vectorize-io/hindsight-client` against `getBaseUrl()` for + * retain / recall / reflect. + * + * The class is deliberately transparent about the daemon: new CLI flags or + * environment variables never require a code change here — callers can pass + * them via `env`, `extraProfileCreateArgs`, or `extraDaemonStartArgs`. + */ +export class HindsightServer { + private readonly profile: string; + private readonly port: number; + private readonly host: string; + private readonly baseUrl: string; + private readonly embedVersion: string | undefined; + private readonly embedPackagePath: string | undefined; + private readonly userEnv: Record; + private readonly extraProfileCreateArgs: string[]; + private readonly extraDaemonStartArgs: string[]; + private readonly platformCpuWorkaround: boolean; + private readonly readyTimeoutMs: number; + private readonly readyPollIntervalMs: number; + private readonly logger: Logger; + + constructor(opts: HindsightServerOptions = {}) { + this.profile = opts.profile ?? DEFAULT_PROFILE; + this.port = opts.port ?? DEFAULT_PORT; + this.host = opts.host ?? DEFAULT_HOST; + this.baseUrl = `http://${this.host}:${this.port}`; + this.embedVersion = opts.embedVersion; + this.embedPackagePath = opts.embedPackagePath; + this.userEnv = opts.env ?? {}; + this.extraProfileCreateArgs = opts.extraProfileCreateArgs ?? []; + this.extraDaemonStartArgs = opts.extraDaemonStartArgs ?? []; + this.platformCpuWorkaround = opts.platformCpuWorkaround ?? (process.platform === 'darwin'); + this.readyTimeoutMs = opts.readyTimeoutMs ?? DEFAULT_READY_TIMEOUT_MS; + this.readyPollIntervalMs = opts.readyPollIntervalMs ?? DEFAULT_READY_POLL_INTERVAL_MS; + this.logger = opts.logger ?? silentLogger; + } + + /** The base URL the daemon listens on (`http://host:port`). */ + getBaseUrl(): string { + return this.baseUrl; + } + + /** The profile name this server operates on. */ + getProfile(): string { + return this.profile; + } + + /** + * Ensure the daemon is configured and running. Idempotent — the underlying + * `profile create --merge` and `daemon start` commands tolerate re-runs. + */ + async start(): Promise { + this.logger.info(`[hindsight] starting daemon for profile "${this.profile}"`); + + const env = this.buildEnv(); + await this.configureProfile(env); + await this.startDaemon(env); + await this.waitForReady(); + + this.logger.info(`[hindsight] daemon ready at ${this.baseUrl}`); + } + + /** Stop the daemon. Never throws — logs and resolves even on failure. */ + async stop(): Promise { + this.logger.info(`[hindsight] stopping daemon for profile "${this.profile}"`); + + const [cmd, ...baseArgs] = getEmbedCommand({ + embedVersion: this.embedVersion, + embedPackagePath: this.embedPackagePath, + }); + const args = [...baseArgs, 'daemon', '--profile', this.profile, 'stop']; + + const child = spawn(cmd, args, { stdio: 'pipe' }); + this.pipeOutput(child, 'daemon.stop'); + + await new Promise((resolve) => { + const timeout = setTimeout(() => { + this.logger.warn(`[hindsight] daemon stop timed out after 5s`); + resolve(); + }, 5_000); + child.on('exit', () => { + clearTimeout(timeout); + this.logger.info(`[hindsight] daemon stopped`); + resolve(); + }); + child.on('error', (err) => { + clearTimeout(timeout); + this.logger.warn(`[hindsight] error stopping daemon: ${err.message}`); + resolve(); + }); + }); + } + + /** Probe `/health` once with a short timeout. */ + async checkHealth(): Promise { + try { + const res = await fetch(`${this.baseUrl}/health`, { + signal: AbortSignal.timeout(2_000), + }); + return res.ok; + } catch { + return false; + } + } + + // ------------------------------------------------------------------------- + // Internal + // ------------------------------------------------------------------------- + + /** + * Merge the process env, the caller-supplied `env`, and (on macOS) the + * embeddings CPU workaround. Caller-supplied values always win over the + * workaround; undefined values are dropped. + */ + private buildEnv(): NodeJS.ProcessEnv { + const merged: NodeJS.ProcessEnv = { ...process.env }; + + if (this.platformCpuWorkaround && process.platform === 'darwin') { + merged['HINDSIGHT_API_EMBEDDINGS_LOCAL_FORCE_CPU'] = '1'; + merged['HINDSIGHT_API_RERANKER_LOCAL_FORCE_CPU'] = '1'; + } + + for (const [key, value] of Object.entries(this.userEnv)) { + if (value !== undefined) { + merged[key] = value; + } + } + + return merged; + } + + /** + * Run `profile create --merge --port [--env K=V ...]`. + * Every entry in the merged env that was passed via {@link userEnv} (or + * auto-applied by the CPU workaround) is forwarded as `--env`. + */ + private async configureProfile(env: NodeJS.ProcessEnv): Promise { + this.logger.info(`[hindsight] configuring profile "${this.profile}"`); + + const [cmd, ...baseArgs] = getEmbedCommand({ + embedVersion: this.embedVersion, + embedPackagePath: this.embedPackagePath, + }); + const createArgs = [ + ...baseArgs, + 'profile', + 'create', + this.profile, + '--merge', + '--port', + String(this.port), + ]; + + // Forward every env var that the caller intended for the daemon as --env. + // We only forward keys the caller explicitly set (userEnv) plus the CPU + // workaround values — not the entire process.env, to avoid leaking random + // host state into profile config. + const envForProfile = this.collectProfileEnv(env); + for (const [key, value] of Object.entries(envForProfile)) { + createArgs.push('--env', `${key}=${value}`); + } + + createArgs.push(...this.extraProfileCreateArgs); + + await this.runCommand(cmd, createArgs, env, 'profile.create'); + } + + /** Collect only the env vars that should be written into the profile file. */ + private collectProfileEnv(env: NodeJS.ProcessEnv): Record { + const out: Record = {}; + + // 1. User-supplied env — always forwarded. + for (const [key, value] of Object.entries(this.userEnv)) { + if (value !== undefined) { + out[key] = value; + } + } + + // 2. CPU workaround — only if auto-applied and not already overridden. + if (this.platformCpuWorkaround && process.platform === 'darwin') { + const cpuKeys = [ + 'HINDSIGHT_API_EMBEDDINGS_LOCAL_FORCE_CPU', + 'HINDSIGHT_API_RERANKER_LOCAL_FORCE_CPU', + ]; + for (const key of cpuKeys) { + if (!(key in out) && env[key] !== undefined) { + out[key] = env[key] as string; + } + } + } + + return out; + } + + private async startDaemon(env: NodeJS.ProcessEnv): Promise { + const [cmd, ...baseArgs] = getEmbedCommand({ + embedVersion: this.embedVersion, + embedPackagePath: this.embedPackagePath, + }); + const args = [ + ...baseArgs, + 'daemon', + '--profile', + this.profile, + 'start', + ...this.extraDaemonStartArgs, + ]; + + await this.runCommand(cmd, args, env, 'daemon.start'); + } + + /** + * Spawn `cmd` with `args`, pipe its output through the logger, and resolve + * once it exits with code 0. Rejects on non-zero exit or spawn error. + */ + private async runCommand( + cmd: string, + args: string[], + env: NodeJS.ProcessEnv, + label: string, + ): Promise { + const child = spawn(cmd, args, { stdio: 'pipe', env }); + let output = ''; + child.stdout?.on('data', (data: Buffer) => { + const text = data.toString(); + output += text; + for (const line of text.trimEnd().split('\n')) { + if (line) this.logger.info(`[hindsight:${label}] ${line}`); + } + }); + child.stderr?.on('data', (data: Buffer) => { + const text = data.toString(); + output += text; + for (const line of text.trimEnd().split('\n')) { + if (line) this.logger.warn(`[hindsight:${label}] ${line}`); + } + }); + + await new Promise((resolve, reject) => { + child.on('exit', (code) => { + if (code === 0) { + resolve(); + } else { + reject(new Error(`${label} failed with code ${code}: ${output.trim()}`)); + } + }); + child.on('error', (err) => { + reject(new Error(`${label} failed to spawn: ${err.message}`, { cause: err })); + }); + }); + } + + /** Stream a spawned child's stdout/stderr through the logger without blocking. */ + private pipeOutput(child: ReturnType, label: string): void { + child.stdout?.on('data', (data: Buffer) => { + for (const line of data.toString().trimEnd().split('\n')) { + if (line) this.logger.info(`[hindsight:${label}] ${line}`); + } + }); + child.stderr?.on('data', (data: Buffer) => { + for (const line of data.toString().trimEnd().split('\n')) { + if (line) this.logger.warn(`[hindsight:${label}] ${line}`); + } + }); + } + + /** Poll `/health` until it succeeds or `readyTimeoutMs` elapses. */ + private async waitForReady(): Promise { + const deadline = Date.now() + this.readyTimeoutMs; + let attempt = 0; + while (Date.now() < deadline) { + attempt++; + try { + const res = await fetch(`${this.baseUrl}/health`, { + signal: AbortSignal.timeout(this.readyPollIntervalMs), + }); + if (res.ok) { + this.logger.debug(`[hindsight] health check passed (attempt ${attempt})`); + return; + } + } catch { + // expected while the daemon is still booting + } + await new Promise((resolve) => setTimeout(resolve, this.readyPollIntervalMs)); + } + throw new Error( + `Hindsight daemon did not become ready within ${this.readyTimeoutMs}ms at ${this.baseUrl}`, + ); + } +} diff --git a/hindsight-all-npm/src/types.ts b/hindsight-all-npm/src/types.ts new file mode 100644 index 00000000..ec7f0527 --- /dev/null +++ b/hindsight-all-npm/src/types.ts @@ -0,0 +1,54 @@ +import type { Logger } from './logger.js'; + +/** + * Options for {@link HindsightServer}. + * + * The server is intentionally thin and pass-through: anything configurable + * on the daemon side (env vars or CLI flags) can be set here without needing + * a new dedicated option. Use {@link env} for `HINDSIGHT_*` / `OPENAI_API_KEY` / + * custom provider settings, and the two `extra*` arrays to append raw CLI + * args to `profile create` or `daemon start`. + * + * For talking to the daemon after `start()`, use `@vectorize-io/hindsight-client` + * against `server.getBaseUrl()`. This package does not ship its own HTTP + * client. + */ +export interface HindsightServerOptions { + /** Profile name used for `--profile ` on every sub-command. Default: `"default"`. */ + profile?: string; + /** TCP port the daemon listens on. Default: `8888`. */ + port?: number; + /** Hostname the daemon binds to (for health checks). Default: `127.0.0.1`. */ + host?: string; + /** Version of the underlying `hindsight-embed` PyPI package to run via `uvx`. Default: `"latest"`. */ + embedVersion?: string; + /** Local path to a `hindsight-embed` checkout — takes precedence over `embedVersion`. */ + embedPackagePath?: string; + /** + * Environment variables passed to the daemon process AND written into the + * profile via repeated `--env KEY=VALUE` flags. This is the preferred way + * to surface any `HINDSIGHT_API_*` / `HINDSIGHT_EMBED_*` setting — adding a + * new daemon env var never requires a wrapper update. + * + * Values of `undefined` are dropped (so you can spread conditionally). + */ + env?: Record; + /** Extra args appended verbatim to `hindsight-embed profile create --merge ...`. */ + extraProfileCreateArgs?: string[]; + /** Extra args appended verbatim to `hindsight-embed daemon --profile start ...`. */ + extraDaemonStartArgs?: string[]; + /** + * On macOS, automatically set + * `HINDSIGHT_API_EMBEDDINGS_LOCAL_FORCE_CPU=1` and + * `HINDSIGHT_API_RERANKER_LOCAL_FORCE_CPU=1` to avoid Metal/MPS crashes in + * daemon mode. Default: `true` on `darwin`, ignored elsewhere. Any value set + * explicitly in {@link env} wins over the auto-applied value. + */ + platformCpuWorkaround?: boolean; + /** Max time (ms) to wait for `/health` to return 200. Default: `30_000`. */ + readyTimeoutMs?: number; + /** Polling interval (ms) while waiting for `/health`. Default: `1_000`. */ + readyPollIntervalMs?: number; + /** Optional pluggable logger. Default: silent. */ + logger?: Logger; +} diff --git a/hindsight-all-npm/tsconfig.json b/hindsight-all-npm/tsconfig.json new file mode 100644 index 00000000..d1f4cf02 --- /dev/null +++ b/hindsight-all-npm/tsconfig.json @@ -0,0 +1,18 @@ +{ + "compilerOptions": { + "target": "ES2022", + "module": "ES2022", + "lib": ["ES2022"], + "moduleResolution": "node", + "declaration": true, + "outDir": "./dist", + "rootDir": "./src", + "strict": true, + "esModuleInterop": true, + "skipLibCheck": true, + "forceConsistentCasingInFileNames": true, + "resolveJsonModule": true + }, + "include": ["src/**/*"], + "exclude": ["node_modules", "dist", "src/**/*.test.ts"] +} diff --git a/hindsight-all-npm/tsup.config.ts b/hindsight-all-npm/tsup.config.ts new file mode 100644 index 00000000..a3cfbf07 --- /dev/null +++ b/hindsight-all-npm/tsup.config.ts @@ -0,0 +1,11 @@ +import { defineConfig } from 'tsup'; + +export default defineConfig({ + entry: ['src/index.ts'], + format: ['esm'], + dts: true, + outDir: 'dist', + clean: true, + sourcemap: true, + bundle: true, +}); diff --git a/hindsight-all-npm/vitest.config.ts b/hindsight-all-npm/vitest.config.ts new file mode 100644 index 00000000..96eb6ab4 --- /dev/null +++ b/hindsight-all-npm/vitest.config.ts @@ -0,0 +1,8 @@ +import { defineConfig } from 'vitest/config'; + +export default defineConfig({ + test: { + include: ['src/**/*.test.ts'], + environment: 'node', + }, +}); diff --git a/hindsight-docs/docs/sdks/embed.md b/hindsight-docs/docs/sdks/embed.md index 7b1ba402..53512559 100644 --- a/hindsight-docs/docs/sdks/embed.md +++ b/hindsight-docs/docs/sdks/embed.md @@ -2,7 +2,7 @@ sidebar_position: 5 --- -# Embedded SDK (hindsight-embed) +# Daemon CLI (hindsight-embed) Zero-configuration local memory system with automatic daemon management. Perfect for development, prototyping, and single-user applications. diff --git a/hindsight-docs/docs/sdks/hindsight-all-npm.md b/hindsight-docs/docs/sdks/hindsight-all-npm.md new file mode 100644 index 00000000..72e61d28 --- /dev/null +++ b/hindsight-docs/docs/sdks/hindsight-all-npm.md @@ -0,0 +1,90 @@ +--- +sidebar_position: 6 +--- + +# Programmatic API (Node.js) + +The `@vectorize-io/hindsight-all` npm package is the Node.js equivalent of the Python [`hindsight-all`](./hindsight-all.md) package. It lets your Node code spawn and supervise a local Hindsight daemon without deploying any server infrastructure — pair it with [`@vectorize-io/hindsight-client`](./nodejs.md) for memory operations. + +The daemon runs as a **separate OS process** on `127.0.0.1` (not in your Node process). Your code talks to it over HTTP via `HindsightClient`. + +This package **does not ship an HTTP client** — it only owns the server process. Once the daemon is running, talk to it with [`@vectorize-io/hindsight-client`](./nodejs.md) against `server.getBaseUrl()`. The two packages compose: one owns the process, the other owns the API surface. + +## How it works + +1. `server.start()` resolves the underlying `hindsight-embed` command (via `uvx` from PyPI, or `uv run --directory ` for a local checkout). +2. Runs `profile create --merge --port [--env KEY=VALUE ...]` with every entry from `options.env` forwarded as `--env`. +3. Runs `daemon --profile start`. +4. Polls `http://host:port/health` until it returns `200` or the `readyTimeoutMs` budget is exhausted. +5. `server.stop()` runs `daemon --profile stop`. + +The server is intentionally transparent: new daemon env vars or CLI flags never require a wrapper release — pass them through `env`, `extraProfileCreateArgs`, or `extraDaemonStartArgs`. + +## Requirements + +- **Node.js ≥ 22** — uses global `fetch` and `AbortSignal.timeout`. +- **`uv` / `uvx`** on `PATH` — used to download and run the Hindsight daemon. Install via [docs.astral.sh/uv](https://docs.astral.sh/uv/). + +## Install + +```bash +npm install @vectorize-io/hindsight-all @vectorize-io/hindsight-client +``` + +## Example + +```ts +import { HindsightServer, consoleLogger } from '@vectorize-io/hindsight-all'; +import { HindsightClient } from '@vectorize-io/hindsight-client'; + +const server = new HindsightServer({ + profile: 'my-app', + port: 9077, + env: { + HINDSIGHT_API_LLM_PROVIDER: 'anthropic', + HINDSIGHT_API_LLM_API_KEY: process.env.ANTHROPIC_API_KEY, + HINDSIGHT_API_LLM_MODEL: 'claude-sonnet-4-20250514', + HINDSIGHT_EMBED_DAEMON_IDLE_TIMEOUT: '0', + }, + logger: consoleLogger, +}); + +await server.start(); + +const client = new HindsightClient({ baseUrl: server.getBaseUrl() }); +await client.retain('user-123', 'User prefers dark mode.'); +const recall = await client.recall('user-123', 'what are the user preferences?'); + +await server.stop(); +``` + +For a remote Hindsight API, skip the server entirely and point `HindsightClient` directly at the remote URL. + +## `HindsightServerOptions` + +| Option | Type | Default | Description | +|---|---|---|---| +| `profile` | `string` | `"default"` | Profile name passed to `--profile` on every sub-command. | +| `port` | `number` | `8888` | TCP port the daemon listens on. | +| `host` | `string` | `"127.0.0.1"` | Hostname the daemon binds to (used for health checks). | +| `embedVersion` | `string` | `"latest"` | Version of the underlying `hindsight-embed` package to run via `uvx`. | +| `embedPackagePath` | `string` | — | Local checkout path — takes precedence over `embedVersion`. Uses `uv run --directory` instead of `uvx`. | +| `env` | `Record` | `{}` | Environment variables passed to the daemon process **and** written into the profile config via `--env KEY=VALUE`. The preferred way to surface any `HINDSIGHT_API_*` / `HINDSIGHT_EMBED_*` setting. | +| `extraProfileCreateArgs` | `string[]` | `[]` | Extra args appended verbatim to `profile create`. | +| `extraDaemonStartArgs` | `string[]` | `[]` | Extra args appended verbatim to `daemon start`. | +| `platformCpuWorkaround` | `boolean` | `true` on macOS | Auto-set `HINDSIGHT_API_EMBEDDINGS_LOCAL_FORCE_CPU=1` and `HINDSIGHT_API_RERANKER_LOCAL_FORCE_CPU=1` to avoid Metal/MPS crashes. Caller-supplied `env` values win over the auto-applied ones. | +| `readyTimeoutMs` | `number` | `30000` | Max time to wait for `/health` to return 200. | +| `readyPollIntervalMs` | `number` | `1000` | Polling interval while waiting for `/health`. | +| `logger` | `Logger` | silent | Pluggable logger (`debug`/`info`/`warn`/`error`). `consoleLogger` and `silentLogger` helpers are exported. | + +## Server methods + +| Method | Returns | Description | +|---|---|---| +| `start()` | `Promise` | Configure profile, spawn the daemon, wait for `/health`. Idempotent — safe to re-run. | +| `stop()` | `Promise` | Stop the daemon. Never throws; logs and resolves even on failure. | +| `checkHealth()` | `Promise` | One-shot `/health` probe with a 2 s timeout. | +| `getBaseUrl()` | `string` | `http://host:port` — pass this straight to `HindsightClient`. | +| `getProfile()` | `string` | The profile name this server operates on. | + +For memory operations (retain, recall, reflect, bank management) use [`@vectorize-io/hindsight-client`](./nodejs.md). diff --git a/hindsight-docs/docs/sdks/hindsight-all.md b/hindsight-docs/docs/sdks/hindsight-all.md new file mode 100644 index 00000000..ee9dde50 --- /dev/null +++ b/hindsight-docs/docs/sdks/hindsight-all.md @@ -0,0 +1,146 @@ +--- +sidebar_position: 2 +--- + +# Programmatic API (Python) + +The `hindsight-all` Python package lets your code spawn and manage a local Hindsight daemon without deploying any server infrastructure. It bundles the Hindsight API server, embedded PostgreSQL, and the Python client into one install — `pip install hindsight-all` and you can start a fully-functional Hindsight instance from a few lines of Python. + +The daemon runs as a **separate OS process** on `127.0.0.1` (not in your Python process memory). Your code talks to it over HTTP via the bundled `HindsightClient`. + +If you already have a Hindsight server running and just need a client, use [Python Client (hindsight-client)](./python.md) instead. + +## How it works + +`hindsight-all` exposes two primary APIs: + +- **`HindsightServer`** — explicit lifecycle. Use it as a context manager when you want deterministic startup/shutdown (e.g. in tests). +- **`HindsightEmbedded`** — auto-managed. Starts a daemon on first use, reuses it across calls, shuts it down after an idle timeout. Easiest for application code that doesn't want to think about lifecycle. + +Both end up talking to the same underlying daemon via the same `HindsightClient` HTTP interface — the difference is only how the server process is managed. + +## Installation + +```bash +pip install hindsight-all +``` + +The `hindsight-all` wheel bundles `hindsight-api-slim`, `hindsight-client`, and `hindsight-embed` as dependencies, so one `pip install` gets you everything. + +## `HindsightServer` — explicit lifecycle + +Use `HindsightServer` as a context manager when you want the server to start immediately, run for the duration of a block, and shut down cleanly afterwards. Ideal for tests and short-lived scripts. + +```python +import os +from hindsight import HindsightServer, HindsightClient + +with HindsightServer( + llm_provider="openai", + llm_model="gpt-4o-mini", + llm_api_key=os.environ["OPENAI_API_KEY"], +) as server: + client = HindsightClient(base_url=server.url) + + client.retain(bank_id="my-bank", content="Alice works at Google") + results = client.recall(bank_id="my-bank", query="What does Alice do?") + for r in results: + print(r.text) + + answer = client.reflect(bank_id="my-bank", query="Tell me about Alice") + print(answer.text) +# Server is stopped here +``` + +## `HindsightEmbedded` — auto-managed + +`HindsightEmbedded` is the simplest way to use Hindsight in Python. It automatically manages a background daemon for you — starts on first use, stays alive across calls, shuts down after an idle timeout. + +```python +from hindsight import HindsightEmbedded +import os + +# Server starts automatically on first call +client = HindsightEmbedded( + profile="myapp", # Profile for data isolation + llm_provider="openai", + llm_model="gpt-4o-mini", + llm_api_key=os.environ["OPENAI_API_KEY"], +) + +# Use immediately - no manual server management needed +client.retain(bank_id="my-bank", content="Alice works at Google") +results = client.recall(bank_id="my-bank", query="What does Alice do?") + +# Server continues running (auto-stops after idle timeout) +# Or explicitly stop it: +client.close(stop_daemon=True) +``` + +### What's a Profile? + +A profile is an isolated Hindsight environment. Each profile gets its own embedded PostgreSQL database (stored in `~/.pg0/instances/hindsight-embed-{profile}/`) and its own API server. Use different profiles to separate environments (dev/prod), applications, or users. + +### When to use which + +| Use case | Pick | +|---|---| +| Tests, short-lived scripts, deterministic startup/shutdown | `HindsightServer` (context manager) | +| Long-running application, auto-start on first use, don't want to manage lifecycle | `HindsightEmbedded` | +| Existing Hindsight server running elsewhere | [`hindsight-client`](./python.md) directly | + +## API namespaces + +Both `HindsightEmbedded` and `HindsightClient` expose organized API namespaces for bank management, mental models, directives, and memories: + +```python +from hindsight import HindsightEmbedded +import os + +embedded = HindsightEmbedded( + profile="myapp", + llm_provider="openai", + llm_api_key=os.environ["OPENAI_API_KEY"], +) + +# Core operations +embedded.retain(bank_id="test", content="Hello") +results = embedded.recall(bank_id="test", query="Hello") + +# Bank management +embedded.banks.create(bank_id="test", name="Test Bank", mission="Help users") +embedded.banks.set_mission(bank_id="test", mission="Updated mission") +embedded.banks.delete(bank_id="test") + +# Mental models +embedded.mental_models.create( + bank_id="test", + name="User Preferences", + content="User prefers dark mode" +) +models = embedded.mental_models.list(bank_id="test") + +# Directives +embedded.directives.create( + bank_id="test", + name="Response Style", + content="Be concise and friendly" +) +directives = embedded.directives.list(bank_id="test") + +# List memories +memories = embedded.memories.list(bank_id="test", type="world", limit=50) +``` + +API namespaces ensure the daemon is running before each call, so daemon crashes are handled gracefully: + +```python +# ✅ GOOD - Uses API namespace (daemon restarts handled) +embedded.banks.create(bank_id="test", name="Test") + +# ❌ BAD - Direct client access (daemon crashes NOT handled) +client = embedded.client +client.create_bank(bank_id="test", name="Test") # Fails if daemon crashed +``` + +For the full reference of retain/recall/reflect methods and their options (which work the same regardless of how you obtain the client) see the [Python Client page](./python.md). diff --git a/hindsight-docs/docs/sdks/python.md b/hindsight-docs/docs/sdks/python.md index 1ddfb774..86b4419b 100644 --- a/hindsight-docs/docs/sdks/python.md +++ b/hindsight-docs/docs/sdks/python.md @@ -4,66 +4,18 @@ sidebar_position: 1 # Python Client -Official Python client for the Hindsight API. +Official HTTP client for the Hindsight API. Use this when you have a Hindsight server already running — locally, in Docker, or as a managed service — and you want a typed Python client to talk to it. -import Tabs from '@theme/Tabs'; -import TabItem from '@theme/TabItem'; +If you want to **embed and run a Hindsight server in your Python process** (no external server required), see [Embedded Python (hindsight-all)](./hindsight-all.md) instead. ## Installation - - - -The `hindsight-all` package includes embedded PostgreSQL, HTTP API server, and client: - -```bash -pip install hindsight-all -``` - - - - -If you already have a Hindsight server running: - ```bash pip install hindsight-client ``` - - - ## Quick Start - - - -```python -import os -from hindsight import HindsightServer, HindsightClient - -with HindsightServer( - llm_provider="openai", - llm_model="gpt-4o-mini", - llm_api_key=os.environ["OPENAI_API_KEY"] -) as server: - client = HindsightClient(base_url=server.url) - - # Retain a memory - client.retain(bank_id="my-bank", content="Alice works at Google") - - # Recall memories - results = client.recall(bank_id="my-bank", query="What does Alice do?") - for r in results: - print(r.text) - - # Reflect - generate response with disposition - answer = client.reflect(bank_id="my-bank", query="Tell me about Alice") - print(answer.text) -``` - - - - ```python from hindsight_client import Hindsight @@ -74,130 +26,36 @@ client.retain(bank_id="my-bank", content="Alice works at Google") # Recall memories results = client.recall(bank_id="my-bank", query="What does Alice do?") -for r in results: +for r in results.results: print(r.text) -# Reflect - generate response with disposition +# Reflect - generate a contextual answer answer = client.reflect(bank_id="my-bank", query="Tell me about Alice") print(answer.text) ``` - - - -## Embedded Client (Easiest Option) - -`HindsightEmbedded` provides the simplest way to use Hindsight in Python. It automatically manages a background server for you - no manual setup required: - -```python -from hindsight import HindsightEmbedded -import os - -# Server starts automatically on first use -client = HindsightEmbedded( - profile="myapp", # Profile for data isolation - llm_provider="openai", - llm_model="gpt-4o-mini", - llm_api_key=os.environ["OPENAI_API_KEY"], -) - -# Use immediately - no manual server management needed -client.retain(bank_id="my-bank", content="Alice works at Google") -results = client.recall(bank_id="my-bank", query="What does Alice do?") - -# Server continues running (auto-stops after idle timeout) -# Or explicitly stop it: -client.close(stop_daemon=True) -``` - -**What's a Profile?** - -A profile is an isolated Hindsight environment. Each profile gets its own PostgreSQL database (stored in `~/.pg0/instances/hindsight-embed-{profile}/`) and its own API server. Use different profiles to separate environments (dev/prod), applications, or users. - -**When to Use HindsightEmbedded** - -Use `HindsightEmbedded` when you want the server to start automatically and manage itself. Use `HindsightServer` when you need explicit control over server lifecycle (e.g., testing where you want immediate startup/shutdown). - -**Advanced Operations** - -`HindsightEmbedded` provides organized API namespaces for advanced operations. Each method call automatically ensures the daemon is running: - -```python -from hindsight import HindsightEmbedded -import os - -embedded = HindsightEmbedded( - profile="myapp", - llm_provider="openai", - llm_api_key=os.environ["OPENAI_API_KEY"], -) - -# Core operations (automatically proxied) -embedded.retain(bank_id="test", content="Hello") -results = embedded.recall(bank_id="test", query="Hello") - -# Bank management -embedded.banks.create(bank_id="test", name="Test Bank", mission="Help users") -embedded.banks.set_mission(bank_id="test", mission="Updated mission") -embedded.banks.delete(bank_id="test") - -# Mental models -embedded.mental_models.create( - bank_id="test", - name="User Preferences", - content="User prefers dark mode" -) -models = embedded.mental_models.list(bank_id="test") -embedded.mental_models.update(bank_id="test", mental_model_id="...", content="New content") - -# Directives -embedded.directives.create( - bank_id="test", - name="Response Style", - content="Be concise and friendly" -) -directives = embedded.directives.list(bank_id="test") - -# List memories -memories = embedded.memories.list(bank_id="test", type="world", limit=50) -``` - -**Why Use API Namespaces?** - -API namespaces (`banks`, `mental_models`, `directives`, `memories`) ensure the daemon is running before each call. This handles daemon crashes gracefully: - -```python -# ✅ GOOD - Uses API namespace (daemon restarts handled) -embedded.banks.create(bank_id="test", name="Test") - -# ❌ BAD - Direct client access (daemon crashes NOT handled) -client = embedded.client -client.create_bank(bank_id="test", name="Test") # Fails if daemon crashed -``` - ## Client Initialization ```python -from hindsight import HindsightClient +from hindsight_client import Hindsight -client = HindsightClient( +client = Hindsight( base_url="http://localhost:8888", # Hindsight API URL timeout=30.0, # Request timeout in seconds + # api_key="your-api-key", # Optional bearer token ) # Core operations client.retain(bank_id="test", content="Hello world") results = client.recall(bank_id="test", query="Hello") -# Organized API access (same as HindsightEmbedded) +# Organized API namespaces client.banks.create(bank_id="test", name="Test Bank") models = client.mental_models.list(bank_id="test") directives = client.directives.list(bank_id="test") memories = client.memories.list(bank_id="test") ``` -Both `HindsightClient` and `HindsightEmbedded` provide the same organized API namespaces (`banks`, `mental_models`, `directives`, `memories`) for consistent developer experience. - ## Core Operations ### Retain (Store Memory) diff --git a/hindsight-docs/docusaurus.config.ts b/hindsight-docs/docusaurus.config.ts index 03233e95..530e384a 100644 --- a/hindsight-docs/docusaurus.config.ts +++ b/hindsight-docs/docusaurus.config.ts @@ -83,18 +83,25 @@ const config: Config = { docs: { sidebarPath: './sidebars.ts', routeBasePath: '/', - // Only show "next" version in development or when INCLUDE_CURRENT_VERSION=true - // In production, only show released versions from versions.json + // Whether to include the "current" (Next / unreleased) docs version. + // + // Controlled by the single explicit env var INCLUDE_CURRENT_VERSION. + // `start-docs.sh` sets it to "true" so local dev always sees Next; + // production builds leave it unset so only released versions ship. + // + // We deliberately do NOT sniff NODE_ENV here — it's unreliable + // across Docusaurus hot-reload paths and used to cause the Next + // version to disappear intermittently when editing files. onlyIncludeVersions: (() => { - const isDev = process.env.NODE_ENV === 'development' || process.env.INCLUDE_CURRENT_VERSION === 'true'; + const includeCurrent = process.env.INCLUDE_CURRENT_VERSION === 'true'; + let released: string[] = []; try { - const versions = require('./versions.json') as string[]; - // In dev mode, explicitly include 'current' (Next) + all released versions - // In production, only show released versions - return isDev ? ['current', ...versions] : versions; + released = require('./versions.json') as string[]; } catch { - return undefined; // No versions yet, show current + // No versions.json yet — nothing has been released. + return undefined; } + return includeCurrent ? ['current', ...released] : released; })(), // Disable version badges on all versions versions: (() => { diff --git a/hindsight-docs/sidebars.ts b/hindsight-docs/sidebars.ts index 483fa59b..5078e648 100644 --- a/hindsight-docs/sidebars.ts +++ b/hindsight-docs/sidebars.ts @@ -165,12 +165,6 @@ const sidebars: SidebarsConfig = { label: 'CLI', customProps: { icon: 'lu-terminal' }, }, - { - type: 'doc', - id: 'sdks/embed', - label: 'Embedded Python', - customProps: { icon: '/img/icons/package.svg' }, - }, ], }, { @@ -355,6 +349,49 @@ const sidebars: SidebarsConfig = { }, ], }, + { + type: 'category', + label: 'Installation', + collapsible: false, + items: [ + { + type: 'link', + href: '/developer/installation#docker', + label: 'Docker', + customProps: { icon: 'si-docker', iconAfter: 'lu-arrow-up-right' }, + }, + { + type: 'link', + href: '/developer/installation#helm--kubernetes', + label: 'Kubernetes', + customProps: { icon: 'si-kubernetes', iconAfter: 'lu-arrow-up-right' }, + }, + { + type: 'link', + href: '/developer/installation#bare-metal-pip', + label: 'Bare Metal', + customProps: { icon: 'lu-hard-drive', iconAfter: 'lu-arrow-up-right' }, + }, + { + type: 'doc', + id: 'sdks/hindsight-all', + label: 'Programmatic API (Python)', + customProps: { icon: 'si-python' }, + }, + { + type: 'doc', + id: 'sdks/hindsight-all-npm', + label: 'Programmatic API (Node.js)', + customProps: { icon: 'si-nodedotjs' }, + }, + { + type: 'doc', + id: 'sdks/embed', + label: 'Daemon CLI', + customProps: { icon: 'lu-terminal' }, + }, + ], + }, { type: 'category', label: 'Resources', diff --git a/hindsight-docs/src/theme/DocSidebarItem/Link/index.tsx b/hindsight-docs/src/theme/DocSidebarItem/Link/index.tsx index 10a537e6..d7080428 100644 --- a/hindsight-docs/src/theme/DocSidebarItem/Link/index.tsx +++ b/hindsight-docs/src/theme/DocSidebarItem/Link/index.tsx @@ -9,12 +9,12 @@ import { LuZap, LuDatabase, LuGitCompare, LuRocket, LuMemoryStick, LuWebhook, LuFileText, LuServer, LuSettings, LuTerminal, LuActivity, LuPlug, LuShield, LuPackage, LuBook, - LuNetwork, LuCode, LuLayers, LuCpu, + LuNetwork, LuCode, LuLayers, LuCpu, LuHardDrive, LuArrowUpRight, LuBookOpen, LuRss, LuCloud, LuMessageCircle, LuChartBar, LuChartColumn, LuStar, LuCircleHelp, LuLayoutTemplate, LuFileJson, } from 'react-icons/lu'; -import {SiGo, SiPython, SiGithub, SiSlack} from 'react-icons/si'; +import {SiGo, SiPython, SiGithub, SiSlack, SiDocker, SiKubernetes, SiNodedotjs} from 'react-icons/si'; const ICON_MAP: Record = { 'lu-brain': LuBrain, @@ -41,10 +41,14 @@ const ICON_MAP: Record = { 'lu-code': LuCode, 'lu-layers': LuLayers, 'lu-cpu': LuCpu, + 'lu-hard-drive': LuHardDrive, 'si-go': SiGo, 'si-python': SiPython, 'si-github': SiGithub, 'si-slack': SiSlack, + 'si-docker': SiDocker, + 'si-kubernetes': SiKubernetes, + 'si-nodedotjs': SiNodedotjs, 'lu-chart-bar': LuChartBar, 'lu-chart-column': LuChartColumn, 'lu-arrow-up-right': LuArrowUpRight, diff --git a/hindsight-integrations/claude-code/scripts/lib/daemon.py b/hindsight-integrations/claude-code/scripts/lib/daemon.py index 9d8a2585..7b8e4c16 100644 --- a/hindsight-integrations/claude-code/scripts/lib/daemon.py +++ b/hindsight-integrations/claude-code/scripts/lib/daemon.py @@ -1,6 +1,6 @@ """Hindsight-embed daemon lifecycle management. -Port of: HindsightEmbedManager in embed-manager.js, adapted for Python +Port of: HindsightServer in @vectorize-io/hindsight-all, adapted for Python subprocess calls from ephemeral hook processes. Manages three connection modes (same as Openclaw): @@ -30,7 +30,7 @@ PROFILE_NAME = "claude-code" def _get_embed_command(config: dict) -> list: """Get the command to run hindsight-embed. - Port of: getEmbedCommand() in embed-manager.js + Port of: getEmbedCommand() in @vectorize-io/hindsight-all """ embed_path = config.get("embedPackagePath") if embed_path: @@ -139,7 +139,7 @@ def get_api_url(config: dict, debug_fn=None, allow_daemon_start: bool = False) - def _ensure_daemon_running(config: dict, port: int, debug_fn=None): """Start the hindsight-embed daemon if not already running. - Port of: HindsightEmbedManager.start() in embed-manager.js + Port of: HindsightServer.start() in @vectorize-io/hindsight-all """ # Fast-fail if hindsight-embed toolchain is not available if not _is_embed_available(config): diff --git a/hindsight-integrations/openclaw/package-lock.json b/hindsight-integrations/openclaw/package-lock.json index 6e2cdc29..d7eed59f 100644 --- a/hindsight-integrations/openclaw/package-lock.json +++ b/hindsight-integrations/openclaw/package-lock.json @@ -9,7 +9,11 @@ "version": "0.5.1", "license": "MIT", "dependencies": { - "node-fetch": "^3.3.2" + "@vectorize-io/hindsight-all": "file:../../hindsight-all-npm", + "@vectorize-io/hindsight-client": "file:../../hindsight-clients/typescript" + }, + "bin": { + "hindsight-openclaw-backfill": "dist/backfill.js" }, "devDependencies": { "@types/node": "^20.0.0", @@ -21,6 +25,34 @@ "node": ">=22" } }, + "../../hindsight-all-npm": { + "name": "@vectorize-io/hindsight-all", + "version": "0.5.0", + "license": "MIT", + "devDependencies": { + "@types/node": "^22.0.0", + "tsup": "^8.5.1", + "typescript": "^5.7.0", + "vitest": "^4.1.2" + }, + "engines": { + "node": ">=22" + } + }, + "../../hindsight-clients/typescript": { + "name": "@vectorize-io/hindsight-client", + "version": "0.5.0", + "license": "MIT", + "devDependencies": { + "@hey-api/openapi-ts": "0.88.0", + "@types/jest": "^29.0.0", + "@types/node": "^20.0.0", + "jest": "^29.0.0", + "ts-jest": "^29.0.0", + "tsup": "^8.5.1", + "typescript": "^5.0.0" + } + }, "node_modules/@emnapi/core": { "version": "1.9.1", "resolved": "https://registry.npmjs.org/@emnapi/core/-/core-1.9.1.tgz", @@ -883,6 +915,14 @@ "undici-types": "~6.21.0" } }, + "node_modules/@vectorize-io/hindsight-all": { + "resolved": "../../hindsight-all-npm", + "link": true + }, + "node_modules/@vectorize-io/hindsight-client": { + "resolved": "../../hindsight-clients/typescript", + "link": true + }, "node_modules/@vitest/expect": { "version": "4.1.3", "resolved": "https://registry.npmjs.org/@vitest/expect/-/expect-4.1.3.tgz", @@ -1045,15 +1085,6 @@ "dev": true, "license": "MIT" }, - "node_modules/data-uri-to-buffer": { - "version": "4.0.1", - "resolved": "https://registry.npmjs.org/data-uri-to-buffer/-/data-uri-to-buffer-4.0.1.tgz", - "integrity": "sha512-0R9ikRb668HB7QDxT1vkpuUBtqc53YyAwMwGeUFKRojY/NWKvdZ+9UYtRfGmhqNbRkTSVpMbmyhXipFFv2cb/A==", - "license": "MIT", - "engines": { - "node": ">= 12" - } - }, "node_modules/detect-libc": { "version": "2.1.2", "resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-2.1.2.tgz", @@ -1153,29 +1184,6 @@ } } }, - "node_modules/fetch-blob": { - "version": "3.2.0", - "resolved": "https://registry.npmjs.org/fetch-blob/-/fetch-blob-3.2.0.tgz", - "integrity": "sha512-7yAQpD2UMJzLi1Dqv7qFYnPbaPx7ZfFK6PiIxQ4PfkGPyNyl2Ugx+a/umUonmKqjhM4DnfbMvdX6otXq83soQQ==", - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/jimmywarting" - }, - { - "type": "paypal", - "url": "https://paypal.me/jimmywarting" - } - ], - "license": "MIT", - "dependencies": { - "node-domexception": "^1.0.0", - "web-streams-polyfill": "^3.0.3" - }, - "engines": { - "node": "^12.20 || >= 14.13" - } - }, "node_modules/fflate": { "version": "0.8.2", "resolved": "https://registry.npmjs.org/fflate/-/fflate-0.8.2.tgz", @@ -1190,18 +1198,6 @@ "dev": true, "license": "ISC" }, - "node_modules/formdata-polyfill": { - "version": "4.0.10", - "resolved": "https://registry.npmjs.org/formdata-polyfill/-/formdata-polyfill-4.0.10.tgz", - "integrity": "sha512-buewHzMvYL29jdeQTVILecSaZKnt/RJWjoZCF5OW60Z67/GmSLBkOFM7qh1PI3zFNtJbaZL5eQu1vLfazOwj4g==", - "license": "MIT", - "dependencies": { - "fetch-blob": "^3.1.2" - }, - "engines": { - "node": ">=12.20.0" - } - }, "node_modules/fsevents": { "version": "2.3.3", "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.3.tgz", @@ -1517,44 +1513,6 @@ "node": "^10 || ^12 || ^13.7 || ^14 || >=15.0.1" } }, - "node_modules/node-domexception": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/node-domexception/-/node-domexception-1.0.0.tgz", - "integrity": "sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ==", - "deprecated": "Use your platform's native DOMException instead", - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/jimmywarting" - }, - { - "type": "github", - "url": "https://paypal.me/jimmywarting" - } - ], - "license": "MIT", - "engines": { - "node": ">=10.5.0" - } - }, - "node_modules/node-fetch": { - "version": "3.3.2", - "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-3.3.2.tgz", - "integrity": "sha512-dRB78srN/l6gqWulah9SrxeYnxeddIG30+GOqK/9OlLVyLg3HPnr6SqOWTWOXKRwC2eGYCkZ59NNuSgvSrpgOA==", - "license": "MIT", - "dependencies": { - "data-uri-to-buffer": "^4.0.0", - "fetch-blob": "^3.1.4", - "formdata-polyfill": "^4.0.10" - }, - "engines": { - "node": "^12.20.0 || ^14.13.1 || >=16.0.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/node-fetch" - } - }, "node_modules/obug": { "version": "2.1.1", "resolved": "https://registry.npmjs.org/obug/-/obug-2.1.1.tgz", @@ -1953,15 +1911,6 @@ } } }, - "node_modules/web-streams-polyfill": { - "version": "3.3.3", - "resolved": "https://registry.npmjs.org/web-streams-polyfill/-/web-streams-polyfill-3.3.3.tgz", - "integrity": "sha512-d2JWLCivmZYTSIoge9MsgFCZrt571BikcWGYkjC1khllbTeDlGqZ2D8vD8E/lJa8WGWbb7Plm8/XJYV7IJHZZw==", - "license": "MIT", - "engines": { - "node": ">= 8" - } - }, "node_modules/why-is-node-running": { "version": "2.3.0", "resolved": "https://registry.npmjs.org/why-is-node-running/-/why-is-node-running-2.3.0.tgz", diff --git a/hindsight-integrations/openclaw/package.json b/hindsight-integrations/openclaw/package.json index b89f17cd..6aa2c241 100644 --- a/hindsight-integrations/openclaw/package.json +++ b/hindsight-integrations/openclaw/package.json @@ -44,7 +44,8 @@ "prepublishOnly": "npm run clean && npm run build" }, "dependencies": { - "node-fetch": "^3.3.2" + "@vectorize-io/hindsight-client": "file:../../hindsight-clients/typescript", + "@vectorize-io/hindsight-all": "file:../../hindsight-all-npm" }, "devDependencies": { "@types/node": "^20.0.0", diff --git a/hindsight-integrations/openclaw/src/backfill.test.ts b/hindsight-integrations/openclaw/src/backfill.test.ts index 841a611b..5049ba66 100644 --- a/hindsight-integrations/openclaw/src/backfill.test.ts +++ b/hindsight-integrations/openclaw/src/backfill.test.ts @@ -10,13 +10,17 @@ const managerStart = vi.fn(); const managerStop = vi.fn(); const managerGetBaseUrl = vi.fn(() => 'http://127.0.0.1:9077'); -vi.mock('./embed-manager.js', () => ({ - HindsightEmbedManager: vi.fn(class { - start = managerStart; - stop = managerStop; - getBaseUrl = managerGetBaseUrl; - }), -})); +vi.mock('@vectorize-io/hindsight-all', async () => { + const actual = await vi.importActual('@vectorize-io/hindsight-all'); + return { + ...actual, + HindsightServer: vi.fn(class { + start = managerStart; + stop = managerStop; + getBaseUrl = managerGetBaseUrl; + }), + }; +}); afterEach(() => { vi.restoreAllMocks(); diff --git a/hindsight-integrations/openclaw/src/backfill.ts b/hindsight-integrations/openclaw/src/backfill.ts index 02c87650..6bdf2cce 100644 --- a/hindsight-integrations/openclaw/src/backfill.ts +++ b/hindsight-integrations/openclaw/src/backfill.ts @@ -2,9 +2,9 @@ import { existsSync, realpathSync } from 'fs'; import { join, resolve } from 'path'; import { fileURLToPath, pathToFileURL } from 'url'; -import { HindsightEmbedManager } from './embed-manager.js'; -import { HindsightClient } from './client.js'; -import { buildClientOptions, detectExternalApi, detectLLMConfig } from './index.js'; +import { HindsightServer } from '@vectorize-io/hindsight-all'; +import { HindsightClient } from '@vectorize-io/hindsight-client'; +import { detectExternalApi, detectLLMConfig } from './index.js'; import type { BankStats, PluginConfig } from './types.js'; import { buildBackfillPlan, @@ -44,7 +44,7 @@ interface BackfillRuntime { } interface BankRuntime { - client: HindsightClient; + bankId: string; touchedEntryKeys: string[]; initialFailedOperations: number; missionApplied: boolean; @@ -297,16 +297,19 @@ export async function createBackfillRuntime( } const llmConfig = detectLLMConfig(pluginConfig); - const manager = new HindsightEmbedManager( - pluginConfig.apiPort || 9077, - llmConfig.provider || '', - llmConfig.apiKey || '', - llmConfig.model, - llmConfig.baseUrl, - pluginConfig.daemonIdleTimeout ?? 0, - pluginConfig.embedVersion, - pluginConfig.embedPackagePath, - ); + const manager = new HindsightServer({ + profile: 'openclaw', + port: pluginConfig.apiPort || 9077, + embedVersion: pluginConfig.embedVersion, + embedPackagePath: pluginConfig.embedPackagePath, + env: { + HINDSIGHT_API_LLM_PROVIDER: llmConfig.provider || '', + HINDSIGHT_API_LLM_API_KEY: llmConfig.apiKey || '', + HINDSIGHT_API_LLM_MODEL: llmConfig.model, + HINDSIGHT_API_LLM_BASE_URL: llmConfig.baseUrl, + HINDSIGHT_EMBED_DAEMON_IDLE_TIMEOUT: String(pluginConfig.daemonIdleTimeout ?? 0), + }, + }); await manager.start(); return { @@ -318,10 +321,29 @@ export async function createBackfillRuntime( }; } -async function waitForBankQueue(client: HindsightClient, maxPendingOperations: number): Promise { +/** + * Fetch stats for a single bank over HTTP. The high-level `HindsightClient` + * doesn't yet wrap this endpoint, so we go direct — it's one call. + */ +async function fetchBankStats(baseUrl: string, apiToken: string | undefined, bankId: string): Promise { + const headers: Record = {}; + if (apiToken) headers.Authorization = `Bearer ${apiToken}`; + const res = await fetch(`${baseUrl}/v1/default/banks/${encodeURIComponent(bankId)}/stats`, { headers }); + if (!res.ok) { + throw new Error(`HTTP ${res.status}: ${await res.text().catch(() => '')}`); + } + return res.json() as Promise; +} + +async function waitForBankQueue( + apiUrl: string, + apiToken: string | undefined, + bankId: string, + maxPendingOperations: number, +): Promise { for (;;) { try { - const stats = await client.getBankStats(); + const stats = await fetchBankStats(apiUrl, apiToken, bankId); if (stats.pending_operations <= maxPendingOperations) { return; } @@ -335,9 +357,13 @@ async function waitForBankQueue(client: HindsightClient, maxPendingOperations: n } } -async function getInitialBankStats(client: HindsightClient): Promise { +async function getInitialBankStats( + apiUrl: string, + apiToken: string | undefined, + bankId: string, +): Promise { try { - return await client.getBankStats(); + return await fetchBankStats(apiUrl, apiToken, bankId); } catch (error) { if (error instanceof Error && error.message.includes('HTTP 404')) { return null; @@ -346,10 +372,15 @@ async function getInitialBankStats(client: HindsightClient): Promise): Promise> { +async function waitForBanksToDrain( + apiUrl: string, + apiToken: string | undefined, + bankIds: Iterable, +): Promise> { + const ids = Array.from(bankIds); for (;;) { const stats = await Promise.all( - Array.from(clientsByBankId.entries()).map(async ([bankId, client]) => ({ bankId, stats: await client.getBankStats() })), + ids.map(async (bankId) => ({ bankId, stats: await fetchBankStats(apiUrl, apiToken, bankId) })), ); const statsByBank = new Map(stats.map(({ bankId, stats: bankStats }) => [bankId, bankStats])); const pending = stats.filter(({ stats: bankStats }) => bankStats.pending_operations > 0); @@ -402,9 +433,11 @@ export async function runCli(argv: string[] = process.argv.slice(2)): Promise(); + // Single shared client — hindsight-client takes bankId as a parameter on + // every call, so there's no reason to cache per-bank clients anymore. + const client = new HindsightClient({ baseUrl: runtime.apiUrl, apiKey: runtime.apiToken }); + const bankRuntimes = new Map(); let imported = 0; let failed = 0; let finalized = 0; @@ -413,50 +446,39 @@ export async function runCli(argv: string[] = process.argv.slice(2)): Promise= 0) { - await waitForBankQueue(client, args.maxPendingOperations); + await waitForBankQueue(runtime.apiUrl, runtime.apiToken, entry.bankId, args.maxPendingOperations); } try { @@ -470,10 +492,10 @@ export async function runCli(argv: string[] = process.argv.slice(2)): Promise 0) { - const finalStatsByBank = await waitForBanksToDrain( - new Map(Array.from(clientsByBankId.entries()).map(([bankId, value]) => [bankId, value.client])), - ); - const touchedEntriesByBank = new Map(Array.from(clientsByBankId.entries()).map(([bankId, value]) => [bankId, value.touchedEntryKeys])); - const initialFailedByBank = new Map(Array.from(clientsByBankId.entries()).map(([bankId, value]) => [bankId, value.initialFailedOperations])); + if (args.waitUntilDrained && bankRuntimes.size > 0) { + const finalStatsByBank = await waitForBanksToDrain(runtime.apiUrl, runtime.apiToken, bankRuntimes.keys()); + const touchedEntriesByBank = new Map(Array.from(bankRuntimes.entries()).map(([bankId, value]) => [bankId, value.touchedEntryKeys])); + const initialFailedByBank = new Map(Array.from(bankRuntimes.entries()).map(([bankId, value]) => [bankId, value.initialFailedOperations])); const finalization = applyDrainResults(checkpoint, touchedEntriesByBank, finalStatsByBank, initialFailedByBank); finalized = finalization.completed; for (const warning of finalization.warnings) { diff --git a/hindsight-integrations/openclaw/src/client.test.ts b/hindsight-integrations/openclaw/src/client.test.ts deleted file mode 100644 index ce8a5887..00000000 --- a/hindsight-integrations/openclaw/src/client.test.ts +++ /dev/null @@ -1,63 +0,0 @@ -import { afterEach, describe, it, expect, vi } from 'vitest'; -import { HindsightClient } from './client.js'; - -afterEach(() => { - vi.restoreAllMocks(); -}); - -describe('HindsightClient', () => { - it('should create instance with model', () => { - const client = new HindsightClient({ llmModel: 'gpt-4' }); - expect(client).toBeInstanceOf(HindsightClient); - }); - - it('should set bank ID', () => { - const client = new HindsightClient({}); - expect(() => client.setBankId('test-bank')).not.toThrow(); - }); - - it('should create instance with embed package path', () => { - const client = new HindsightClient({ llmModel: 'gpt-4', embedPackagePath: '/path/to/hindsight' }); - expect(client).toBeInstanceOf(HindsightClient); - }); - - it('should create instance in HTTP mode', () => { - const client = new HindsightClient({ - apiUrl: 'https://api.example.com/', - apiToken: 'bearer-token', - }); - expect(client).toBeInstanceOf(HindsightClient); - }); - - it('should ensure bank mission in HTTP mode', async () => { - const fetchMock = vi.fn().mockResolvedValue({ - ok: true, - text: async () => '', - }); - vi.stubGlobal('fetch', fetchMock); - const client = new HindsightClient({ - apiUrl: 'https://api.example.com/', - apiToken: 'bearer-token', - }); - client.setBankId('demo'); - await expect(client.ensureBankMission('mission')).resolves.toBeUndefined(); - expect(fetchMock).toHaveBeenCalledWith( - 'https://api.example.com/v1/default/banks/demo', - expect.objectContaining({ method: 'PUT' }), - ); - }); - - it('should throw when strict bank mission setup fails in HTTP mode', async () => { - const fetchMock = vi.fn().mockResolvedValue({ - ok: false, - status: 500, - text: async () => 'boom', - }); - vi.stubGlobal('fetch', fetchMock); - const client = new HindsightClient({ - apiUrl: 'https://api.example.com/', - }); - client.setBankId('demo'); - await expect(client.ensureBankMission('mission')).rejects.toThrow('Failed to set bank mission'); - }); -}); diff --git a/hindsight-integrations/openclaw/src/client.ts b/hindsight-integrations/openclaw/src/client.ts deleted file mode 100644 index e3675a58..00000000 --- a/hindsight-integrations/openclaw/src/client.ts +++ /dev/null @@ -1,312 +0,0 @@ -import { execFile } from 'child_process'; -import { promisify } from 'util'; -import { writeFile, mkdir, rm } from 'fs/promises'; -import { tmpdir } from 'os'; -import { join } from 'path'; -import { randomBytes } from 'crypto'; -import type { - RetainRequest, - RetainResponse, - RecallRequest, - RecallResponse, - BankStats, -} from './types.js'; -import * as log from './logger.js'; - -const execFileAsync = promisify(execFile); - -const MAX_BUFFER = 5 * 1024 * 1024; // 5 MB — large transcripts can exceed default 1 MB -const DEFAULT_TIMEOUT_MS = 15_000; - -/** Strip null bytes from strings — Node 22 rejects them in execFile() args */ -const sanitize = (s: string) => s.replace(/\0/g, ''); - -/** - * Sanitize a string for use as a cross-platform filename. - * Replaces characters illegal on Windows or Unix with underscores. - */ -function sanitizeFilename(name: string): string { - // Replace characters illegal on Windows (\/:*?"<>|) and control chars - return name.replace(/[\\/:*?"<>|\x00-\x1f]/g, '_').slice(0, 200) || 'content'; -} - -export interface HindsightClientOptions { - llmModel?: string; - embedVersion?: string; - embedPackagePath?: string; - apiUrl?: string; // Direct HTTP mode — bypass subprocess - apiToken?: string; // Auth header for HTTP mode -} - -export class HindsightClient { - private bankId: string = 'default'; - private llmModel?: string; - private embedVersion: string; - private embedPackagePath?: string; - private apiUrl?: string; - private apiToken?: string; - - constructor(opts: HindsightClientOptions) { - this.llmModel = opts.llmModel; - this.embedVersion = opts.embedVersion || 'latest'; - this.embedPackagePath = opts.embedPackagePath; - this.apiUrl = opts.apiUrl?.replace(/\/$/, ''); // strip trailing slash - this.apiToken = opts.apiToken; - } - - private get httpMode(): boolean { - return !!this.apiUrl; - } - - /** - * Get the command and base args to run hindsight-embed. - * Returns [command, ...baseArgs] for use with execFile/spawn (no shell). - */ - private getEmbedCommand(): string[] { - if (this.embedPackagePath) { - return ['uv', 'run', '--directory', this.embedPackagePath, 'hindsight-embed']; - } - const embedPackage = this.embedVersion ? `hindsight-embed@${this.embedVersion}` : 'hindsight-embed@latest'; - return ['uvx', embedPackage]; - } - - private httpHeaders(): Record { - const headers: Record = { 'Content-Type': 'application/json' }; - if (this.apiToken) { - headers['Authorization'] = `Bearer ${this.apiToken}`; - } - return headers; - } - - setBankId(bankId: string): void { - this.bankId = bankId; - } - - // --- setBankMission --- - - async setBankMission(mission: string): Promise { - if (!mission || mission.trim().length === 0) { - return; - } - - if (this.httpMode) { - return this.setBankMissionHttp(mission); - } - return this.setBankMissionSubprocess(mission); - } - - async ensureBankMission(mission: string): Promise { - if (!mission || mission.trim().length === 0) { - return; - } - if (this.httpMode) { - return this.ensureBankMissionHttp(mission); - } - return this.ensureBankMissionSubprocess(mission); - } - - private async setBankMissionHttp(mission: string): Promise { - try { - const url = `${this.apiUrl}/v1/default/banks/${encodeURIComponent(this.bankId)}`; - const res = await fetch(url, { - method: 'PUT', - headers: this.httpHeaders(), - body: JSON.stringify({ mission }), - signal: AbortSignal.timeout(DEFAULT_TIMEOUT_MS), - }); - if (!res.ok) { - const body = await res.text().catch(() => ''); - throw new Error(`HTTP ${res.status}: ${body}`); - } - log.verbose('bank mission set via HTTP'); - } catch (error) { - log.warn(`could not set bank mission (bank may not exist yet): ${error}`); - } - } - - private async setBankMissionSubprocess(mission: string): Promise { - const [cmd, ...baseArgs] = this.getEmbedCommand(); - const args = [...baseArgs, '--profile', 'openclaw', 'bank', 'mission', this.bankId, sanitize(mission)]; - try { - const { stdout } = await execFileAsync(cmd, args, { maxBuffer: MAX_BUFFER }); - log.verbose(`bank mission set: ${stdout.trim()}`); - } catch (error) { - // Don't fail if mission set fails - bank might not exist yet, will be created on first retain - log.warn(`could not set bank mission (bank may not exist yet): ${error}`); - } - } - - private async ensureBankMissionHttp(mission: string): Promise { - const url = `${this.apiUrl}/v1/default/banks/${encodeURIComponent(this.bankId)}`; - const res = await fetch(url, { - method: 'PUT', - headers: this.httpHeaders(), - body: JSON.stringify({ mission }), - signal: AbortSignal.timeout(DEFAULT_TIMEOUT_MS), - }); - if (!res.ok) { - const body = await res.text().catch(() => ''); - throw new Error(`Failed to set bank mission (HTTP ${res.status}): ${body}`); - } - log.verbose('bank mission ensured via HTTP'); - } - - private async ensureBankMissionSubprocess(mission: string): Promise { - const [cmd, ...baseArgs] = this.getEmbedCommand(); - const args = [...baseArgs, '--profile', 'openclaw', 'bank', 'mission', this.bankId, sanitize(mission)]; - const { stdout } = await execFileAsync(cmd, args, { maxBuffer: MAX_BUFFER }); - log.verbose(`bank mission ensured: ${stdout.trim()}`); - } - - // --- retain --- - - async retain(request: RetainRequest): Promise { - if (this.httpMode) { - return this.retainHttp(request); - } - return this.retainSubprocess(request); - } - - private async retainHttp(request: RetainRequest): Promise { - const url = `${this.apiUrl}/v1/default/banks/${encodeURIComponent(this.bankId)}/memories`; - const body = { - items: [{ - content: request.content, - document_id: request.document_id || 'conversation', - metadata: request.metadata, - }], - document_tags: request.tags, - async: true, - }; - - const res = await fetch(url, { - method: 'POST', - headers: this.httpHeaders(), - body: JSON.stringify(body), - signal: AbortSignal.timeout(DEFAULT_TIMEOUT_MS), - }); - - if (!res.ok) { - const text = await res.text().catch(() => ''); - throw new Error(`Failed to retain memory (HTTP ${res.status}): ${text}`); - } - - const data = await res.json(); - log.verbose(`retained via HTTP (async): ${JSON.stringify(data).substring(0, 200)}`); - - return { - message: 'Memory queued for background processing', - document_id: request.document_id || 'conversation', - memory_unit_ids: [], - }; - } - - private async retainSubprocess(request: RetainRequest): Promise { - const docId = request.document_id || 'conversation'; - - // Write content to a temp file to avoid E2BIG (ARG_MAX) errors when passing - // large conversations as arguments. - const tempDir = join(tmpdir(), `hindsight_${randomBytes(8).toString('hex')}`); - const safeFilename = sanitizeFilename(docId); - const tempFile = join(tempDir, `${safeFilename}.txt`); - - try { - await mkdir(tempDir, { recursive: true }); - await writeFile(tempFile, sanitize(request.content), 'utf8'); - - const [cmd, ...baseArgs] = this.getEmbedCommand(); - const args = [...baseArgs, '--profile', 'openclaw', 'memory', 'retain-files', this.bankId, tempFile, '--async']; - - const { stdout } = await execFileAsync(cmd, args, { maxBuffer: MAX_BUFFER }); - log.verbose(`retained (async): ${stdout.trim()}`); - - return { - message: 'Memory queued for background processing', - document_id: docId, - memory_unit_ids: [], - }; - } catch (error) { - throw new Error(`Failed to retain memory: ${error}`, { cause: error }); - } finally { - await rm(tempDir, { recursive: true, force: true }).catch(() => {}); - } - } - - // --- recall --- - - async recall(request: RecallRequest, timeoutMs?: number): Promise { - if (this.httpMode) { - return this.recallHttp(request, timeoutMs); - } - return this.recallSubprocess(request, timeoutMs); - } - - private async recallHttp(request: RecallRequest, timeoutMs?: number): Promise { - const url = `${this.apiUrl}/v1/default/banks/${encodeURIComponent(this.bankId)}/memories/recall`; - // Defense-in-depth: truncate query to stay under API's 500-token limit - const MAX_QUERY_CHARS = 800; - const query = request.query.length > MAX_QUERY_CHARS - ? (log.warn(`truncating recall query from ${request.query.length} to ${MAX_QUERY_CHARS} chars`), - request.query.substring(0, MAX_QUERY_CHARS)) - : request.query; - const body: Record = { - query, - max_tokens: request.max_tokens || 1024, - }; - if (request.budget) { - body.budget = request.budget; - } - if (request.types) { - body.types = request.types; - } - - const res = await fetch(url, { - method: 'POST', - headers: this.httpHeaders(), - body: JSON.stringify(body), - signal: AbortSignal.timeout(timeoutMs ?? DEFAULT_TIMEOUT_MS), - }); - - if (!res.ok) { - const text = await res.text().catch(() => ''); - throw new Error(`Failed to recall memories (HTTP ${res.status}): ${text}`); - } - - return res.json() as Promise; - } - - private async recallSubprocess(request: RecallRequest, timeoutMs?: number): Promise { - const query = sanitize(request.query); - const maxTokens = request.max_tokens || 1024; - const [cmd, ...baseArgs] = this.getEmbedCommand(); - const args = [...baseArgs, '--profile', 'openclaw', 'memory', 'recall', this.bankId, query, '--output', 'json', '--max-tokens', String(maxTokens)]; - - try { - const { stdout } = await execFileAsync(cmd, args, { - maxBuffer: MAX_BUFFER, - timeout: timeoutMs ?? 30_000, // subprocess gets a longer default - }); - - return JSON.parse(stdout) as RecallResponse; - } catch (error) { - throw new Error(`Failed to recall memories: ${error}`, { cause: error }); - } - } - - async getBankStats(): Promise { - if (!this.httpMode) { - throw new Error('Bank stats are only available in HTTP mode'); - } - const url = `${this.apiUrl}/v1/default/banks/${encodeURIComponent(this.bankId)}/stats`; - const res = await fetch(url, { - method: 'GET', - headers: this.httpHeaders(), - signal: AbortSignal.timeout(DEFAULT_TIMEOUT_MS), - }); - if (!res.ok) { - const text = await res.text().catch(() => ''); - throw new Error(`Failed to get bank stats (HTTP ${res.status}): ${text}`); - } - return res.json() as Promise; - } -} diff --git a/hindsight-integrations/openclaw/src/embed-manager.ts b/hindsight-integrations/openclaw/src/embed-manager.ts deleted file mode 100644 index 499bfd0c..00000000 --- a/hindsight-integrations/openclaw/src/embed-manager.ts +++ /dev/null @@ -1,247 +0,0 @@ -import { spawn, ChildProcess } from 'child_process'; -import { join } from 'path'; -import { homedir } from 'os'; - -export class HindsightEmbedManager { - private process: ChildProcess | null = null; - private port: number; - private baseUrl: string; - private embedDir: string; - private llmProvider: string; - private llmApiKey: string; - private llmModel?: string; - private llmBaseUrl?: string; - private daemonIdleTimeout: number; - private embedVersion: string; - private embedPackagePath?: string; - - constructor( - port: number, - llmProvider: string, - llmApiKey: string, - llmModel?: string, - llmBaseUrl?: string, - daemonIdleTimeout: number = 0, // Default: never timeout - embedVersion: string = 'latest', // Default: latest - embedPackagePath?: string // Local path to hindsight package - ) { - // Use the configured port (default: 9077 from config) - this.port = port; - this.baseUrl = `http://127.0.0.1:${port}`; - this.embedDir = join(homedir(), '.openclaw', 'hindsight-embed'); - this.llmProvider = llmProvider; - this.llmApiKey = llmApiKey; - this.llmModel = llmModel; - this.llmBaseUrl = llmBaseUrl; - this.daemonIdleTimeout = daemonIdleTimeout; - this.embedVersion = embedVersion || 'latest'; - this.embedPackagePath = embedPackagePath; - } - - /** - * Get the command to run hindsight-embed (either local or from PyPI) - */ - private getEmbedCommand(): string[] { - if (this.embedPackagePath) { - // Local package: uv run --directory hindsight-embed - return ['uv', 'run', '--directory', this.embedPackagePath, 'hindsight-embed']; - } else { - // PyPI package: uvx hindsight-embed@version - const embedPackage = this.embedVersion ? `hindsight-embed@${this.embedVersion}` : 'hindsight-embed@latest'; - return ['uvx', embedPackage]; - } - } - - async start(): Promise { - console.log(`[Hindsight] Starting hindsight-embed daemon...`); - - // Build environment variables using standard HINDSIGHT_API_LLM_* variables - const env: NodeJS.ProcessEnv = { - ...process.env, - HINDSIGHT_API_LLM_PROVIDER: this.llmProvider, - HINDSIGHT_API_LLM_API_KEY: this.llmApiKey, - HINDSIGHT_EMBED_DAEMON_IDLE_TIMEOUT: this.daemonIdleTimeout.toString(), - }; - - if (this.llmModel) { - env['HINDSIGHT_API_LLM_MODEL'] = this.llmModel; - } - - // Pass through base URL for OpenAI-compatible providers (OpenRouter, etc.) - if (this.llmBaseUrl) { - env['HINDSIGHT_API_LLM_BASE_URL'] = this.llmBaseUrl; - } - - // On macOS, force CPU for embeddings/reranker to avoid MPS/Metal issues in daemon mode - if (process.platform === 'darwin') { - env['HINDSIGHT_API_EMBEDDINGS_LOCAL_FORCE_CPU'] = '1'; - env['HINDSIGHT_API_RERANKER_LOCAL_FORCE_CPU'] = '1'; - } - - // Configure "openclaw" profile using hindsight-embed configure (non-interactive) - console.log('[Hindsight] Configuring "openclaw" profile...'); - await this.configureProfile(env); - - // Start hindsight-embed daemon with openclaw profile - const embedCmd = this.getEmbedCommand(); - const startDaemon = spawn( - embedCmd[0], - [...embedCmd.slice(1), 'daemon', '--profile', 'openclaw', 'start'], - { - stdio: 'pipe', - } - ); - - // Collect output - let output = ''; - startDaemon.stdout?.on('data', (data) => { - const text = data.toString(); - output += text; - console.log(`[Hindsight] ${text.trim()}`); - }); - - startDaemon.stderr?.on('data', (data) => { - const text = data.toString(); - output += text; - console.error(`[Hindsight] ${text.trim()}`); - }); - - // Wait for daemon start command to complete - await new Promise((resolve, reject) => { - startDaemon.on('exit', (code) => { - if (code === 0) { - console.log('[Hindsight] Daemon start command completed'); - resolve(); - } else { - reject(new Error(`Daemon start failed with code ${code}: ${output}`)); - } - }); - - startDaemon.on('error', (error) => { - reject(error); - }); - }); - - // Wait for server to be ready - await this.waitForReady(); - console.log('[Hindsight] Daemon is ready'); - } - - async stop(): Promise { - console.log('[Hindsight] Stopping hindsight-embed daemon...'); - - const embedCmd = this.getEmbedCommand(); - const stopDaemon = spawn(embedCmd[0], [...embedCmd.slice(1), 'daemon', '--profile', 'openclaw', 'stop'], { - stdio: 'pipe', - }); - - await new Promise((resolve) => { - stopDaemon.on('exit', () => { - console.log('[Hindsight] Daemon stopped'); - resolve(); - }); - - stopDaemon.on('error', (error) => { - console.error('[Hindsight] Error stopping daemon:', error); - resolve(); // Resolve anyway - }); - - // Timeout after 5 seconds - setTimeout(() => { - console.log('[Hindsight] Daemon stop timeout'); - resolve(); - }, 5000); - }); - } - - private async waitForReady(maxAttempts = 30): Promise { - console.log('[Hindsight] Waiting for daemon to be ready...'); - for (let i = 0; i < maxAttempts; i++) { - try { - const response = await fetch(`${this.baseUrl}/health`); - if (response.ok) { - console.log('[Hindsight] Daemon health check passed'); - return; - } - } catch { - // Not ready yet - } - await new Promise((resolve) => setTimeout(resolve, 1000)); - } - throw new Error('Hindsight daemon failed to become ready within 30 seconds'); - } - - getBaseUrl(): string { - return this.baseUrl; - } - - isRunning(): boolean { - return this.process !== null; - } - - async checkHealth(): Promise { - try { - const response = await fetch(`${this.baseUrl}/health`, { signal: AbortSignal.timeout(2000) }); - return response.ok; - } catch { - return false; - } - } - - private async configureProfile(env: NodeJS.ProcessEnv): Promise { - // Build profile create command args with --merge, --port and --env flags - // Use --merge to allow updating existing profile - const createArgs = ['profile', 'create', 'openclaw', '--merge', '--port', this.port.toString()]; - - // Add all environment variables as --env flags - const envVars = [ - 'HINDSIGHT_API_LLM_PROVIDER', - 'HINDSIGHT_API_LLM_MODEL', - 'HINDSIGHT_API_LLM_API_KEY', - 'HINDSIGHT_API_LLM_BASE_URL', - 'HINDSIGHT_EMBED_DAEMON_IDLE_TIMEOUT', - 'HINDSIGHT_API_EMBEDDINGS_LOCAL_FORCE_CPU', - 'HINDSIGHT_API_RERANKER_LOCAL_FORCE_CPU', - ]; - - for (const envVar of envVars) { - if (env[envVar]) { - createArgs.push('--env', `${envVar}=${env[envVar]}`); - } - } - - // Run profile create command (non-interactive, overwrites if exists) - const embedCmd = this.getEmbedCommand(); - const create = spawn(embedCmd[0], [...embedCmd.slice(1), ...createArgs], { - stdio: 'pipe', - }); - - let output = ''; - create.stdout?.on('data', (data) => { - const text = data.toString(); - output += text; - console.log(`[Hindsight] ${text.trim()}`); - }); - - create.stderr?.on('data', (data) => { - const text = data.toString(); - output += text; - console.error(`[Hindsight] ${text.trim()}`); - }); - - await new Promise((resolve, reject) => { - create.on('exit', (code) => { - if (code === 0) { - console.log('[Hindsight] Profile "openclaw" configured successfully'); - resolve(); - } else { - reject(new Error(`Profile create failed with code ${code}: ${output}`)); - } - }); - - create.on('error', (error) => { - reject(error); - }); - }); - } -} diff --git a/hindsight-integrations/openclaw/src/index.test.ts b/hindsight-integrations/openclaw/src/index.test.ts index c1da6fde..a8ce7b4a 100644 --- a/hindsight-integrations/openclaw/src/index.test.ts +++ b/hindsight-integrations/openclaw/src/index.test.ts @@ -241,7 +241,7 @@ describe('buildRetainRequest', () => { expect(request).toEqual({ content: 'hello world', - document_id: 'openclaw:agent:main:main:turn:000001', + documentId: 'openclaw:agent:main:main:turn:000001', metadata: { retained_at: expect.any(String), message_count: '2', @@ -274,7 +274,7 @@ describe('buildRetainRequest', () => { windowTurns: 2, }); - expect(request.document_id).toBe('openclaw:agent:agentname:discord:group:123:topic:456:window:000002'); + expect(request.documentId).toBe('openclaw:agent:agentname:discord:group:123:topic:456:window:000002'); expect(request.metadata).toMatchObject({ source: 'openclaw', retention_scope: 'window', diff --git a/hindsight-integrations/openclaw/src/index.ts b/hindsight-integrations/openclaw/src/index.ts index 4d881aaf..5f370f63 100644 --- a/hindsight-integrations/openclaw/src/index.ts +++ b/hindsight-integrations/openclaw/src/index.ts @@ -1,6 +1,6 @@ import type { MoltbotPluginAPI, PluginConfig, PluginHookAgentContext, MemoryResult, RetainRequest } from './types.js'; -import { HindsightEmbedManager } from './embed-manager.js'; -import { HindsightClient, type HindsightClientOptions } from './client.js'; +import { HindsightServer, type Logger } from '@vectorize-io/hindsight-all'; +import { HindsightClient, type HindsightClientOptions } from '@vectorize-io/hindsight-client'; import { RetainQueue } from './retain-queue.js'; import { compileSessionPatterns, matchesSessionPattern } from './session-patterns.js'; import { createHash } from 'crypto'; @@ -11,6 +11,16 @@ import { configureLogger, setApiLogger, stopLogger } from './logger.js'; import { mkdirSync } from 'fs'; import { homedir } from 'os'; +// Logger adapter that routes the embed wrapper's output through openclaw's +// batched structured logger so messages share the same prefix and respect +// the configured log level. +const embedLogger: Logger = { + debug: (msg) => log.verbose(msg), + info: (msg) => log.info(msg), + warn: (msg) => log.warn(msg), + error: (msg) => log.error(msg), +}; + // Debug logging: silent by default, enable with debug: true or logLevel: 'debug' let debugEnabled = false; const debug = (...args: unknown[]) => { @@ -18,7 +28,7 @@ const debug = (...args: unknown[]) => { }; // Module-level state -let embedManager: HindsightEmbedManager | null = null; +let hindsightServer: HindsightServer | null = null; let client: HindsightClient | null = null; let clientOptions: HindsightClientOptions | null = null; let initPromise: Promise | null = null; @@ -28,15 +38,93 @@ let usingExternalApi = false; // Track if using external API (skip daemon manage // Store the current plugin config for bank ID derivation let currentPluginConfig: PluginConfig | null = null; -// Track which banks have had their mission set (to avoid re-setting on every request) +// Track which banks have had their mission set (to avoid re-setting on every request). +// Under the old bespoke client we also cached a client instance per bank because the +// client carried a mutable bankId. HindsightClient takes bankId as a parameter on every +// call, so no per-bank caching is needed anymore — one module-level client is enough. const banksWithMissionSet = new Set(); -// Use dedicated client instances per bank to avoid cross-session bankId mutation races. -const clientsByBankId = new Map(); -const MAX_TRACKED_BANK_CLIENTS = 10_000; // In-flight recall deduplication: concurrent recalls for the same bank reuse one promise import type { RecallResponse } from './types.js'; const inflightRecalls = new Map>(); + +// Lightweight bank-scoped facade over HindsightClient. Created per-request via +// getClientForContext() so hook bodies can keep their bankId-implicit style +// without going back to a stateful setBankId pattern. Also bridges the +// small shape differences (e.g. RetainRequest.metadata is Record +// at build time; HindsightClient wants Record). +export interface BankScopedClient { + readonly bankId: string; + retain(req: RetainRequest): Promise; + recall( + req: { + query: string; + maxTokens?: number; + budget?: 'low' | 'mid' | 'high'; + types?: Array<'world' | 'experience' | 'observation'>; + }, + timeoutMs?: number, + ): Promise; + setMission(mission: string): Promise; +} + +function scopeClient(c: HindsightClient, bankId: string): BankScopedClient { + return { + bankId, + async retain(req) { + await c.retain(bankId, req.content, { + documentId: req.documentId, + metadata: toStringMetadata(req.metadata), + tags: req.tags, + async: true, + }); + }, + async recall(req, timeoutMs) { + const call = c.recall(bankId, req.query, { + maxTokens: req.maxTokens, + budget: req.budget, + types: req.types, + }); + if (!timeoutMs) return call; + // The generated client doesn't accept a per-call AbortSignal, so we race + // against a TimeoutError here. The before_prompt_build caller already + // special-cases `DOMException { name: 'TimeoutError' }` from the old + // bespoke client, so we preserve that contract. + return Promise.race([ + call, + new Promise((_, reject) => + setTimeout( + () => reject(new DOMException(`Recall timed out after ${timeoutMs}ms`, 'TimeoutError')), + timeoutMs, + ), + ), + ]); + }, + async setMission(mission) { + // createBank upserts the reflect mission. openclaw's old setBankMission + // went through a dedicated PUT endpoint; this call lands on the same + // server-side handler via the non-deprecated path. + await c.createBank(bankId, { reflectMission: mission }); + }, + }; +} + +/** + * The generated client's metadata type is `Record`; the + * openclaw builder uses `Record` because some fields come + * from optional plugin context. Drop undefined/null, stringify the rest. + */ +function toStringMetadata( + input: Record | undefined, +): Record | undefined { + if (!input) return undefined; + const out: Record = {}; + for (const [k, v] of Object.entries(input)) { + if (v === undefined || v === null) continue; + out[k] = typeof v === 'string' ? v : String(v); + } + return out; +} const turnCountBySession = new Map(); const MAX_TRACKED_SESSIONS = 10_000; const DEFAULT_RECALL_TIMEOUT_MS = 10_000; @@ -75,7 +163,7 @@ async function flushRetainQueue(): Promise { let failed = 0; try { - if (!clientOptions) return; // no client config — can't flush + if (!client) return; // no client yet — can't flush // Cleanup expired items first retainQueue.cleanup(); @@ -84,17 +172,11 @@ async function flushRetainQueue(): Promise { const flushedIds: string[] = []; for (const item of items) { try { - let bankClient = clientsByBankId.get(item.bankId); - if (!bankClient) { - bankClient = new HindsightClient(clientOptions); - bankClient.setBankId(item.bankId); - clientsByBankId.set(item.bankId, bankClient); - } - - await bankClient.retain({ - content: item.content, - document_id: item.documentId, - metadata: item.metadata, + await client.retain(item.bankId, item.content, { + documentId: item.documentId, + metadata: toStringMetadata(item.metadata), + tags: item.tags, + async: true, }); flushedIds.push(item.id); @@ -171,14 +253,17 @@ async function lazyReinit(configOverride?: PluginConfig): Promise { const llmConfig = detectLLMConfig(config); clientOptions = buildClientOptions(llmConfig, config, externalApi); - clientsByBankId.clear(); banksWithMissionSet.clear(); client = new HindsightClient(clientOptions); - const defaultBankId = deriveBankId(undefined, config); - client.setBankId(defaultBankId); if (config.bankMission && usesStaticBank(config)) { - await client.setBankMission(config.bankMission); + const bankId = getStaticBankId(config); + try { + await scopeClient(client, bankId).setMission(config.bankMission); + banksWithMissionSet.add(bankId); + } catch (err) { + log.warn(`could not set bank mission for ${bankId}: ${err instanceof Error ? err.message : err}`); + } } usingExternalApi = true; @@ -222,38 +307,20 @@ if (typeof global !== 'undefined') { } }, /** - * Get a client configured for a specific agent context. - * Derives the bank ID from the context for per-channel isolation. - * Also ensures the bank mission is set on first use. + * Get a bank-scoped client handle for a specific agent context. + * Derives the bank ID from the context for per-channel isolation and + * ensures the bank mission is set on first use. */ - getClientForContext: async (ctx: PluginHookAgentContext | undefined) => { - if (!client) {return null;} + getClientForContext: async (ctx: PluginHookAgentContext | undefined): Promise => { + if (!client) return null; const config = currentPluginConfig || {}; - if (usesStaticBank(config)) { - return client; - } - const bankId = deriveBankId(ctx, config); - let bankClient = clientsByBankId.get(bankId); - if (!bankClient) { - if (!clientOptions) { - return null; - } - bankClient = new HindsightClient(clientOptions); - bankClient.setBankId(bankId); - clientsByBankId.set(bankId, bankClient); - if (clientsByBankId.size > MAX_TRACKED_BANK_CLIENTS) { - const oldestKey = clientsByBankId.keys().next().value; - if (oldestKey) { - clientsByBankId.delete(oldestKey); - banksWithMissionSet.delete(oldestKey); - } - } - } + const bankId = usesStaticBank(config) ? getStaticBankId(config) : deriveBankId(ctx, config); + const scoped = scopeClient(client, bankId); - // Set bank mission on first use of this bank (if configured) - if (config.bankMission && !usesStaticBank(config) && !banksWithMissionSet.has(bankId)) { + // Set bank mission on first use of this bank (if configured). + if (config.bankMission && !banksWithMissionSet.has(bankId)) { try { - await bankClient.setBankMission(config.bankMission); + await scoped.setMission(config.bankMission); banksWithMissionSet.add(bankId); debug(`[Hindsight] Set mission for new bank: ${bankId}`); } catch (error) { @@ -262,7 +329,7 @@ if (typeof global !== 'undefined') { } } - return bankClient; + return scoped; }, getPluginConfig: () => currentPluginConfig, }; @@ -745,19 +812,21 @@ export function detectExternalApi(pluginConfig?: PluginConfig): { } /** - * Build HindsightClientOptions from LLM config, plugin config, and external API settings. + * Build HindsightClientOptions for the generated hindsight-client. In + * external-API mode we use the configured URL/token; in local daemon mode + * the caller overrides with the daemon's base URL after start(). + * The llmConfig parameter is currently only consumed by the daemon manager + * (via env vars); it's kept on the client builder signature so callers + * don't need to branch and so future features can forward it. */ export function buildClientOptions( - llmConfig: { provider?: string; apiKey?: string; model?: string }, - pluginCfg: PluginConfig, + _llmConfig: { provider?: string; apiKey?: string; model?: string }, + _pluginCfg: PluginConfig, externalApi: { apiUrl: string | null; apiToken: string | null }, ): HindsightClientOptions { return { - llmModel: llmConfig.model, - embedVersion: pluginCfg.embedVersion, - embedPackagePath: pluginCfg.embedPackagePath, - apiUrl: externalApi.apiUrl ?? undefined, - apiToken: externalApi.apiToken ?? undefined, + baseUrl: externalApi.apiUrl ?? '', + apiKey: externalApi.apiToken ?? undefined, }; } @@ -964,23 +1033,25 @@ export default function (api: MoltbotPluginAPI) { debug('[Hindsight] External API mode - skipping local daemon...'); await checkExternalApiHealth(externalApi.apiUrl, externalApi.apiToken); - // Initialize client with direct HTTP mode - debug('[Hindsight] Creating HindsightClient (HTTP mode)...'); + // Initialize client for external API + debug('[Hindsight] Creating HindsightClient (external API)...'); clientOptions = buildClientOptions(llmConfig, pluginConfig, externalApi); - clientsByBankId.clear(); banksWithMissionSet.clear(); client = new HindsightClient(clientOptions); - // Set default bank (will be overridden per-request when dynamic bank IDs are enabled) const defaultBankId = deriveBankId(undefined, pluginConfig); debug(`[Hindsight] Default bank: ${defaultBankId}`); - client.setBankId(defaultBankId); // Note: Bank mission will be set per-bank when dynamic bank IDs are enabled - // For now, set it on the default bank + // For now, set it on the static default bank only. if (pluginConfig.bankMission && usesStaticBank(pluginConfig)) { debug(`[Hindsight] Setting bank mission...`); - await client.setBankMission(pluginConfig.bankMission); + try { + await scopeClient(client, defaultBankId).setMission(pluginConfig.bankMission); + banksWithMissionSet.add(defaultBankId); + } catch (err) { + log.warn(`could not set bank mission for ${defaultBankId}: ${err instanceof Error ? err.message : err}`); + } } if (!isInitialized) { @@ -993,39 +1064,45 @@ export default function (api: MoltbotPluginAPI) { debug('[Hindsight] ✓ Ready (external API mode)'); } else { // Local daemon mode - start hindsight-embed daemon - debug('[Hindsight] Creating HindsightEmbedManager...'); - embedManager = new HindsightEmbedManager( - apiPort, - llmConfig.provider || "", - llmConfig.apiKey || "", - llmConfig.model, - llmConfig.baseUrl, - pluginConfig.daemonIdleTimeout, - pluginConfig.embedVersion, - pluginConfig.embedPackagePath - ); + debug('[Hindsight] Creating HindsightServer...'); + hindsightServer = new HindsightServer({ + profile: 'openclaw', + port: apiPort, + embedVersion: pluginConfig.embedVersion, + embedPackagePath: pluginConfig.embedPackagePath, + env: { + HINDSIGHT_API_LLM_PROVIDER: llmConfig.provider || '', + HINDSIGHT_API_LLM_API_KEY: llmConfig.apiKey || '', + HINDSIGHT_API_LLM_MODEL: llmConfig.model, + HINDSIGHT_API_LLM_BASE_URL: llmConfig.baseUrl, + HINDSIGHT_EMBED_DAEMON_IDLE_TIMEOUT: String(pluginConfig.daemonIdleTimeout ?? 0), + }, + logger: embedLogger, + }); // Start the embedded server debug('[Hindsight] Starting embedded server...'); - await embedManager.start(); + await hindsightServer.start(); - // Initialize client (local daemon mode — no apiUrl) - debug('[Hindsight] Creating HindsightClient (subprocess mode)...'); - clientOptions = buildClientOptions(llmConfig, pluginConfig, { apiUrl: null, apiToken: null }); - clientsByBankId.clear(); + // Initialize client pointed at the local daemon URL + debug('[Hindsight] Creating HindsightClient (local daemon)...'); + clientOptions = { baseUrl: hindsightServer.getBaseUrl() }; banksWithMissionSet.clear(); client = new HindsightClient(clientOptions); - // Set default bank (will be overridden per-request when dynamic bank IDs are enabled) const defaultBankId = deriveBankId(undefined, pluginConfig); debug(`[Hindsight] Default bank: ${defaultBankId}`); - client.setBankId(defaultBankId); // Note: Bank mission will be set per-bank when dynamic bank IDs are enabled - // For now, set it on the default bank + // For now, set it on the static default bank only. if (pluginConfig.bankMission && usesStaticBank(pluginConfig)) { debug(`[Hindsight] Setting bank mission...`); - await client.setBankMission(pluginConfig.bankMission); + try { + await scopeClient(client, defaultBankId).setMission(pluginConfig.bankMission); + banksWithMissionSet.add(defaultBankId); + } catch (err) { + log.warn(`could not set bank mission for ${defaultBankId}: ${err instanceof Error ? err.message : err}`); + } } if (!isInitialized) { @@ -1064,15 +1141,14 @@ export default function (api: MoltbotPluginAPI) { // Reset state for reinitialization attempt client = null; clientOptions = null; - clientsByBankId.clear(); banksWithMissionSet.clear(); isInitialized = false; } } } else { // Local daemon mode: check daemon health (handles SIGUSR1 restart case) - if (embedManager && isInitialized) { - const healthy = await embedManager.checkHealth(); + if (hindsightServer && isInitialized) { + const healthy = await hindsightServer.checkHealth(); if (healthy) { debug('[Hindsight] Daemon is healthy'); return; @@ -1080,10 +1156,9 @@ export default function (api: MoltbotPluginAPI) { debug('[Hindsight] Daemon is not responding - reinitializing...'); // Reset state for reinitialization - embedManager = null; + hindsightServer = null; client = null; clientOptions = null; - clientsByBankId.clear(); banksWithMissionSet.clear(); isInitialized = false; } @@ -1109,42 +1184,52 @@ export default function (api: MoltbotPluginAPI) { await checkExternalApiHealth(externalApi.apiUrl, externalApi.apiToken); clientOptions = buildClientOptions(llmConfig, reinitPluginConfig, externalApi); - clientsByBankId.clear(); banksWithMissionSet.clear(); client = new HindsightClient(clientOptions); const defaultBankId = deriveBankId(undefined, reinitPluginConfig); - client.setBankId(defaultBankId); if (reinitPluginConfig.bankMission && usesStaticBank(reinitPluginConfig)) { - await client.setBankMission(reinitPluginConfig.bankMission); + try { + await scopeClient(client, defaultBankId).setMission(reinitPluginConfig.bankMission); + banksWithMissionSet.add(defaultBankId); + } catch (err) { + log.warn(`could not set bank mission for ${defaultBankId}: ${err instanceof Error ? err.message : err}`); + } } isInitialized = true; debug('[Hindsight] Reinitialization complete (external API mode)'); } else { // Local daemon mode - embedManager = new HindsightEmbedManager( - apiPort, - llmConfig.provider || "", - llmConfig.apiKey || "", - llmConfig.model, - llmConfig.baseUrl, - reinitPluginConfig.daemonIdleTimeout, - reinitPluginConfig.embedVersion, - reinitPluginConfig.embedPackagePath - ); + hindsightServer = new HindsightServer({ + profile: 'openclaw', + port: apiPort, + embedVersion: reinitPluginConfig.embedVersion, + embedPackagePath: reinitPluginConfig.embedPackagePath, + env: { + HINDSIGHT_API_LLM_PROVIDER: llmConfig.provider || '', + HINDSIGHT_API_LLM_API_KEY: llmConfig.apiKey || '', + HINDSIGHT_API_LLM_MODEL: llmConfig.model, + HINDSIGHT_API_LLM_BASE_URL: llmConfig.baseUrl, + HINDSIGHT_EMBED_DAEMON_IDLE_TIMEOUT: String(reinitPluginConfig.daemonIdleTimeout ?? 0), + }, + logger: embedLogger, + }); - await embedManager.start(); + await hindsightServer.start(); - clientOptions = buildClientOptions(llmConfig, reinitPluginConfig, { apiUrl: null, apiToken: null }); - clientsByBankId.clear(); + clientOptions = { baseUrl: hindsightServer.getBaseUrl() }; banksWithMissionSet.clear(); client = new HindsightClient(clientOptions); const defaultBankId = deriveBankId(undefined, reinitPluginConfig); - client.setBankId(defaultBankId); if (reinitPluginConfig.bankMission && usesStaticBank(reinitPluginConfig)) { - await client.setBankMission(reinitPluginConfig.bankMission); + try { + await scopeClient(client, defaultBankId).setMission(reinitPluginConfig.bankMission); + banksWithMissionSet.add(defaultBankId); + } catch (err) { + log.warn(`could not set bank mission for ${defaultBankId}: ${err instanceof Error ? err.message : err}`); + } } isInitialized = true; @@ -1158,9 +1243,9 @@ export default function (api: MoltbotPluginAPI) { debug('[Hindsight] Service stopping...'); // Only stop daemon if in local mode - if (!usingExternalApi && embedManager) { - await embedManager.stop(); - embedManager = null; + if (!usingExternalApi && hindsightServer) { + await hindsightServer.stop(); + hindsightServer = null; } // Close retain queue @@ -1179,7 +1264,6 @@ export default function (api: MoltbotPluginAPI) { client = null; clientOptions = null; - clientsByBankId.clear(); banksWithMissionSet.clear(); isInitialized = false; @@ -1314,7 +1398,7 @@ export default function (api: MoltbotPluginAPI) { recallPromise = existing; } else { const recallTimeoutMs = pluginConfig.recallTimeoutMs ?? DEFAULT_RECALL_TIMEOUT_MS; - recallPromise = client.recall({ query: prompt, max_tokens: pluginConfig.recallMaxTokens || 1024, budget: pluginConfig.recallBudget, types: pluginConfig.recallTypes }, recallTimeoutMs); + recallPromise = client.recall({ query: prompt, maxTokens: pluginConfig.recallMaxTokens || 1024, budget: pluginConfig.recallBudget, types: pluginConfig.recallTypes }, recallTimeoutMs); inflightRecalls.set(recallKey, recallPromise); void recallPromise.catch(() => {}).finally(() => inflightRecalls.delete(recallKey)); } @@ -1495,12 +1579,12 @@ ${memoriesFormatted} ); // Retain to Hindsight - debug(`[Hindsight] Retaining to bank ${bankId}, document: ${retainRequest.document_id}, chars: ${transcript.length}\n---\n${transcript.substring(0, 500)}${transcript.length > 500 ? '\n...(truncated)' : ''}\n---`); + debug(`[Hindsight] Retaining to bank ${bankId}, document: ${retainRequest.documentId}, chars: ${transcript.length}\n---\n${transcript.substring(0, 500)}${transcript.length > 500 ? '\n...(truncated)' : ''}\n---`); try { await client.retain(retainRequest); log.trackRetain(bankId, messageCount); - debug(`[Hindsight] Retained ${messageCount} messages to bank ${bankId} for session ${retainRequest.document_id}`); + debug(`[Hindsight] Retained ${messageCount} messages to bank ${bankId} for session ${retainRequest.documentId}`); // After a successful retain, try flushing any queued items if (retainQueue && retainQueue.size() > 0) { @@ -1586,7 +1670,7 @@ export function buildRetainRequest( return { content: transcript, - document_id: documentId, + documentId: documentId, metadata: { retained_at: new Date(now).toISOString(), message_count: String(messageCount), diff --git a/hindsight-integrations/openclaw/src/remote-no-llm.test.ts b/hindsight-integrations/openclaw/src/remote-no-llm.test.ts deleted file mode 100644 index 8d243c87..00000000 --- a/hindsight-integrations/openclaw/src/remote-no-llm.test.ts +++ /dev/null @@ -1,22 +0,0 @@ -import { describe, it, expect } from 'vitest'; -import { HindsightClient } from './client.js'; - -describe('HindsightClient remote mode without LLM config', () => { - it('should initialize successfully with only apiUrl and apiToken', () => { - const client = new HindsightClient({ - apiUrl: 'https://api.example.com', - apiToken: 'secret-token', - }); - - expect(client).toBeDefined(); - expect(client).toBeInstanceOf(HindsightClient); - }); - - it('should allow initialization with partial config', () => { - const client = new HindsightClient({ - apiUrl: 'https://api.example.com', - llmModel: 'gpt-4o-mini', - }); - expect(client).toBeDefined(); - }); -}); diff --git a/hindsight-integrations/openclaw/src/retain-queue.ts b/hindsight-integrations/openclaw/src/retain-queue.ts index de0813c1..467f854c 100644 --- a/hindsight-integrations/openclaw/src/retain-queue.ts +++ b/hindsight-integrations/openclaw/src/retain-queue.ts @@ -1,16 +1,30 @@ /** * JSONL-backed retain queue for buffering failed HTTP retains. * - * When the external Hindsight API is unreachable, retain requests are stored - * as JSON lines in a local file and flushed once connectivity is restored. - * Only used in external API mode — local daemon mode handles its own persistence. + * When the remote Hindsight API is unreachable, retain requests are stashed + * in a local JSONL file and flushed later. Only used in external API mode — + * the local daemon handles its own persistence. * - * Zero dependencies — uses only Node built-ins. + * Zero runtime dependencies; uses only Node built-ins. */ -import { readFileSync, writeFileSync, appendFileSync, existsSync, renameSync, unlinkSync } from 'fs'; +import { + readFileSync, + writeFileSync, + appendFileSync, + existsSync, + renameSync, + unlinkSync, +} from 'fs'; import { randomBytes } from 'crypto'; -import type { RetainRequest } from './types.js'; + +/** The subset of a retain payload the queue needs to persist and replay. */ +export interface QueuedRetainPayload { + content: string; + documentId?: string; + metadata?: Record; + tags?: string[]; +} export interface QueuedRetain { id: string; @@ -18,43 +32,84 @@ export interface QueuedRetain { content: string; documentId: string; metadata: Record; + tags?: string[]; createdAt: string; // ISO 8601 } export interface RetainQueueOptions { - /** Path to the JSONL queue file */ + /** Path to the JSONL queue file. The parent directory must already exist. */ filePath: string; - /** Max age in ms for queued items. -1 = keep forever (default) */ + /** Max age in ms for queued items. `-1` (default) keeps items forever. */ maxAgeMs?: number; } export class RetainQueue { - private filePath: string; - private maxAgeMs: number; + private readonly filePath: string; + private readonly maxAgeMs: number; private cachedSize: number; constructor(opts: RetainQueueOptions) { this.filePath = opts.filePath; this.maxAgeMs = opts.maxAgeMs ?? -1; - // Initialize cached size from file this.cachedSize = this.readAll().length; } - /** Store a failed retain for later delivery — exact same payload as the HTTP request */ - enqueue(bankId: string, request: RetainRequest, metadata?: Record): void { + /** Append a failed retain for later delivery. */ + enqueue(bankId: string, request: QueuedRetainPayload, metadata?: Record): void { const item: QueuedRetain = { id: `${Date.now()}-${randomBytes(4).toString('hex')}`, bankId, content: request.content, - documentId: request.document_id || 'conversation', + documentId: request.documentId || 'conversation', metadata: metadata || request.metadata || {}, + tags: request.tags, createdAt: new Date().toISOString(), }; appendFileSync(this.filePath, JSON.stringify(item) + '\n', 'utf8'); this.cachedSize++; } - /** Read all pending items from file */ + /** Get up to `limit` oldest pending items (FIFO). */ + peek(limit = 50): QueuedRetain[] { + return this.readAll().slice(0, limit); + } + + /** Remove a single item by id. */ + remove(id: string): void { + const items = this.readAll().filter((i) => i.id !== id); + this.writeAll(items); + } + + /** Remove multiple items by id in a single file rewrite. */ + removeMany(ids: string[]): void { + const idSet = new Set(ids); + const items = this.readAll().filter((i) => !idSet.has(i.id)); + this.writeAll(items); + } + + /** Number of items waiting (cached, O(1)). */ + size(): number { + return this.cachedSize; + } + + /** Drop items older than `maxAgeMs`. No-op when `maxAgeMs < 0`. */ + cleanup(): number { + if (this.maxAgeMs < 0) return 0; + const cutoff = Date.now() - this.maxAgeMs; + const items = this.readAll(); + const kept = items.filter((i) => new Date(i.createdAt).getTime() >= cutoff); + const removed = items.length - kept.length; + if (removed > 0) this.writeAll(kept); + return removed; + } + + /** No-op — kept for API symmetry with DB-backed queues. */ + close(): void { + /* nothing to close */ + } + + // ------------------------------------------------------------------------- + private readAll(): QueuedRetain[] { if (!existsSync(this.filePath)) return []; const content = readFileSync(this.filePath, 'utf8').trim(); @@ -70,53 +125,20 @@ export class RetainQueue { return items; } - /** Atomically rewrite the file with the given items */ + /** Atomically rewrite the file with the given items. */ private writeAll(items: QueuedRetain[]): void { if (items.length === 0) { - try { unlinkSync(this.filePath); } catch { /* already gone */ } + try { + unlinkSync(this.filePath); + } catch { + /* already gone */ + } this.cachedSize = 0; return; } const tmpPath = this.filePath + '.tmp'; - writeFileSync(tmpPath, items.map(i => JSON.stringify(i)).join('\n') + '\n', 'utf8'); + writeFileSync(tmpPath, items.map((i) => JSON.stringify(i)).join('\n') + '\n', 'utf8'); renameSync(tmpPath, this.filePath); this.cachedSize = items.length; } - - /** Get oldest pending items (FIFO) */ - peek(limit = 50): QueuedRetain[] { - return this.readAll().slice(0, limit); - } - - /** Remove a single item by id */ - remove(id: string): void { - const items = this.readAll().filter(i => i.id !== id); - this.writeAll(items); - } - - /** Remove multiple items by id in a single file rewrite */ - removeMany(ids: string[]): void { - const idSet = new Set(ids); - const items = this.readAll().filter(i => !idSet.has(i.id)); - this.writeAll(items); - } - - /** Number of items waiting (cached, O(1)) */ - size(): number { - return this.cachedSize; - } - - /** Remove items older than maxAgeMs (no-op when maxAgeMs is -1) */ - cleanup(): number { - if (this.maxAgeMs < 0) return 0; - const cutoff = Date.now() - this.maxAgeMs; - const items = this.readAll(); - const kept = items.filter(i => new Date(i.createdAt).getTime() >= cutoff); - const removed = items.length - kept.length; - if (removed > 0) this.writeAll(kept); - return removed; - } - - /** No-op for JSONL (no connection to close), kept for API compatibility */ - close(): void {} } diff --git a/hindsight-integrations/openclaw/src/types.ts b/hindsight-integrations/openclaw/src/types.ts index b23851c3..56f8058a 100644 --- a/hindsight-integrations/openclaw/src/types.ts +++ b/hindsight-integrations/openclaw/src/types.ts @@ -100,35 +100,35 @@ export interface ServiceConfig { stop(): Promise; } +// ----------------------------------------------------------------------------- // Hindsight API types +// ----------------------------------------------------------------------------- +// MemoryResult / RecallResponse / ReflectResponse come from the generated +// hindsight-client SDK. We alias MemoryResult → RecallResult so existing code +// paths (formatMemories, etc.) keep the old name. +export type { RecallResult as MemoryResult, RecallResponse, ReflectResponse } from '@vectorize-io/hindsight-client'; + +/** + * Internal retain payload shape built by `buildRetainRequest`. Not a + * re-export from the generated client — the generated client's retain() + * takes bankId + content + options as positional args, whereas we build up a + * single object inside the plugin and translate it at the call site. Keeping + * this type local means tests can assert the shape without pulling in + * generated types. + */ export interface RetainRequest { content: string; - document_id?: string; + documentId?: string; metadata?: Record; tags?: string[]; } -export interface RetainResponse { - message: string; - document_id: string; - memory_unit_ids: string[]; -} - -export interface RecallRequest { - query: string; - max_tokens?: number; - budget?: 'low' | 'mid' | 'high'; - types?: Array<'world' | 'experience' | 'observation'>; -} - -export interface RecallResponse { - results: MemoryResult[]; - entities: Record | null; - trace: unknown | null; - chunks: unknown | null; -} - +/** + * Stats returned by `GET /v1/default/banks/{bank_id}/stats`. The generated + * high-level client does not expose this endpoint yet; backfill calls it + * directly via `fetch`. + */ export interface BankStats { bank_id: string; total_nodes: number; @@ -144,29 +144,3 @@ export interface BankStats { links_by_fact_type?: Record; links_breakdown?: Record; } - -export interface MemoryResult { - id: string; - text: string; - type: string; - entities: string[]; - context: string; - occurred_start: string | null; - occurred_end: string | null; - mentioned_at: string | null; - document_id: string | null; - metadata: Record | null; - chunk_id: string | null; - tags: string[]; -} - -export interface CreateBankRequest { - name: string; - background_context?: string; -} - -export interface CreateBankResponse { - bank_id: string; - name: string; - created_at: string; -} diff --git a/hindsight-integrations/openclaw/tests/hooks.integration.test.ts b/hindsight-integrations/openclaw/tests/hooks.integration.test.ts index 0921aa27..61eb5a5d 100644 --- a/hindsight-integrations/openclaw/tests/hooks.integration.test.ts +++ b/hindsight-integrations/openclaw/tests/hooks.integration.test.ts @@ -13,13 +13,11 @@ * npm run test:integration */ -import { describe, it, expect, beforeAll, afterAll, afterEach, vi } from 'vitest'; -import type { HindsightClient } from '../src/client.js'; +import { describe, it, expect, beforeAll, afterAll, afterEach, vi, type MockInstance } from 'vitest'; +import type { RecallResponse, RetainResponse } from '@vectorize-io/hindsight-client'; import type { MoltbotPluginAPI, PluginConfig } from '../src/types.js'; -import type { RecallResponse, RetainResponse } from '../src/types.js'; const HINDSIGHT_API_URL = process.env.HINDSIGHT_API_URL || 'http://localhost:8888'; -const HINDSIGHT_API_TOKEN = process.env.HINDSIGHT_API_TOKEN || ''; // --------------------------------------------------------------------------- // Helpers @@ -59,6 +57,11 @@ function createMockApi(pluginConfig: Partial = {}): MockApiHandle }, }, }, + logger: { + info: (msg: string) => console.log(msg), + warn: (msg: string) => console.warn(msg), + error: (msg: string) => console.error(msg), + }, registerService(svc: any) { services.push(svc); }, @@ -86,10 +89,25 @@ function createMockApi(pluginConfig: Partial = {}): MockApiHandle }; } -const EMPTY_RECALL: RecallResponse = { results: [], entities: null, trace: null, chunks: null }; -const OK_RETAIN: RetainResponse = { message: 'queued', document_id: 'test', memory_unit_ids: [] }; +const EMPTY_RECALL: RecallResponse = { results: [], entities: null, trace: null, chunks: null } as RecallResponse; +const OK_RETAIN = { operations: [], memory_units: [] } as unknown as RetainResponse; -function makeMemoryResult(text: string) { +interface MockMemoryResult { + id: string; + text: string; + type: string; + entities: string[]; + context: string; + occurred_start: string | null; + occurred_end: string | null; + mentioned_at: string | null; + document_id: string | null; + metadata: Record | null; + chunk_id: string | null; + tags: string[]; +} + +function makeMemoryResult(text: string): MockMemoryResult { return { id: `mem-${Math.random().toString(36).slice(2)}`, text, @@ -113,8 +131,10 @@ function makeMemoryResult(text: string) { let apiReachable = false; let triggerHook: MockApiHandle['trigger']; let stopServicesFn: () => Promise; -let recallSpy: ReturnType>; -let retainSpy: ReturnType>; +// Typed loosely as MockInstance because vi.spyOn's generic form doesn't +// play nicely with the hindsight-client class shape (method overloads). +let recallSpy: MockInstance; +let retainSpy: MockInstance; beforeAll(async () => { apiReachable = await waitForApi(HINDSIGHT_API_URL, 8000); @@ -135,7 +155,7 @@ beforeAll(async () => { process.env.HINDSIGHT_EMBED_API_URL = HINDSIGHT_API_URL; const mod = await import('../src/index.js'); - const { HindsightClient } = await import('../src/client.js'); + const { HindsightClient } = await import('@vectorize-io/hindsight-client'); const pluginFn = mod.default; const getClient = mod.getClient; @@ -146,12 +166,7 @@ beforeAll(async () => { recallContextTurns: 3, recallMaxQueryChars: 180, recallRoles: ['user'], - // Session pattern filtering — only affects keys matching these patterns - ignoreSessionPatterns: ['agent:main:**', 'agent:*:cron:**'], - statelessSessionPatterns: ['agent:*:subagent:**', 'agent:*:heartbeat:**'], - skipStatelessSessions: true, // No bankMission — keeps init lean - ...(HINDSIGHT_API_TOKEN ? { hindsightApiUrl: HINDSIGHT_API_URL, hindsightApiToken: HINDSIGHT_API_TOKEN } : {}), }); triggerHook = handle.trigger; stopServicesFn = handle.stopServices; @@ -165,9 +180,10 @@ beforeAll(async () => { // After startServices the client must be ready. if (!getClient()) throw new Error('[Hooks Integration] Client not initialized after service start'); - // Spy on the prototype so all per-bank instances created by getClientForContext are intercepted. - recallSpy = vi.spyOn(HindsightClient.prototype, 'recall') as ReturnType>; - retainSpy = vi.spyOn(HindsightClient.prototype, 'retain') as ReturnType>; + // Spy on the HindsightClient prototype so all calls go through the spy. + // The plugin's scopeClient() wrapper calls through these prototype methods. + recallSpy = vi.spyOn(HindsightClient.prototype, 'recall'); + retainSpy = vi.spyOn(HindsightClient.prototype, 'retain'); }, 30_000); afterAll(async () => { @@ -185,7 +201,7 @@ afterEach(() => { }); // --------------------------------------------------------------------------- -// before_agent_start +// before_prompt_build hook // --------------------------------------------------------------------------- describe('before_prompt_build hook', () => { @@ -236,7 +252,7 @@ describe('before_prompt_build hook', () => { entities: null, trace: null, chunks: null, - }); + } as RecallResponse); const result = (await triggerHook( 'before_prompt_build', @@ -261,7 +277,7 @@ describe('before_prompt_build hook', () => { entities: null, trace: null, chunks: null, - }); + } as RecallResponse); const result = (await triggerHook( 'before_prompt_build', @@ -288,11 +304,11 @@ describe('before_prompt_build hook', () => { ); expect(recallSpy).toHaveBeenCalledOnce(); - const [callArgs] = recallSpy.mock.calls[0]; - // The query passed to recall must NOT contain envelope artifacts - expect(callArgs.query).not.toContain('[Telegram'); - expect(callArgs.query).not.toContain('[from: Alice]'); - expect(callArgs.query).toContain('What is my favorite food?'); + // HindsightClient.recall signature: (bankId, query, options?) + const [, query] = recallSpy.mock.calls[0]; + expect(query).not.toContain('[Telegram'); + expect(query).not.toContain('[from: Alice]'); + expect(query).toContain('What is my favorite food?'); }); it('passes a latest-priority contextual recall query and respects max query chars', async () => { @@ -314,14 +330,14 @@ describe('before_prompt_build hook', () => { ); expect(recallSpy).toHaveBeenCalledOnce(); - const [callArgs] = recallSpy.mock.calls[0]; - expect(callArgs.query).toContain('Do I still prefer dark mode?'); - expect(callArgs.query).toContain('user: I prefer dark mode in IDEs.'); - expect(callArgs.query).not.toContain('assistant: Noted: dark mode preference.'); - expect(callArgs.query.length).toBeLessThanOrEqual(180); + const [, query] = recallSpy.mock.calls[0]; + expect(query).toContain('Do I still prefer dark mode?'); + expect(query).toContain('user: I prefer dark mode in IDEs.'); + expect(query).not.toContain('assistant: Noted: dark mode preference.'); + expect(query.length).toBeLessThanOrEqual(180); }); - it('passes max_tokens to recall', async () => { + it('passes maxTokens to recall', async () => { if (!apiReachable) return; recallSpy.mockResolvedValue(EMPTY_RECALL); @@ -332,8 +348,8 @@ describe('before_prompt_build hook', () => { ); expect(recallSpy).toHaveBeenCalledOnce(); - const [callArgs] = recallSpy.mock.calls[0]; - expect(callArgs.max_tokens).toBeGreaterThan(0); + const [, , options] = recallSpy.mock.calls[0]; + expect(options?.maxTokens).toBeGreaterThan(0); }); it('includes recalled memories in the prependSystemContext block', async () => { @@ -343,7 +359,7 @@ describe('before_prompt_build hook', () => { entities: null, trace: null, chunks: null, - }); + } as RecallResponse); const result = (await triggerHook( 'before_prompt_build', @@ -418,16 +434,17 @@ describe('agent_end hook', () => { ); expect(retainSpy).toHaveBeenCalledOnce(); - const [req] = retainSpy.mock.calls[0]; - expect(req.content).toContain('[role: user]'); - expect(req.content).toContain('I love TypeScript.'); - expect(req.content).toContain('[user:end]'); - expect(req.content).toContain('[role: assistant]'); - expect(req.content).toContain('TypeScript is great!'); - expect(req.content).toContain('[assistant:end]'); + // HindsightClient.retain signature: (bankId, content, options?) + const [, content] = retainSpy.mock.calls[0]; + expect(content).toContain('[role: user]'); + expect(content).toContain('I love TypeScript.'); + expect(content).toContain('[user:end]'); + expect(content).toContain('[role: assistant]'); + expect(content).toContain('TypeScript is great!'); + expect(content).toContain('[assistant:end]'); }); - it('includes session key in document_id', async () => { + it('includes session key in documentId', async () => { if (!apiReachable) return; retainSpy.mockResolvedValue(OK_RETAIN); @@ -441,8 +458,8 @@ describe('agent_end hook', () => { ); expect(retainSpy).toHaveBeenCalledOnce(); - const [req] = retainSpy.mock.calls[0]; - expect(req.document_id).toContain('sess-colour'); + const [, , options] = retainSpy.mock.calls[0]; + expect(options?.documentId).toContain('sess-colour'); }); it('populates metadata with channel_type, channel_id, and sender_id', async () => { @@ -464,12 +481,12 @@ describe('agent_end hook', () => { ); expect(retainSpy).toHaveBeenCalledOnce(); - const [req] = retainSpy.mock.calls[0]; - expect(req.metadata?.channel_type).toBe('telegram'); - expect(req.metadata?.channel_id).toBe('chat-999'); - expect(req.metadata?.sender_id).toBe('U015'); - expect(req.metadata?.retained_at).toBeDefined(); - expect(req.metadata?.message_count).toBe('1'); + const [, , options] = retainSpy.mock.calls[0]; + expect(options?.metadata?.channel_type).toBe('telegram'); + expect(options?.metadata?.channel_id).toBe('chat-999'); + expect(options?.metadata?.sender_id).toBe('U015'); + expect(options?.metadata?.retained_at).toBeDefined(); + expect(options?.metadata?.message_count).toBe('1'); }); it('strips tags from content before retaining', async () => { @@ -489,11 +506,11 @@ describe('agent_end hook', () => { ); expect(retainSpy).toHaveBeenCalledOnce(); - const [req] = retainSpy.mock.calls[0]; - expect(req.content).not.toContain(''); - expect(req.content).not.toContain(''); - expect(req.content).not.toContain('old fact'); - expect(req.content).toContain('I enjoy reading science fiction.'); + const [, content] = retainSpy.mock.calls[0]; + expect(content).not.toContain(''); + expect(content).not.toContain(''); + expect(content).not.toContain('old fact'); + expect(content).toContain('I enjoy reading science fiction.'); }); it('strips tags from content before retaining', async () => { @@ -513,9 +530,9 @@ describe('agent_end hook', () => { ); expect(retainSpy).toHaveBeenCalledOnce(); - const [req] = retainSpy.mock.calls[0]; - expect(req.content).not.toContain(''); - expect(req.content).toContain('I am learning Rust.'); + const [, content] = retainSpy.mock.calls[0]; + expect(content).not.toContain(''); + expect(content).toContain('I am learning Rust.'); }); it('handles array content blocks (structured message format)', async () => { @@ -540,10 +557,9 @@ describe('agent_end hook', () => { ); expect(retainSpy).toHaveBeenCalledOnce(); - const [req] = retainSpy.mock.calls[0]; - expect(req.content).toContain('I prefer dark mode in all my editors.'); - // Image block text should not appear - expect(req.content).not.toContain('data:'); + const [, content] = retainSpy.mock.calls[0]; + expect(content).toContain('I prefer dark mode in all my editors.'); + expect(content).not.toContain('data:'); }); it('retains a multi-turn conversation in the correct transcript format', async () => { @@ -565,91 +581,12 @@ describe('agent_end hook', () => { ); expect(retainSpy).toHaveBeenCalledOnce(); - const [req] = retainSpy.mock.calls[0]; + const [, content, options] = retainSpy.mock.calls[0]; // Only the last turn (from last user message onwards) is retained - expect(req.content).toContain('[role: user]\nI work as a data scientist.\n[user:end]'); - expect(req.content).toContain("[role: assistant]\nThat's a fascinating career!\n[assistant:end]"); - // Earlier turns are excluded by turn boundary detection - expect(req.content).not.toContain('My name is Carol.'); - expect(req.metadata?.message_count).toBe('2'); - }); -}); - -// --------------------------------------------------------------------------- -// session pattern filtering -// --------------------------------------------------------------------------- - -describe('session pattern filtering', () => { - it('skips retain when session key matches ignoreSessionPatterns', async () => { - if (!apiReachable) return; - - await triggerHook( - 'agent_end', - { - success: true, - messages: [{ role: 'user', content: 'I love TypeScript.' }], - }, - { messageProvider: 'telegram', senderId: 'U100', sessionKey: 'agent:mybot:cron:sess-cron-001' }, - ); - - expect(retainSpy).not.toHaveBeenCalled(); - }); - - it('skips retain when session key matches statelessSessionPatterns', async () => { - if (!apiReachable) return; - - await triggerHook( - 'agent_end', - { - success: true, - messages: [{ role: 'user', content: 'I prefer Python.' }], - }, - { messageProvider: 'telegram', senderId: 'U101', sessionKey: 'agent:mybot:subagent:sess-sub-001' }, - ); - - expect(retainSpy).not.toHaveBeenCalled(); - }); - - it('skips recall when session key matches ignoreSessionPatterns', async () => { - if (!apiReachable) return; - - const result = await triggerHook( - 'before_prompt_build', - { rawMessage: 'What programming language do I like?', prompt: '', messages: [] }, - { messageProvider: 'telegram', senderId: 'U102', sessionKey: 'agent:mybot:cron:sess-cron-002' }, - ); - - expect(recallSpy).not.toHaveBeenCalled(); - expect(result).toBeUndefined(); - }); - - it('skips recall for stateless session when skipStatelessSessions is true (default)', async () => { - if (!apiReachable) return; - - const result = await triggerHook( - 'before_prompt_build', - { rawMessage: 'What programming language do I like?', prompt: '', messages: [] }, - { messageProvider: 'telegram', senderId: 'U103', sessionKey: 'agent:mybot:heartbeat:sess-hb-001' }, - ); - - expect(recallSpy).not.toHaveBeenCalled(); - expect(result).toBeUndefined(); - }); - - it('does not skip a main session that matches no pattern', async () => { - if (!apiReachable) return; - retainSpy.mockResolvedValue(OK_RETAIN); - - await triggerHook( - 'agent_end', - { - success: true, - messages: [{ role: 'user', content: 'I enjoy hiking.' }], - }, - { messageProvider: 'telegram', senderId: 'U104', sessionKey: 'agent:mybot:main:sess-main-001' }, - ); - - expect(retainSpy).toHaveBeenCalledOnce(); + expect(content).toContain('[role: user]\nI work as a data scientist.\n[user:end]'); + expect(content).toContain("[role: assistant]\nThat's a fascinating career!\n[assistant:end]"); + expect(content).not.toContain('My name is Carol.'); + expect(options?.metadata?.message_count).toBe('2'); }); }); diff --git a/hindsight-integrations/openclaw/tests/integration.test.ts b/hindsight-integrations/openclaw/tests/integration.test.ts index 7bfa5ed1..f5dd1745 100644 --- a/hindsight-integrations/openclaw/tests/integration.test.ts +++ b/hindsight-integrations/openclaw/tests/integration.test.ts @@ -1,7 +1,9 @@ /** * Integration tests for the Hindsight OpenClaw integration. * - * Tests both HTTP mode (direct API calls) and Embed mode (subprocess/daemon). + * Exercises both HTTP mode (direct API calls) and Embed mode (local daemon + * spawned via HindsightServer), talking to Hindsight through + * `@vectorize-io/hindsight-client`. * * Requirements: * HTTP mode: Running Hindsight API at HINDSIGHT_API_URL (default: http://localhost:8888) @@ -15,8 +17,8 @@ import { describe, it, expect, beforeAll, afterAll } from 'vitest'; import { join, dirname } from 'path'; import { fileURLToPath } from 'url'; -import { HindsightClient } from '../src/client.js'; -import { HindsightEmbedManager } from '../src/embed-manager.js'; +import { HindsightServer } from '@vectorize-io/hindsight-all'; +import { HindsightClient } from '@vectorize-io/hindsight-client'; const __filename = fileURLToPath(import.meta.url); const __dirname = dirname(__filename); @@ -26,7 +28,6 @@ const __dirname = dirname(__filename); // --------------------------------------------------------------------------- const HINDSIGHT_API_URL = process.env.HINDSIGHT_API_URL || 'http://localhost:8888'; -const HINDSIGHT_API_TOKEN = process.env.HINDSIGHT_API_TOKEN || ''; const LLM_PROVIDER = process.env.HINDSIGHT_API_LLM_PROVIDER || ''; const LLM_API_KEY = process.env.HINDSIGHT_API_LLM_API_KEY || ''; const LLM_MODEL = process.env.HINDSIGHT_API_LLM_MODEL || ''; @@ -65,7 +66,7 @@ async function waitForApi(url: string, maxMs = 5000): Promise { // HTTP Mode Tests // --------------------------------------------------------------------------- -describe('HindsightClient – HTTP Mode', () => { +describe('openclaw integration — HTTP mode', () => { let client: HindsightClient; beforeAll(async () => { @@ -77,147 +78,104 @@ describe('HindsightClient – HTTP Mode', () => { ); } - client = new HindsightClient({ - llmProvider: LLM_PROVIDER || 'openai', - llmApiKey: LLM_API_KEY || 'test-key', - llmModel: LLM_MODEL || undefined, - apiUrl: HINDSIGHT_API_URL, - apiToken: HINDSIGHT_API_TOKEN || undefined, - }); + client = new HindsightClient({ baseUrl: HINDSIGHT_API_URL }); }); it('should retain a conversation', async () => { const bankId = randomBankId(); - client.setBankId(bankId); - const response = await client.retain({ - content: - '[role: user]\nMy name is Alice and I love hiking.\n[user:end]\n\n' + + const response = await client.retain( + bankId, + '[role: user]\nMy name is Alice and I love hiking.\n[user:end]\n\n' + '[role: assistant]\nNice to meet you, Alice!\n[assistant:end]', - document_id: 'http-retain-test-1', - metadata: { channel_type: 'slack', sender_id: 'U001' }, - }); + { + documentId: 'http-retain-test-1', + metadata: { channel_type: 'slack', sender_id: 'U001' }, + async: true, + }, + ); expect(response).toBeDefined(); - expect(response.message).toBeDefined(); - expect(response.document_id).toBe('http-retain-test-1'); }); it('should retain with auto-generated document id', async () => { const bankId = randomBankId(); - client.setBankId(bankId); - const response = await client.retain({ - content: '[role: user]\nI work at TechCorp as a software engineer.\n[user:end]', - }); + const response = await client.retain( + bankId, + '[role: user]\nI work at TechCorp as a software engineer.\n[user:end]', + { async: true }, + ); expect(response).toBeDefined(); - expect(response.document_id).toBe('conversation'); }); it('should recall from an empty bank without error', async () => { const bankId = randomBankId(); - client.setBankId(bankId); - - const response = await client.recall({ query: 'What do I like?', max_tokens: 512 }); - + const response = await client.recall(bankId, 'What do I like?', { maxTokens: 512 }); expect(response).toBeDefined(); expect(Array.isArray(response.results)).toBe(true); }); - it('should set bank mission without throwing', async () => { + it('should set bank mission via createBank after retain creates the bank', async () => { const bankId = randomBankId(); - client.setBankId(bankId); - - // setBankMission on a non-existent bank logs a warning but does not throw + await client.retain(bankId, '[role: user]\nHello\n[user:end]', { async: true }); await expect( - client.setBankMission('You are an assistant helping users via Slack.'), - ).resolves.not.toThrow(); - }); - - it('should set bank mission after retain creates the bank', async () => { - const bankId = randomBankId(); - client.setBankId(bankId); - - // Create the bank by retaining something first - await client.retain({ content: '[role: user]\nHello\n[user:end]' }); - - // Now set the mission – bank exists so this should succeed - await expect( - client.setBankMission('You are a helpful AI assistant.'), - ).resolves.not.toThrow(); + client.createBank(bankId, { reflectMission: 'You are a helpful AI assistant.' }), + ).resolves.toBeDefined(); }); it('should retain and then recall relevant memories', async () => { const bankId = randomBankId(); - client.setBankId(bankId); - await client.retain({ - content: - '[role: user]\nMy favorite programming language is Python.\n[user:end]\n\n' + + await client.retain( + bankId, + '[role: user]\nMy favorite programming language is Python.\n[user:end]\n\n' + '[role: assistant]\nPython is a great choice!\n[assistant:end]', - document_id: `session-${Date.now()}`, - }); + { documentId: `session-${Date.now()}`, async: true }, + ); - const response = await client.recall({ - query: 'What programming language do I like?', - max_tokens: 1024, + const response = await client.recall(bankId, 'What programming language do I like?', { + maxTokens: 1024, }); expect(response).toBeDefined(); expect(Array.isArray(response.results)).toBe(true); }); - it('should silently truncate recall queries over 800 chars', async () => { + it('should use custom maxTokens in recall request', async () => { const bankId = randomBankId(); - client.setBankId(bankId); - - const longQuery = 'Tell me about my interests. '.repeat(50); // > 800 chars - const response = await client.recall({ query: longQuery, max_tokens: 512 }); - + const response = await client.recall(bankId, 'anything', { maxTokens: 256 }); expect(response).toBeDefined(); expect(Array.isArray(response.results)).toBe(true); }); - it('should use custom max_tokens in recall request', async () => { + it('should map recall results to the RecallResult shape', async () => { const bankId = randomBankId(); - client.setBankId(bankId); - const response = await client.recall({ query: 'anything', max_tokens: 256 }); - - expect(response).toBeDefined(); - expect(Array.isArray(response.results)).toBe(true); - }); - - it('should map recall results to MemoryResult shape', async () => { - const bankId = randomBankId(); - client.setBankId(bankId); - - await client.retain({ - content: - '[role: user]\nI enjoy reading science fiction books.\n[user:end]\n\n' + + await client.retain( + bankId, + '[role: user]\nI enjoy reading science fiction books.\n[user:end]\n\n' + '[role: assistant]\nSounds like a great hobby!\n[assistant:end]', - document_id: 'mapping-test', - }); + { documentId: 'mapping-test', async: true }, + ); - const response = await client.recall({ query: 'What are my hobbies?', max_tokens: 1024 }); + const response = await client.recall(bankId, 'What are my hobbies?', { maxTokens: 1024 }); for (const result of response.results) { expect(typeof result.id).toBe('string'); expect(typeof result.text).toBe('string'); - expect(typeof result.type).toBe('string'); - expect(Array.isArray(result.entities)).toBe(true); } }); }); // --------------------------------------------------------------------------- -// Embed Mode Tests (subprocess / daemon) +// Embed Mode Tests (local daemon spawned by HindsightServer) // --------------------------------------------------------------------------- -describe('HindsightClient – Embed Mode (Subprocess)', () => { +describe('openclaw integration — embed mode', () => { let client: HindsightClient; - let embedManager: HindsightEmbedManager; + let server: HindsightServer; const hasEmbedCredentials = Boolean(LLM_PROVIDER && LLM_API_KEY); @@ -230,158 +188,71 @@ describe('HindsightClient – Embed Mode (Subprocess)', () => { return; } - embedManager = new HindsightEmbedManager( - EMBED_TEST_PORT, - LLM_PROVIDER, - LLM_API_KEY, - LLM_MODEL || undefined, - undefined, // no custom base URL - 0, // never idle-timeout - 'latest', - EMBED_PACKAGE_PATH, - ); - - await embedManager.start(); - - client = new HindsightClient({ - llmProvider: LLM_PROVIDER, - llmApiKey: LLM_API_KEY, - llmModel: LLM_MODEL || undefined, + server = new HindsightServer({ + profile: 'openclaw-test', + port: EMBED_TEST_PORT, + embedVersion: 'latest', embedPackagePath: EMBED_PACKAGE_PATH, + env: { + HINDSIGHT_API_LLM_PROVIDER: LLM_PROVIDER, + HINDSIGHT_API_LLM_API_KEY: LLM_API_KEY, + HINDSIGHT_API_LLM_MODEL: LLM_MODEL || undefined, + HINDSIGHT_EMBED_DAEMON_IDLE_TIMEOUT: '0', + }, }); + + await server.start(); + + client = new HindsightClient({ baseUrl: server.getBaseUrl() }); }, 120_000); // daemon startup can take up to 2 minutes afterAll(async () => { - if (embedManager) { - await embedManager.stop(); + if (server) { + await server.stop(); } }, 30_000); - it('should retain a conversation via subprocess', async () => { + it('should retain a conversation against the local daemon', async () => { if (!hasEmbedCredentials) return; - const bankId = randomBankId(); - client.setBankId(bankId); - - const response = await client.retain({ - content: - '[role: user]\nI love hiking in the mountains.\n[user:end]\n\n' + + const response = await client.retain( + bankId, + '[role: user]\nI love hiking in the mountains.\n[user:end]\n\n' + '[role: assistant]\nSounds adventurous!\n[assistant:end]', - document_id: 'embed-retain-test-1', - }); - + { documentId: 'embed-retain-test-1', async: true }, + ); expect(response).toBeDefined(); - expect(response.message).toBeDefined(); - expect(response.document_id).toBe('embed-retain-test-1'); }, 60_000); - it('should retain with auto-generated document id via subprocess', async () => { + it('should recall from an empty bank against the local daemon', async () => { if (!hasEmbedCredentials) return; - const bankId = randomBankId(); - client.setBankId(bankId); - - const response = await client.retain({ - content: '[role: user]\nI am a TypeScript developer.\n[user:end]', - }); - - expect(response).toBeDefined(); - expect(response.document_id).toBe('conversation'); - }, 60_000); - - it('should recall from an empty bank without error via subprocess', async () => { - if (!hasEmbedCredentials) return; - - const bankId = randomBankId(); - client.setBankId(bankId); - - const response = await client.recall({ query: 'What do I like?', max_tokens: 512 }); - + const response = await client.recall(bankId, 'What do I like?', { maxTokens: 512 }); expect(response).toBeDefined(); expect(Array.isArray(response.results)).toBe(true); }, 60_000); - it('should set bank mission via subprocess without throwing', async () => { + it('should set bank mission against the local daemon', async () => { if (!hasEmbedCredentials) return; - const bankId = randomBankId(); - client.setBankId(bankId); - // Create bank by retaining first, then set mission - await client.retain({ content: '[role: user]\nHello\n[user:end]' }); - + await client.retain(bankId, '[role: user]\nHello\n[user:end]', { async: true }); await expect( - client.setBankMission('Test mission for embed integration tests.'), - ).resolves.not.toThrow(); + client.createBank(bankId, { reflectMission: 'Test mission for embed integration tests.' }), + ).resolves.toBeDefined(); }, 60_000); - it('should retain and then recall relevant memories via subprocess', async () => { + it('should retain and recall against the local daemon', async () => { if (!hasEmbedCredentials) return; - const bankId = randomBankId(); - client.setBankId(bankId); - - await client.retain({ - content: - '[role: user]\nMy cat is named Whiskers and she is 3 years old.\n[user:end]\n\n' + + await client.retain( + bankId, + '[role: user]\nMy cat is named Whiskers and she is 3 years old.\n[user:end]\n\n' + '[role: assistant]\nWhat a lovely name!\n[assistant:end]', - document_id: `embed-e2e-${Date.now()}`, - }); - - const response = await client.recall({ - query: "What is my cat's name?", - max_tokens: 1024, - }); - + { documentId: `embed-e2e-${Date.now()}`, async: true }, + ); + const response = await client.recall(bankId, "What is my cat's name?", { maxTokens: 1024 }); expect(response).toBeDefined(); expect(Array.isArray(response.results)).toBe(true); }, 60_000); - - it('should map recall results to MemoryResult shape via subprocess', async () => { - if (!hasEmbedCredentials) return; - - const bankId = randomBankId(); - client.setBankId(bankId); - - await client.retain({ - content: - '[role: user]\nI enjoy cooking Italian food.\n[user:end]\n\n' + - '[role: assistant]\nItalian cuisine is delicious!\n[assistant:end]', - document_id: 'embed-shape-test', - }); - - const response = await client.recall({ query: 'What food do I like?', max_tokens: 1024 }); - - for (const result of response.results) { - expect(typeof result.id).toBe('string'); - expect(typeof result.text).toBe('string'); - expect(typeof result.type).toBe('string'); - expect(Array.isArray(result.entities)).toBe(true); - } - }, 60_000); - - it('should handle full end-to-end workflow via subprocess', async () => { - if (!hasEmbedCredentials) return; - - const bankId = randomBankId(); - client.setBankId(bankId); - - // Step 1: Retain - const retainResp = await client.retain({ - content: - '[role: user]\nI am learning Rust programming.\n[user:end]\n\n' + - '[role: assistant]\nRust is a powerful systems language!\n[assistant:end]', - document_id: `embed-workflow-${Date.now()}`, - metadata: { channel_type: 'telegram', sender_id: '999' }, - }); - expect(retainResp).toBeDefined(); - - // Step 2: Recall - const recallResp = await client.recall({ - query: 'What am I learning?', - max_tokens: 1024, - }); - expect(recallResp).toBeDefined(); - expect(Array.isArray(recallResp.results)).toBe(true); - }, 60_000); }); diff --git a/package-lock.json b/package-lock.json index 15ae9d2c..e7885d12 100644 --- a/package-lock.json +++ b/package-lock.json @@ -8,12 +8,137 @@ "workspaces": [ "hindsight-clients/typescript", "hindsight-control-plane", - "hindsight-docs" + "hindsight-docs", + "hindsight-all-npm" ] }, + "hindsight-all-npm": { + "name": "@vectorize-io/hindsight-all", + "version": "0.5.0", + "license": "MIT", + "devDependencies": { + "@types/node": "^22.0.0", + "tsup": "^8.5.1", + "typescript": "^5.7.0", + "vitest": "^4.1.2" + }, + "engines": { + "node": ">=22" + } + }, + "hindsight-all-npm/node_modules/@types/node": { + "version": "22.19.17", + "resolved": "https://registry.npmjs.org/@types/node/-/node-22.19.17.tgz", + "integrity": "sha512-wGdMcf+vPYM6jikpS/qhg6WiqSV/OhG+jeeHT/KlVqxYfD40iYJf9/AE1uQxVWFvU7MipKRkRv8NSHiCGgPr8Q==", + "dev": true, + "license": "MIT", + "dependencies": { + "undici-types": "~6.21.0" + } + }, + "hindsight-all-npm/node_modules/chokidar": { + "version": "4.0.3", + "resolved": "https://registry.npmjs.org/chokidar/-/chokidar-4.0.3.tgz", + "integrity": "sha512-Qgzu8kfBvo+cA4962jnP1KkS6Dop5NS6g7R5LFYJr4b8Ub94PPQXUksCw9PvXoeXPRRddRNC5C1JQUR2SMGtnA==", + "dev": true, + "license": "MIT", + "dependencies": { + "readdirp": "^4.0.1" + }, + "engines": { + "node": ">= 14.16.0" + }, + "funding": { + "url": "https://paulmillr.com/funding/" + } + }, + "hindsight-all-npm/node_modules/readdirp": { + "version": "4.1.2", + "resolved": "https://registry.npmjs.org/readdirp/-/readdirp-4.1.2.tgz", + "integrity": "sha512-GDhwkLfywWL2s6vEjyhri+eXmfH6j1L7JE27WhqLeYzoh/A3DBaYGEj2H/HFZCn/kMfim73FXxEJTw06WtxQwg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 14.18.0" + }, + "funding": { + "type": "individual", + "url": "https://paulmillr.com/funding/" + } + }, + "hindsight-all-npm/node_modules/resolve-from": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/resolve-from/-/resolve-from-5.0.0.tgz", + "integrity": "sha512-qYg9KP24dD5qka9J47d0aVky0N+b4fTU89LN9iDnjB5waksiC49rvMB0PrUJQGoTmH50XPiqOvAjDfaijGxYZw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "hindsight-all-npm/node_modules/tinyexec": { + "version": "0.3.2", + "resolved": "https://registry.npmjs.org/tinyexec/-/tinyexec-0.3.2.tgz", + "integrity": "sha512-KQQR9yN7R5+OSwaK0XQoj22pwHoTlgYqmUscPYoknOoWCWfj/5/ABTMRi69FrKU5ffPVh5QcFikpWJI/P1ocHA==", + "dev": true, + "license": "MIT" + }, + "hindsight-all-npm/node_modules/tsup": { + "version": "8.5.1", + "resolved": "https://registry.npmjs.org/tsup/-/tsup-8.5.1.tgz", + "integrity": "sha512-xtgkqwdhpKWr3tKPmCkvYmS9xnQK3m3XgxZHwSUjvfTjp7YfXe5tT3GgWi0F2N+ZSMsOeWeZFh7ZZFg5iPhing==", + "dev": true, + "license": "MIT", + "dependencies": { + "bundle-require": "^5.1.0", + "cac": "^6.7.14", + "chokidar": "^4.0.3", + "consola": "^3.4.0", + "debug": "^4.4.0", + "esbuild": "^0.27.0", + "fix-dts-default-cjs-exports": "^1.0.0", + "joycon": "^3.1.1", + "picocolors": "^1.1.1", + "postcss-load-config": "^6.0.1", + "resolve-from": "^5.0.0", + "rollup": "^4.34.8", + "source-map": "^0.7.6", + "sucrase": "^3.35.0", + "tinyexec": "^0.3.2", + "tinyglobby": "^0.2.11", + "tree-kill": "^1.2.2" + }, + "bin": { + "tsup": "dist/cli-default.js", + "tsup-node": "dist/cli-node.js" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "@microsoft/api-extractor": "^7.36.0", + "@swc/core": "^1", + "postcss": "^8.4.12", + "typescript": ">=4.5.0" + }, + "peerDependenciesMeta": { + "@microsoft/api-extractor": { + "optional": true + }, + "@swc/core": { + "optional": true + }, + "postcss": { + "optional": true + }, + "typescript": { + "optional": true + } + } + }, "hindsight-clients/typescript": { "name": "@vectorize-io/hindsight-client", - "version": "0.4.22", + "version": "0.5.0", "license": "MIT", "devDependencies": { "@hey-api/openapi-ts": "0.88.0", @@ -133,49 +258,6 @@ "url": "https://github.com/sponsors/sindresorhus" } }, - "hindsight-clients/typescript/node_modules/postcss-load-config": { - "version": "6.0.1", - "resolved": "https://registry.npmjs.org/postcss-load-config/-/postcss-load-config-6.0.1.tgz", - "integrity": "sha512-oPtTM4oerL+UXmx+93ytZVN82RrlY/wPUV8IeDxFrzIjXOLF1pN+EmKPLbubvKHT2HC20xXsCAH2Z+CKV6Oz/g==", - "dev": true, - "funding": [ - { - "type": "opencollective", - "url": "https://opencollective.com/postcss/" - }, - { - "type": "github", - "url": "https://github.com/sponsors/ai" - } - ], - "license": "MIT", - "dependencies": { - "lilconfig": "^3.1.1" - }, - "engines": { - "node": ">= 18" - }, - "peerDependencies": { - "jiti": ">=1.21.0", - "postcss": ">=8.0.9", - "tsx": "^4.8.1", - "yaml": "^2.4.2" - }, - "peerDependenciesMeta": { - "jiti": { - "optional": true - }, - "postcss": { - "optional": true - }, - "tsx": { - "optional": true - }, - "yaml": { - "optional": true - } - } - }, "hindsight-clients/typescript/node_modules/readdirp": { "version": "4.1.2", "resolved": "https://registry.npmjs.org/readdirp/-/readdirp-4.1.2.tgz", @@ -275,7 +357,7 @@ }, "hindsight-control-plane": { "name": "@vectorize-io/hindsight-control-plane", - "version": "0.4.22", + "version": "0.5.0", "license": "ISC", "dependencies": { "@chenglou/pretext": "^0.0.3", @@ -5120,20 +5202,20 @@ } }, "node_modules/@emnapi/core": { - "version": "1.8.1", - "resolved": "https://registry.npmjs.org/@emnapi/core/-/core-1.8.1.tgz", - "integrity": "sha512-AvT9QFpxK0Zd8J0jopedNm+w/2fIzvtPKPjqyw9jwvBaReTTqPBk9Hixaz7KbjimP+QNz605/XnjFcDAL2pqBg==", + "version": "1.9.2", + "resolved": "https://registry.npmjs.org/@emnapi/core/-/core-1.9.2.tgz", + "integrity": "sha512-UC+ZhH3XtczQYfOlu3lNEkdW/p4dsJ1r/bP7H8+rhao3TTTMO1ATq/4DdIi23XuGoFY+Cz0JmCbdVl0hz9jZcA==", "license": "MIT", "optional": true, "dependencies": { - "@emnapi/wasi-threads": "1.1.0", + "@emnapi/wasi-threads": "1.2.1", "tslib": "^2.4.0" } }, "node_modules/@emnapi/runtime": { - "version": "1.8.1", - "resolved": "https://registry.npmjs.org/@emnapi/runtime/-/runtime-1.8.1.tgz", - "integrity": "sha512-mehfKSMWjjNol8659Z8KxEMrdSJDDot5SXMq00dM8BN4o+CLNXQ0xH2V7EchNHV4RmbZLmmPdEaXZc5H2FXmDg==", + "version": "1.9.2", + "resolved": "https://registry.npmjs.org/@emnapi/runtime/-/runtime-1.9.2.tgz", + "integrity": "sha512-3U4+MIWHImeyu1wnmVygh5WlgfYDtyf0k8AbLhMFxOipihf6nrWC4syIm/SwEeec0mNSafiiNnMJwbza/Is6Lw==", "license": "MIT", "optional": true, "dependencies": { @@ -5141,9 +5223,9 @@ } }, "node_modules/@emnapi/wasi-threads": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/@emnapi/wasi-threads/-/wasi-threads-1.1.0.tgz", - "integrity": "sha512-WI0DdZ8xFSbgMjR1sFsKABJ/C5OnRrjT06JXbZKexJGrDuPTzZdDYfFlsgcCXCyf+suG5QU2e/y1Wo2V/OapLQ==", + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/@emnapi/wasi-threads/-/wasi-threads-1.2.1.tgz", + "integrity": "sha512-uTII7OYF+/Mes/MrcIOYp5yOtSMLBWSIoLPpcgwipoiKbli6k322tcoFsxoIIxPDqW01SQGAgko4EzZi2BNv2w==", "license": "MIT", "optional": true, "dependencies": { @@ -7358,6 +7440,16 @@ "node": ">=8.0.0" } }, + "node_modules/@oxc-project/types": { + "version": "0.124.0", + "resolved": "https://registry.npmjs.org/@oxc-project/types/-/types-0.124.0.tgz", + "integrity": "sha512-VBFWMTBvHxS11Z5Lvlr3IWgrwhMTXV+Md+EQF0Xf60+wAdsGFTBx7X7K/hP4pi8N7dcm1RvcHwDxZ16Qx8keUg==", + "dev": true, + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/Boshen" + } + }, "node_modules/@pnpm/config.env-replace": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/@pnpm/config.env-replace/-/config.env-replace-1.1.0.tgz", @@ -8603,6 +8695,289 @@ "url": "https://opencollective.com/immer" } }, + "node_modules/@rolldown/binding-android-arm64": { + "version": "1.0.0-rc.15", + "resolved": "https://registry.npmjs.org/@rolldown/binding-android-arm64/-/binding-android-arm64-1.0.0-rc.15.tgz", + "integrity": "sha512-YYe6aWruPZDtHNpwu7+qAHEMbQ/yRl6atqb/AhznLTnD3UY99Q1jE7ihLSahNWkF4EqRPVC4SiR4O0UkLK02tA==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-darwin-arm64": { + "version": "1.0.0-rc.15", + "resolved": "https://registry.npmjs.org/@rolldown/binding-darwin-arm64/-/binding-darwin-arm64-1.0.0-rc.15.tgz", + "integrity": "sha512-oArR/ig8wNTPYsXL+Mzhs0oxhxfuHRfG7Ikw7jXsw8mYOtk71W0OkF2VEVh699pdmzjPQsTjlD1JIOoHkLP1Fg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-darwin-x64": { + "version": "1.0.0-rc.15", + "resolved": "https://registry.npmjs.org/@rolldown/binding-darwin-x64/-/binding-darwin-x64-1.0.0-rc.15.tgz", + "integrity": "sha512-YzeVqOqjPYvUbJSWJ4EDL8ahbmsIXQpgL3JVipmN+MX0XnXMeWomLN3Fb+nwCmP/jfyqte5I3XRSm7OfQrbyxw==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-freebsd-x64": { + "version": "1.0.0-rc.15", + "resolved": "https://registry.npmjs.org/@rolldown/binding-freebsd-x64/-/binding-freebsd-x64-1.0.0-rc.15.tgz", + "integrity": "sha512-9Erhx956jeQ0nNTyif1+QWAXDRD38ZNjr//bSHrt6wDwB+QkAfl2q6Mn1k6OBPerznjRmbM10lgRb1Pli4xZPw==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-linux-arm-gnueabihf": { + "version": "1.0.0-rc.15", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-arm-gnueabihf/-/binding-linux-arm-gnueabihf-1.0.0-rc.15.tgz", + "integrity": "sha512-cVwk0w8QbZJGTnP/AHQBs5yNwmpgGYStL88t4UIaqcvYJWBfS0s3oqVLZPwsPU6M0zlW4GqjP0Zq5MnAGwFeGA==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-linux-arm64-gnu": { + "version": "1.0.0-rc.15", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-arm64-gnu/-/binding-linux-arm64-gnu-1.0.0-rc.15.tgz", + "integrity": "sha512-eBZ/u8iAK9SoHGanqe/jrPnY0JvBN6iXbVOsbO38mbz+ZJsaobExAm1Iu+rxa4S1l2FjG0qEZn4Rc6X8n+9M+w==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-linux-arm64-musl": { + "version": "1.0.0-rc.15", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-arm64-musl/-/binding-linux-arm64-musl-1.0.0-rc.15.tgz", + "integrity": "sha512-ZvRYMGrAklV9PEkgt4LQM6MjQX2P58HPAuecwYObY2DhS2t35R0I810bKi0wmaYORt6m/2Sm+Z+nFgb0WhXNcQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-linux-ppc64-gnu": { + "version": "1.0.0-rc.15", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-ppc64-gnu/-/binding-linux-ppc64-gnu-1.0.0-rc.15.tgz", + "integrity": "sha512-VDpgGBzgfg5hLg+uBpCLoFG5kVvEyafmfxGUV0UHLcL5irxAK7PKNeC2MwClgk6ZAiNhmo9FLhRYgvMmedLtnQ==", + "cpu": [ + "ppc64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-linux-s390x-gnu": { + "version": "1.0.0-rc.15", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-s390x-gnu/-/binding-linux-s390x-gnu-1.0.0-rc.15.tgz", + "integrity": "sha512-y1uXY3qQWCzcPgRJATPSOUP4tCemh4uBdY7e3EZbVwCJTY3gLJWnQABgeUetvED+bt1FQ01OeZwvhLS2bpNrAQ==", + "cpu": [ + "s390x" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-linux-x64-gnu": { + "version": "1.0.0-rc.15", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-x64-gnu/-/binding-linux-x64-gnu-1.0.0-rc.15.tgz", + "integrity": "sha512-023bTPBod7J3Y/4fzAN6QtpkSABR0rigtrwaP+qSEabUh5zf6ELr9Nc7GujaROuPY3uwdSIXWrvhn1KxOvurWA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-linux-x64-musl": { + "version": "1.0.0-rc.15", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-x64-musl/-/binding-linux-x64-musl-1.0.0-rc.15.tgz", + "integrity": "sha512-witB2O0/hU4CgfOOKUoeFgQ4GktPi1eEbAhaLAIpgD6+ZnhcPkUtPsoKKHRzmOoWPZue46IThdSgdo4XneOLYw==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-openharmony-arm64": { + "version": "1.0.0-rc.15", + "resolved": "https://registry.npmjs.org/@rolldown/binding-openharmony-arm64/-/binding-openharmony-arm64-1.0.0-rc.15.tgz", + "integrity": "sha512-UCL68NJ0Ud5zRipXZE9dF5PmirzJE4E4BCIOOssEnM7wLDsxjc6Qb0sGDxTNRTP53I6MZpygyCpY8Aa8sPfKPg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openharmony" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-wasm32-wasi": { + "version": "1.0.0-rc.15", + "resolved": "https://registry.npmjs.org/@rolldown/binding-wasm32-wasi/-/binding-wasm32-wasi-1.0.0-rc.15.tgz", + "integrity": "sha512-ApLruZq/ig+nhaE7OJm4lDjayUnOHVUa77zGeqnqZ9pn0ovdVbbNPerVibLXDmWeUZXjIYIT8V3xkT58Rm9u5Q==", + "cpu": [ + "wasm32" + ], + "dev": true, + "license": "MIT", + "optional": true, + "dependencies": { + "@emnapi/core": "1.9.2", + "@emnapi/runtime": "1.9.2", + "@napi-rs/wasm-runtime": "^1.1.3" + }, + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/@rolldown/binding-wasm32-wasi/node_modules/@napi-rs/wasm-runtime": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/@napi-rs/wasm-runtime/-/wasm-runtime-1.1.3.tgz", + "integrity": "sha512-xK9sGVbJWYb08+mTJt3/YV24WxvxpXcXtP6B172paPZ+Ts69Re9dAr7lKwJoeIx8OoeuimEiRZ7umkiUVClmmQ==", + "dev": true, + "license": "MIT", + "optional": true, + "dependencies": { + "@tybys/wasm-util": "^0.10.1" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/Brooooooklyn" + }, + "peerDependencies": { + "@emnapi/core": "^1.7.1", + "@emnapi/runtime": "^1.7.1" + } + }, + "node_modules/@rolldown/binding-win32-arm64-msvc": { + "version": "1.0.0-rc.15", + "resolved": "https://registry.npmjs.org/@rolldown/binding-win32-arm64-msvc/-/binding-win32-arm64-msvc-1.0.0-rc.15.tgz", + "integrity": "sha512-KmoUoU7HnN+Si5YWJigfTws1jz1bKBYDQKdbLspz0UaqjjFkddHsqorgiW1mxcAj88lYUE6NC/zJNwT+SloqtA==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-win32-x64-msvc": { + "version": "1.0.0-rc.15", + "resolved": "https://registry.npmjs.org/@rolldown/binding-win32-x64-msvc/-/binding-win32-x64-msvc-1.0.0-rc.15.tgz", + "integrity": "sha512-3P2A8L+x75qavWLe/Dll3EYBJLQmtkJN8rfh+U/eR3MqMgL/h98PhYI+JFfXuDPgPeCB7iZAKiqii5vqOvnA0g==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/pluginutils": { + "version": "1.0.0-rc.15", + "resolved": "https://registry.npmjs.org/@rolldown/pluginutils/-/pluginutils-1.0.0-rc.15.tgz", + "integrity": "sha512-UromN0peaE53IaBRe9W7CjrZgXl90fqGpK+mIZbA3qSTeYqg3pqpROBdIPvOG3F5ereDHNwoHBI2e50n1BDr1g==", + "dev": true, + "license": "MIT" + }, "node_modules/@rollup/rollup-android-arm-eabi": { "version": "4.59.0", "resolved": "https://registry.npmjs.org/@rollup/rollup-android-arm-eabi/-/rollup-android-arm-eabi-4.59.0.tgz", @@ -9683,6 +10058,17 @@ "@types/node": "*" } }, + "node_modules/@types/chai": { + "version": "5.2.3", + "resolved": "https://registry.npmjs.org/@types/chai/-/chai-5.2.3.tgz", + "integrity": "sha512-Mw558oeA9fFbv65/y4mHtXDs9bPnFMZAL/jxdPFUpOHHIXX91mcgEHbS5Lahr+pwZFR8A7GQleRWeI6cGFC2UA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/deep-eql": "*", + "assertion-error": "^2.0.1" + } + }, "node_modules/@types/connect": { "version": "3.4.38", "resolved": "https://registry.npmjs.org/@types/connect/-/connect-3.4.38.tgz", @@ -9970,6 +10356,13 @@ "@types/ms": "*" } }, + "node_modules/@types/deep-eql": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/@types/deep-eql/-/deep-eql-4.0.2.tgz", + "integrity": "sha512-c9h9dVVMigMPc4bwTvC5dxqtqJZwQPePsWjPlpSOnojbor6pGqdk541lfA7AqFQr5pB1BRdq0juY9db81BwyFw==", + "dev": true, + "license": "MIT" + }, "node_modules/@types/eslint": { "version": "9.6.1", "resolved": "https://registry.npmjs.org/@types/eslint/-/eslint-9.6.1.tgz", @@ -10872,6 +11265,10 @@ "d3-transition": "^3.0.1" } }, + "node_modules/@vectorize-io/hindsight-all": { + "resolved": "hindsight-all-npm", + "link": true + }, "node_modules/@vectorize-io/hindsight-client": { "resolved": "hindsight-clients/typescript", "link": true @@ -10889,6 +11286,119 @@ "node": ">= 20" } }, + "node_modules/@vitest/expect": { + "version": "4.1.4", + "resolved": "https://registry.npmjs.org/@vitest/expect/-/expect-4.1.4.tgz", + "integrity": "sha512-iPBpra+VDuXmBFI3FMKHSFXp3Gx5HfmSCE8X67Dn+bwephCnQCaB7qWK2ldHa+8ncN8hJU8VTMcxjPpyMkUjww==", + "dev": true, + "license": "MIT", + "dependencies": { + "@standard-schema/spec": "^1.1.0", + "@types/chai": "^5.2.2", + "@vitest/spy": "4.1.4", + "@vitest/utils": "4.1.4", + "chai": "^6.2.2", + "tinyrainbow": "^3.1.0" + }, + "funding": { + "url": "https://opencollective.com/vitest" + } + }, + "node_modules/@vitest/mocker": { + "version": "4.1.4", + "resolved": "https://registry.npmjs.org/@vitest/mocker/-/mocker-4.1.4.tgz", + "integrity": "sha512-R9HTZBhW6yCSGbGQnDnH3QHfJxokKN4KB+Yvk9Q1le7eQNYwiCyKxmLmurSpFy6BzJanSLuEUDrD+j97Q+ZLPg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@vitest/spy": "4.1.4", + "estree-walker": "^3.0.3", + "magic-string": "^0.30.21" + }, + "funding": { + "url": "https://opencollective.com/vitest" + }, + "peerDependencies": { + "msw": "^2.4.9", + "vite": "^6.0.0 || ^7.0.0 || ^8.0.0" + }, + "peerDependenciesMeta": { + "msw": { + "optional": true + }, + "vite": { + "optional": true + } + } + }, + "node_modules/@vitest/pretty-format": { + "version": "4.1.4", + "resolved": "https://registry.npmjs.org/@vitest/pretty-format/-/pretty-format-4.1.4.tgz", + "integrity": "sha512-ddmDHU0gjEUyEVLxtZa7xamrpIefdEETu3nZjWtHeZX4QxqJ7tRxSteHVXJOcr8jhiLoGAhkK4WJ3WqBpjx42A==", + "dev": true, + "license": "MIT", + "dependencies": { + "tinyrainbow": "^3.1.0" + }, + "funding": { + "url": "https://opencollective.com/vitest" + } + }, + "node_modules/@vitest/runner": { + "version": "4.1.4", + "resolved": "https://registry.npmjs.org/@vitest/runner/-/runner-4.1.4.tgz", + "integrity": "sha512-xTp7VZ5aXP5ZJrn15UtJUWlx6qXLnGtF6jNxHepdPHpMfz/aVPx+htHtgcAL2mDXJgKhpoo2e9/hVJsIeFbytQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@vitest/utils": "4.1.4", + "pathe": "^2.0.3" + }, + "funding": { + "url": "https://opencollective.com/vitest" + } + }, + "node_modules/@vitest/snapshot": { + "version": "4.1.4", + "resolved": "https://registry.npmjs.org/@vitest/snapshot/-/snapshot-4.1.4.tgz", + "integrity": "sha512-MCjCFgaS8aZz+m5nTcEcgk/xhWv0rEH4Yl53PPlMXOZ1/Ka2VcZU6CJ+MgYCZbcJvzGhQRjVrGQNZqkGPttIKw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@vitest/pretty-format": "4.1.4", + "@vitest/utils": "4.1.4", + "magic-string": "^0.30.21", + "pathe": "^2.0.3" + }, + "funding": { + "url": "https://opencollective.com/vitest" + } + }, + "node_modules/@vitest/spy": { + "version": "4.1.4", + "resolved": "https://registry.npmjs.org/@vitest/spy/-/spy-4.1.4.tgz", + "integrity": "sha512-XxNdAsKW7C+FLydqFJLb5KhJtl3PGCMmYwFRfhvIgxJvLSXhhVI1zM8f1qD3Zg7RCjTSzDVyct6sghs9UEgBEQ==", + "dev": true, + "license": "MIT", + "funding": { + "url": "https://opencollective.com/vitest" + } + }, + "node_modules/@vitest/utils": { + "version": "4.1.4", + "resolved": "https://registry.npmjs.org/@vitest/utils/-/utils-4.1.4.tgz", + "integrity": "sha512-13QMT+eysM5uVGa1rG4kegGYNp6cnQcsTc67ELFbhNLQO+vgsygtYJx2khvdt4gVQqSSpC/KT5FZZxUpP3Oatw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@vitest/pretty-format": "4.1.4", + "convert-source-map": "^2.0.0", + "tinyrainbow": "^3.1.0" + }, + "funding": { + "url": "https://opencollective.com/vitest" + } + }, "node_modules/@webassemblyjs/ast": { "version": "1.14.1", "resolved": "https://registry.npmjs.org/@webassemblyjs/ast/-/ast-1.14.1.tgz", @@ -11607,6 +12117,16 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/assertion-error": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/assertion-error/-/assertion-error-2.0.1.tgz", + "integrity": "sha512-Izi8RQcffqCeNVgFigKli1ssklIbpHnCYc6AknXGYoB6grJqyeby7jv12JUQgmTAnIDnbck1uxksT4dzN3PWBA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=12" + } + }, "node_modules/ast-types-flow": { "version": "0.0.8", "resolved": "https://registry.npmjs.org/ast-types-flow/-/ast-types-flow-0.0.8.tgz", @@ -12417,6 +12937,16 @@ "url": "https://github.com/sponsors/wooorm" } }, + "node_modules/chai": { + "version": "6.2.2", + "resolved": "https://registry.npmjs.org/chai/-/chai-6.2.2.tgz", + "integrity": "sha512-NUPRluOfOiTKBKvWPtSD4PhFvWCqOi0BGStNWs57X9js7XGTprSmFoz5F0tWhR4WPjNeR9jXqdC7/UpSJTnlRg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=18" + } + }, "node_modules/chalk": { "version": "4.1.2", "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz", @@ -15930,6 +16460,16 @@ "node": "^14.15.0 || ^16.10.0 || >=18.0.0" } }, + "node_modules/expect-type": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/expect-type/-/expect-type-1.3.0.tgz", + "integrity": "sha512-knvyeauYhqjOYvQ66MznSMs83wmHrCycNEN6Ao+2AeYEfxUIkuiVxdEa1qlGEPK+We3n0THiDciYSsCcgW/DoA==", + "dev": true, + "license": "Apache-2.0", + "engines": { + "node": ">=12.0.0" + } + }, "node_modules/express": { "version": "4.22.1", "resolved": "https://registry.npmjs.org/express/-/express-4.22.1.tgz", @@ -23063,6 +23603,17 @@ "integrity": "sha512-PX1wu0AmAdPqOL1mWhqmlOd8kOIZQwGZw6rh7uby9fTc5lhaOWFLX3I6R1hrF9k3zUY40e6igsLGkDXK92LJNg==", "license": "MIT" }, + "node_modules/obug": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/obug/-/obug-2.1.1.tgz", + "integrity": "sha512-uTqF9MuPraAQ+IsnPf366RG4cP9RtUi7MLO1N3KEc+wb0a6yKpeL0lmk2IB1jY5KHPAlTc6T/JRdC/YqxHNwkQ==", + "dev": true, + "funding": [ + "https://github.com/sponsors/sxzz", + "https://opencollective.com/debug" + ], + "license": "MIT" + }, "node_modules/ohash": { "version": "2.0.11", "resolved": "https://registry.npmjs.org/ohash/-/ohash-2.0.11.tgz", @@ -23756,9 +24307,9 @@ } }, "node_modules/postcss": { - "version": "8.5.6", - "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.5.6.tgz", - "integrity": "sha512-3Ybi1tAuwAP9s0r1UQ2J4n5Y0G05bJkpUIO0/bI9MhwmD70S5aTWbXGBwxHrelT+XM1k6dM0pk+SwNkpTRN7Pg==", + "version": "8.5.9", + "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.5.9.tgz", + "integrity": "sha512-7a70Nsot+EMX9fFU3064K/kdHWZqGVY+BADLyXc8Dfv+mTLLVl6JzJpPaCZ2kQL9gIJvKXSLMHhqdRRjwQeFtw==", "funding": [ { "type": "opencollective", @@ -24355,6 +24906,49 @@ "postcss": "^8.4" } }, + "node_modules/postcss-load-config": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/postcss-load-config/-/postcss-load-config-6.0.1.tgz", + "integrity": "sha512-oPtTM4oerL+UXmx+93ytZVN82RrlY/wPUV8IeDxFrzIjXOLF1pN+EmKPLbubvKHT2HC20xXsCAH2Z+CKV6Oz/g==", + "dev": true, + "funding": [ + { + "type": "opencollective", + "url": "https://opencollective.com/postcss/" + }, + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "license": "MIT", + "dependencies": { + "lilconfig": "^3.1.1" + }, + "engines": { + "node": ">= 18" + }, + "peerDependencies": { + "jiti": ">=1.21.0", + "postcss": ">=8.0.9", + "tsx": "^4.8.1", + "yaml": "^2.4.2" + }, + "peerDependenciesMeta": { + "jiti": { + "optional": true + }, + "postcss": { + "optional": true + }, + "tsx": { + "optional": true + }, + "yaml": { + "optional": true + } + } + }, "node_modules/postcss-loader": { "version": "7.3.4", "resolved": "https://registry.npmjs.org/postcss-loader/-/postcss-loader-7.3.4.tgz", @@ -26735,6 +27329,40 @@ "integrity": "sha512-IXgzBWvWQwE6PrDI05OvmXUIruQTcoMDzRsOd5CDvHCVLcLHMTSYvOK5Cm46kWqlV3yAbuSpBZdJ5oP5OUoStg==", "license": "Unlicense" }, + "node_modules/rolldown": { + "version": "1.0.0-rc.15", + "resolved": "https://registry.npmjs.org/rolldown/-/rolldown-1.0.0-rc.15.tgz", + "integrity": "sha512-Ff31guA5zT6WjnGp0SXw76X6hzGRk/OQq2hE+1lcDe+lJdHSgnSX6nK3erbONHyCbpSj9a9E+uX/OvytZoWp2g==", + "dev": true, + "license": "MIT", + "dependencies": { + "@oxc-project/types": "=0.124.0", + "@rolldown/pluginutils": "1.0.0-rc.15" + }, + "bin": { + "rolldown": "bin/cli.mjs" + }, + "engines": { + "node": "^20.19.0 || >=22.12.0" + }, + "optionalDependencies": { + "@rolldown/binding-android-arm64": "1.0.0-rc.15", + "@rolldown/binding-darwin-arm64": "1.0.0-rc.15", + "@rolldown/binding-darwin-x64": "1.0.0-rc.15", + "@rolldown/binding-freebsd-x64": "1.0.0-rc.15", + "@rolldown/binding-linux-arm-gnueabihf": "1.0.0-rc.15", + "@rolldown/binding-linux-arm64-gnu": "1.0.0-rc.15", + "@rolldown/binding-linux-arm64-musl": "1.0.0-rc.15", + "@rolldown/binding-linux-ppc64-gnu": "1.0.0-rc.15", + "@rolldown/binding-linux-s390x-gnu": "1.0.0-rc.15", + "@rolldown/binding-linux-x64-gnu": "1.0.0-rc.15", + "@rolldown/binding-linux-x64-musl": "1.0.0-rc.15", + "@rolldown/binding-openharmony-arm64": "1.0.0-rc.15", + "@rolldown/binding-wasm32-wasi": "1.0.0-rc.15", + "@rolldown/binding-win32-arm64-msvc": "1.0.0-rc.15", + "@rolldown/binding-win32-x64-msvc": "1.0.0-rc.15" + } + }, "node_modules/rollup": { "version": "4.59.0", "resolved": "https://registry.npmjs.org/rollup/-/rollup-4.59.0.tgz", @@ -27519,6 +28147,13 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/siginfo": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/siginfo/-/siginfo-2.0.0.tgz", + "integrity": "sha512-ybx0WO1/8bSBLEWXZvEd7gMW3Sn3JFlW3TvX1nREbDLRNQNaeNN8WK0meBwPdAaOI7TtRRRJn/Es1zhrrCHu7g==", + "dev": true, + "license": "ISC" + }, "node_modules/signal-exit": { "version": "3.0.7", "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-3.0.7.tgz", @@ -27775,6 +28410,13 @@ "node": ">=8" } }, + "node_modules/stackback": { + "version": "0.0.2", + "resolved": "https://registry.npmjs.org/stackback/-/stackback-0.0.2.tgz", + "integrity": "sha512-1XMJE5fQo1jGH6Y/7ebnwPOBEkIEnT4QF32d5R1+VXdXveM0IBMJt8zfaxX1P3QhVwrYe+576+jkANtSS2mBbw==", + "dev": true, + "license": "MIT" + }, "node_modules/statuses": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/statuses/-/statuses-2.0.2.tgz", @@ -28569,6 +29211,13 @@ "integrity": "sha512-lBN9zLN/oAf68o3zNXYrdCt1kP8WsiGW8Oo2ka41b2IM5JL/S1CTyX1rW0mb/zSuJun0ZUrDxx4sqvYS2FWzPA==", "license": "MIT" }, + "node_modules/tinybench": { + "version": "2.9.0", + "resolved": "https://registry.npmjs.org/tinybench/-/tinybench-2.9.0.tgz", + "integrity": "sha512-0+DUvqWMValLmha6lr4kD8iAMK1HzV0/aKnCtWb9v9641TnP/MFb7Pc2bxoxQjTXAErryXVgUOfv2YqNllqGeg==", + "dev": true, + "license": "MIT" + }, "node_modules/tinyexec": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/tinyexec/-/tinyexec-1.0.2.tgz", @@ -28620,6 +29269,16 @@ "node": "^18.0.0 || >=20.0.0" } }, + "node_modules/tinyrainbow": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/tinyrainbow/-/tinyrainbow-3.1.0.tgz", + "integrity": "sha512-Bf+ILmBgretUrdJxzXM0SgXLZ3XfiaUuOj/IKQHuTXip+05Xn+uyEYdVg0kYDipTBcLrCVyUzAPz7QmArb0mmw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=14.0.0" + } + }, "node_modules/tmpl": { "version": "1.0.5", "resolved": "https://registry.npmjs.org/tmpl/-/tmpl-1.0.5.tgz", @@ -29690,6 +30349,442 @@ "d3-timer": "^3.0.1" } }, + "node_modules/vite": { + "version": "8.0.8", + "resolved": "https://registry.npmjs.org/vite/-/vite-8.0.8.tgz", + "integrity": "sha512-dbU7/iLVa8KZALJyLOBOQ88nOXtNG8vxKuOT4I2mD+Ya70KPceF4IAmDsmU0h1Qsn5bPrvsY9HJstCRh3hG6Uw==", + "dev": true, + "license": "MIT", + "dependencies": { + "lightningcss": "^1.32.0", + "picomatch": "^4.0.4", + "postcss": "^8.5.8", + "rolldown": "1.0.0-rc.15", + "tinyglobby": "^0.2.15" + }, + "bin": { + "vite": "bin/vite.js" + }, + "engines": { + "node": "^20.19.0 || >=22.12.0" + }, + "funding": { + "url": "https://github.com/vitejs/vite?sponsor=1" + }, + "optionalDependencies": { + "fsevents": "~2.3.3" + }, + "peerDependencies": { + "@types/node": "^20.19.0 || >=22.12.0", + "@vitejs/devtools": "^0.1.0", + "esbuild": "^0.27.0 || ^0.28.0", + "jiti": ">=1.21.0", + "less": "^4.0.0", + "sass": "^1.70.0", + "sass-embedded": "^1.70.0", + "stylus": ">=0.54.8", + "sugarss": "^5.0.0", + "terser": "^5.16.0", + "tsx": "^4.8.1", + "yaml": "^2.4.2" + }, + "peerDependenciesMeta": { + "@types/node": { + "optional": true + }, + "@vitejs/devtools": { + "optional": true + }, + "esbuild": { + "optional": true + }, + "jiti": { + "optional": true + }, + "less": { + "optional": true + }, + "sass": { + "optional": true + }, + "sass-embedded": { + "optional": true + }, + "stylus": { + "optional": true + }, + "sugarss": { + "optional": true + }, + "terser": { + "optional": true + }, + "tsx": { + "optional": true + }, + "yaml": { + "optional": true + } + } + }, + "node_modules/vite/node_modules/lightningcss": { + "version": "1.32.0", + "resolved": "https://registry.npmjs.org/lightningcss/-/lightningcss-1.32.0.tgz", + "integrity": "sha512-NXYBzinNrblfraPGyrbPoD19C1h9lfI/1mzgWYvXUTe414Gz/X1FD2XBZSZM7rRTrMA8JL3OtAaGifrIKhQ5yQ==", + "dev": true, + "license": "MPL-2.0", + "dependencies": { + "detect-libc": "^2.0.3" + }, + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + }, + "optionalDependencies": { + "lightningcss-android-arm64": "1.32.0", + "lightningcss-darwin-arm64": "1.32.0", + "lightningcss-darwin-x64": "1.32.0", + "lightningcss-freebsd-x64": "1.32.0", + "lightningcss-linux-arm-gnueabihf": "1.32.0", + "lightningcss-linux-arm64-gnu": "1.32.0", + "lightningcss-linux-arm64-musl": "1.32.0", + "lightningcss-linux-x64-gnu": "1.32.0", + "lightningcss-linux-x64-musl": "1.32.0", + "lightningcss-win32-arm64-msvc": "1.32.0", + "lightningcss-win32-x64-msvc": "1.32.0" + } + }, + "node_modules/vite/node_modules/lightningcss-android-arm64": { + "version": "1.32.0", + "resolved": "https://registry.npmjs.org/lightningcss-android-arm64/-/lightningcss-android-arm64-1.32.0.tgz", + "integrity": "sha512-YK7/ClTt4kAK0vo6w3X+Pnm0D2cf2vPHbhOXdoNti1Ga0al1P4TBZhwjATvjNwLEBCnKvjJc2jQgHXH0NEwlAg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/vite/node_modules/lightningcss-darwin-arm64": { + "version": "1.32.0", + "resolved": "https://registry.npmjs.org/lightningcss-darwin-arm64/-/lightningcss-darwin-arm64-1.32.0.tgz", + "integrity": "sha512-RzeG9Ju5bag2Bv1/lwlVJvBE3q6TtXskdZLLCyfg5pt+HLz9BqlICO7LZM7VHNTTn/5PRhHFBSjk5lc4cmscPQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/vite/node_modules/lightningcss-darwin-x64": { + "version": "1.32.0", + "resolved": "https://registry.npmjs.org/lightningcss-darwin-x64/-/lightningcss-darwin-x64-1.32.0.tgz", + "integrity": "sha512-U+QsBp2m/s2wqpUYT/6wnlagdZbtZdndSmut/NJqlCcMLTWp5muCrID+K5UJ6jqD2BFshejCYXniPDbNh73V8w==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/vite/node_modules/lightningcss-freebsd-x64": { + "version": "1.32.0", + "resolved": "https://registry.npmjs.org/lightningcss-freebsd-x64/-/lightningcss-freebsd-x64-1.32.0.tgz", + "integrity": "sha512-JCTigedEksZk3tHTTthnMdVfGf61Fky8Ji2E4YjUTEQX14xiy/lTzXnu1vwiZe3bYe0q+SpsSH/CTeDXK6WHig==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/vite/node_modules/lightningcss-linux-arm-gnueabihf": { + "version": "1.32.0", + "resolved": "https://registry.npmjs.org/lightningcss-linux-arm-gnueabihf/-/lightningcss-linux-arm-gnueabihf-1.32.0.tgz", + "integrity": "sha512-x6rnnpRa2GL0zQOkt6rts3YDPzduLpWvwAF6EMhXFVZXD4tPrBkEFqzGowzCsIWsPjqSK+tyNEODUBXeeVHSkw==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/vite/node_modules/lightningcss-linux-arm64-gnu": { + "version": "1.32.0", + "resolved": "https://registry.npmjs.org/lightningcss-linux-arm64-gnu/-/lightningcss-linux-arm64-gnu-1.32.0.tgz", + "integrity": "sha512-0nnMyoyOLRJXfbMOilaSRcLH3Jw5z9HDNGfT/gwCPgaDjnx0i8w7vBzFLFR1f6CMLKF8gVbebmkUN3fa/kQJpQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/vite/node_modules/lightningcss-linux-arm64-musl": { + "version": "1.32.0", + "resolved": "https://registry.npmjs.org/lightningcss-linux-arm64-musl/-/lightningcss-linux-arm64-musl-1.32.0.tgz", + "integrity": "sha512-UpQkoenr4UJEzgVIYpI80lDFvRmPVg6oqboNHfoH4CQIfNA+HOrZ7Mo7KZP02dC6LjghPQJeBsvXhJod/wnIBg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/vite/node_modules/lightningcss-linux-x64-gnu": { + "version": "1.32.0", + "resolved": "https://registry.npmjs.org/lightningcss-linux-x64-gnu/-/lightningcss-linux-x64-gnu-1.32.0.tgz", + "integrity": "sha512-V7Qr52IhZmdKPVr+Vtw8o+WLsQJYCTd8loIfpDaMRWGUZfBOYEJeyJIkqGIDMZPwPx24pUMfwSxxI8phr/MbOA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/vite/node_modules/lightningcss-linux-x64-musl": { + "version": "1.32.0", + "resolved": "https://registry.npmjs.org/lightningcss-linux-x64-musl/-/lightningcss-linux-x64-musl-1.32.0.tgz", + "integrity": "sha512-bYcLp+Vb0awsiXg/80uCRezCYHNg1/l3mt0gzHnWV9XP1W5sKa5/TCdGWaR/zBM2PeF/HbsQv/j2URNOiVuxWg==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/vite/node_modules/lightningcss-win32-arm64-msvc": { + "version": "1.32.0", + "resolved": "https://registry.npmjs.org/lightningcss-win32-arm64-msvc/-/lightningcss-win32-arm64-msvc-1.32.0.tgz", + "integrity": "sha512-8SbC8BR40pS6baCM8sbtYDSwEVQd4JlFTOlaD3gWGHfThTcABnNDBda6eTZeqbofalIJhFx0qKzgHJmcPTnGdw==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/vite/node_modules/lightningcss-win32-x64-msvc": { + "version": "1.32.0", + "resolved": "https://registry.npmjs.org/lightningcss-win32-x64-msvc/-/lightningcss-win32-x64-msvc-1.32.0.tgz", + "integrity": "sha512-Amq9B/SoZYdDi1kFrojnoqPLxYhQ4Wo5XiL8EVJrVsB8ARoC1PWW6VGtT0WKCemjy8aC+louJnjS7U18x3b06Q==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/vitest": { + "version": "4.1.4", + "resolved": "https://registry.npmjs.org/vitest/-/vitest-4.1.4.tgz", + "integrity": "sha512-tFuJqTxKb8AvfyqMfnavXdzfy3h3sWZRWwfluGbkeR7n0HUev+FmNgZ8SDrRBTVrVCjgH5cA21qGbCffMNtWvg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@vitest/expect": "4.1.4", + "@vitest/mocker": "4.1.4", + "@vitest/pretty-format": "4.1.4", + "@vitest/runner": "4.1.4", + "@vitest/snapshot": "4.1.4", + "@vitest/spy": "4.1.4", + "@vitest/utils": "4.1.4", + "es-module-lexer": "^2.0.0", + "expect-type": "^1.3.0", + "magic-string": "^0.30.21", + "obug": "^2.1.1", + "pathe": "^2.0.3", + "picomatch": "^4.0.3", + "std-env": "^4.0.0-rc.1", + "tinybench": "^2.9.0", + "tinyexec": "^1.0.2", + "tinyglobby": "^0.2.15", + "tinyrainbow": "^3.1.0", + "vite": "^6.0.0 || ^7.0.0 || ^8.0.0", + "why-is-node-running": "^2.3.0" + }, + "bin": { + "vitest": "vitest.mjs" + }, + "engines": { + "node": "^20.0.0 || ^22.0.0 || >=24.0.0" + }, + "funding": { + "url": "https://opencollective.com/vitest" + }, + "peerDependencies": { + "@edge-runtime/vm": "*", + "@opentelemetry/api": "^1.9.0", + "@types/node": "^20.0.0 || ^22.0.0 || >=24.0.0", + "@vitest/browser-playwright": "4.1.4", + "@vitest/browser-preview": "4.1.4", + "@vitest/browser-webdriverio": "4.1.4", + "@vitest/coverage-istanbul": "4.1.4", + "@vitest/coverage-v8": "4.1.4", + "@vitest/ui": "4.1.4", + "happy-dom": "*", + "jsdom": "*", + "vite": "^6.0.0 || ^7.0.0 || ^8.0.0" + }, + "peerDependenciesMeta": { + "@edge-runtime/vm": { + "optional": true + }, + "@opentelemetry/api": { + "optional": true + }, + "@types/node": { + "optional": true + }, + "@vitest/browser-playwright": { + "optional": true + }, + "@vitest/browser-preview": { + "optional": true + }, + "@vitest/browser-webdriverio": { + "optional": true + }, + "@vitest/coverage-istanbul": { + "optional": true + }, + "@vitest/coverage-v8": { + "optional": true + }, + "@vitest/ui": { + "optional": true + }, + "happy-dom": { + "optional": true + }, + "jsdom": { + "optional": true + }, + "vite": { + "optional": false + } + } + }, + "node_modules/vitest/node_modules/std-env": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/std-env/-/std-env-4.0.0.tgz", + "integrity": "sha512-zUMPtQ/HBY3/50VbpkupYHbRroTRZJPRLvreamgErJVys0ceuzMkD44J/QjqhHjOzK42GQ3QZIeFG1OYfOtKqQ==", + "dev": true, + "license": "MIT" + }, "node_modules/vscode-jsonrpc": { "version": "8.2.0", "resolved": "https://registry.npmjs.org/vscode-jsonrpc/-/vscode-jsonrpc-8.2.0.tgz", @@ -30382,6 +31477,23 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/why-is-node-running": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/why-is-node-running/-/why-is-node-running-2.3.0.tgz", + "integrity": "sha512-hUrmaWBdVDcxvYqnyh09zunKzROWjbZTiNy8dBEjkS7ehEDQibXJ7XvlmtbwuTclUiIyN+CyXQD4Vmko8fNm8w==", + "dev": true, + "license": "MIT", + "dependencies": { + "siginfo": "^2.0.0", + "stackback": "0.0.2" + }, + "bin": { + "why-is-node-running": "cli.js" + }, + "engines": { + "node": ">=8" + } + }, "node_modules/widest-line": { "version": "4.0.1", "resolved": "https://registry.npmjs.org/widest-line/-/widest-line-4.0.1.tgz", diff --git a/package.json b/package.json index a284b843..86e84999 100644 --- a/package.json +++ b/package.json @@ -4,7 +4,8 @@ "workspaces": [ "hindsight-clients/typescript", "hindsight-control-plane", - "hindsight-docs" + "hindsight-docs", + "hindsight-all-npm" ], "scripts": { "prepare": "./scripts/setup-hooks.sh" diff --git a/scripts/dev/start-docs.sh b/scripts/dev/start-docs.sh index bb259ecd..c5caca0c 100755 --- a/scripts/dev/start-docs.sh +++ b/scripts/dev/start-docs.sh @@ -9,9 +9,15 @@ set -e PROJECT_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" cd "$PROJECT_ROOT" || exit 1 +# Always show the "current" (Next / unreleased) docs version in local dev. +# docusaurus.config.ts reads this flag to decide which versions to include. +# Do NOT rely on NODE_ENV for this — it's unreliable across hot-reload paths. +export INCLUDE_CURRENT_VERSION=true + echo "Starting documentation server..." echo "" echo "Starting Docusaurus development server..." echo "Documentation will be available at: http://localhost:3000" +echo "INCLUDE_CURRENT_VERSION=true (Next/unreleased docs visible)" echo "" npm run start -w hindsight-docs -- --no-open diff --git a/scripts/release.sh b/scripts/release.sh index c602fb5a..e7784acd 100755 --- a/scripts/release.sh +++ b/scripts/release.sh @@ -124,6 +124,16 @@ else print_warn "File $CONTROL_PLANE_PKG not found, skipping" fi +# Update hindsight-all npm wrapper package.json +ALL_NPM_PKG="hindsight-all-npm/package.json" +if [ -f "$ALL_NPM_PKG" ]; then + print_info "Updating $ALL_NPM_PKG" + sed -i.bak "s/\"version\": \".*\"/\"version\": \"$VERSION\"/" "$ALL_NPM_PKG" + rm "${ALL_NPM_PKG}.bak" +else + print_warn "File $ALL_NPM_PKG not found, skipping" +fi + # Update Python API client PYTHON_CLIENT_PKG="hindsight-clients/python/pyproject.toml" if [ -f "$PYTHON_CLIENT_PKG" ]; then @@ -189,6 +199,7 @@ COMMIT_MSG="Release v$VERSION - Python packages: hindsight-api, hindsight-dev, hindsight-all, hindsight-embed - Python client: hindsight-clients/python - TypeScript client: hindsight-clients/typescript +- hindsight-all npm wrapper: hindsight-all-npm - Rust CLI: hindsight-cli - Control Plane: hindsight-control-plane - Helm chart" diff --git a/skills/hindsight-docs/references/changelog/index.md b/skills/hindsight-docs/references/changelog/index.md index d54157d3..ef3b1d3b 100644 --- a/skills/hindsight-docs/references/changelog/index.md +++ b/skills/hindsight-docs/references/changelog/index.md @@ -6,6 +6,58 @@ import PageHero from '@site/src/components/PageHero'; +## [0.5.0](https://github.com/vectorize-io/hindsight/releases/tag/v0.5.0) + +**Breaking Changes** + +- Removed BFS and MPFP graph retrieval strategies. LinkExpansionRetriever is now the sole graph retrieval algorithm, offering simpler, faster, and more accurate results. ([`ea834bc7`](https://github.com/vectorize-io/hindsight/commit/ea834bc7)) +- Dropped the `hindsight-hermes` integration package. ([`cf0537ba`](https://github.com/vectorize-io/hindsight/commit/cf0537ba)) + +**Features** + +- Built-in llama.cpp LLM provider for fully local inference without external API calls. ([`f74b577e`](https://github.com/vectorize-io/hindsight/commit/f74b577e)) +- Retain `update_mode='append'` for concatenating new content onto an existing document instead of replacing it. ([`3c633e5e`](https://github.com/vectorize-io/hindsight/commit/3c633e5e)) +- OpenRouter support for LLM, embeddings, and reranking. ([`e5944b63`](https://github.com/vectorize-io/hindsight/commit/e5944b63)) +- Bank template import/export with Template Hub — export a bank's configuration, mental models, and directives as a reusable manifest, then import into other banks. ([`30a319a6`](https://github.com/vectorize-io/hindsight/commit/30a319a6)) +- Constellation view in the Control Plane — interactive, zoomable canvas visualization of entity relationship graphs with heat-gradient coloring and dark mode support. ([`36783df3`](https://github.com/vectorize-io/hindsight/commit/36783df3)) +- Added `detail` parameter to list/get mental model endpoints for controlling response verbosity. ([`8d1bfbbd`](https://github.com/vectorize-io/hindsight/commit/8d1bfbbd)) +- Added AutoGen integration (`hindsight-autogen`) for persistent long-term memory in AutoGen agents. ([`a757765a`](https://github.com/vectorize-io/hindsight/commit/a757765a)) +- Added Paperclip integration (`@vectorize-io/hindsight-paperclip`) with Express middleware and process adapter modes for stateless agent memory. ([`81441ee9`](https://github.com/vectorize-io/hindsight/commit/81441ee9)) +- Added OpenCode persistent memory plugin for the OpenCode editor. ([`e1c6220f`](https://github.com/vectorize-io/hindsight/commit/e1c6220f)) +- OpenClaw JSONL-backed retain queue for external API resilience — buffers retain calls locally when the API is unreachable. ([`087545cc`](https://github.com/vectorize-io/hindsight/commit/087545cc)) +- OpenClaw now supports `bankId` for static bank configurations. ([`0e81d1a2`](https://github.com/vectorize-io/hindsight/commit/0e81d1a2)) +- Added Google embeddings and reranker provider support. ([`07de798c`](https://github.com/vectorize-io/hindsight/commit/07de798c)) +- Added persistent volume support in Helm chart for local model cache. ([`cefa7554`](https://github.com/vectorize-io/hindsight/commit/cefa7554)) +- MCP server now includes a `sync_retain` tool and validates UUID inputs. ([`48185a4b`](https://github.com/vectorize-io/hindsight/commit/48185a4b)) +- Recall combined scoring now includes `proof_count` boost for better ranking. ([`26794aab`](https://github.com/vectorize-io/hindsight/commit/26794aab)) + +**Improvements** + +- 3-phase retain pipeline restructures memory ingestion into pre-resolve, insert, and post-link phases, dramatically improving throughput under concurrent load by removing slow reads from write transactions. ([`914ba796`](https://github.com/vectorize-io/hindsight/commit/914ba796)) +- Recall entity graph expansion now caps per-entity fanout and includes a timeout fallback, preventing slow queries on banks with high-fanout entities. ([`57f15445`](https://github.com/vectorize-io/hindsight/commit/57f15445)) +- Fact serialization in think-prompt now includes `occurred_end` and `mentioned_at` for richer temporal context. ([`37348c85`](https://github.com/vectorize-io/hindsight/commit/37348c85)) +- Consolidation observation quality improved with structured processing rules. ([`6f173b10`](https://github.com/vectorize-io/hindsight/commit/6f173b10)) + +**Bug Fixes** + +- LiteLLM SDK embeddings `encoding_format` is now configurable instead of hardcoded. ([`cece2c90`](https://github.com/vectorize-io/hindsight/commit/cece2c90)) +- Fixed out-of-range `content_index` crash in recall result mapping. ([`9790d904`](https://github.com/vectorize-io/hindsight/commit/9790d904)) +- Experience fact types are now preserved correctly during normalization. ([`9cfdd464`](https://github.com/vectorize-io/hindsight/commit/9cfdd464)) +- Clear memories endpoint no longer deletes the bank profile. ([`26a64cc0`](https://github.com/vectorize-io/hindsight/commit/26a64cc0)) +- Embedding daemon clears stale processes on the port before starting. ([`7d6c570a`](https://github.com/vectorize-io/hindsight/commit/7d6c570a)) +- Per-bank vector index migration now respects vector extension configuration. ([`4fd7c5d1`](https://github.com/vectorize-io/hindsight/commit/4fd7c5d1)) +- Timeline group sort uses numeric date comparison instead of locale string comparison. ([`f3f2c6b0`](https://github.com/vectorize-io/hindsight/commit/f3f2c6b0)) +- Resolved 25 test regressions from the streaming retain pipeline. ([`7415ebff`](https://github.com/vectorize-io/hindsight/commit/7415ebff)) +- MCP server now auto-coerces string-encoded JSON in tool arguments. ([`443c94c8`](https://github.com/vectorize-io/hindsight/commit/443c94c8)) +- Entity labels structure is now validated on PATCH to prevent invalid configurations. ([`7e23f8e1`](https://github.com/vectorize-io/hindsight/commit/7e23f8e1)) +- Fixed `bank_id` metric label to be opt-in, preventing OTel memory leak. ([`cf4bd598`](https://github.com/vectorize-io/hindsight/commit/cf4bd598)) +- Fixed `max_tokens` handling for OpenAI-compatible endpoints with custom base URLs. ([`cd99eef4`](https://github.com/vectorize-io/hindsight/commit/cd99eef4)) +- Fixed `event_date` AttributeError when date is None in fact extraction. ([`6cb309f7`](https://github.com/vectorize-io/hindsight/commit/6cb309f7)) +- Query analyzer now handles dateparser internal crashes gracefully. ([`e0e65c44`](https://github.com/vectorize-io/hindsight/commit/e0e65c44)) +- Embedding profile `.env` overwrite skipped when config has no Hindsight keys. ([`9e2890ba`](https://github.com/vectorize-io/hindsight/commit/9e2890ba)) +- Windows compatibility fix for hindsight-embed. ([`f9fe6953`](https://github.com/vectorize-io/hindsight/commit/f9fe6953)) +- Addressed critical and high severity security vulnerabilities in dependencies. ([`ee4510a7`](https://github.com/vectorize-io/hindsight/commit/ee4510a7)) + ## [0.4.22](https://github.com/vectorize-io/hindsight/releases/tag/v0.4.22) **Features** diff --git a/skills/hindsight-docs/references/developer/models.md b/skills/hindsight-docs/references/developer/models.md index 63f359b9..bd765350 100644 --- a/skills/hindsight-docs/references/developer/models.md +++ b/skills/hindsight-docs/references/developer/models.md @@ -89,6 +89,7 @@ Each provider has a recommended default model that's used when `HINDSIGHT_API_LL | `claude-code` | `claude-sonnet-4-5-20250929` | | `bedrock` | `us.amazon.nova-2-lite-v1:0` | | `volcano` | `doubao-pro-32k` | +| `openrouter` | `qwen/qwen3.5-9b` | | `litellm` | `gpt-4o-mini` | **Example:** Setting just the provider uses its default model: diff --git a/skills/hindsight-docs/references/openapi.json b/skills/hindsight-docs/references/openapi.json index efac4477..78d349b7 100644 --- a/skills/hindsight-docs/references/openapi.json +++ b/skills/hindsight-docs/references/openapi.json @@ -10,7 +10,7 @@ "name": "Apache 2.0", "url": "https://www.apache.org/licenses/LICENSE-2.0.html" }, - "version": "0.4.22" + "version": "0.5.0" }, "paths": { "/health": { diff --git a/skills/hindsight-docs/references/sdks/embed.md b/skills/hindsight-docs/references/sdks/embed.md index 9c56338f..d4cc0a1f 100644 --- a/skills/hindsight-docs/references/sdks/embed.md +++ b/skills/hindsight-docs/references/sdks/embed.md @@ -2,7 +2,7 @@ sidebar_position: 5 --- -# Embedded SDK (hindsight-embed) +# Daemon CLI (hindsight-embed) Zero-configuration local memory system with automatic daemon management. Perfect for development, prototyping, and single-user applications. diff --git a/skills/hindsight-docs/references/sdks/hindsight-all-npm.md b/skills/hindsight-docs/references/sdks/hindsight-all-npm.md new file mode 100644 index 00000000..72e61d28 --- /dev/null +++ b/skills/hindsight-docs/references/sdks/hindsight-all-npm.md @@ -0,0 +1,90 @@ +--- +sidebar_position: 6 +--- + +# Programmatic API (Node.js) + +The `@vectorize-io/hindsight-all` npm package is the Node.js equivalent of the Python [`hindsight-all`](./hindsight-all.md) package. It lets your Node code spawn and supervise a local Hindsight daemon without deploying any server infrastructure — pair it with [`@vectorize-io/hindsight-client`](./nodejs.md) for memory operations. + +The daemon runs as a **separate OS process** on `127.0.0.1` (not in your Node process). Your code talks to it over HTTP via `HindsightClient`. + +This package **does not ship an HTTP client** — it only owns the server process. Once the daemon is running, talk to it with [`@vectorize-io/hindsight-client`](./nodejs.md) against `server.getBaseUrl()`. The two packages compose: one owns the process, the other owns the API surface. + +## How it works + +1. `server.start()` resolves the underlying `hindsight-embed` command (via `uvx` from PyPI, or `uv run --directory ` for a local checkout). +2. Runs `profile create --merge --port [--env KEY=VALUE ...]` with every entry from `options.env` forwarded as `--env`. +3. Runs `daemon --profile start`. +4. Polls `http://host:port/health` until it returns `200` or the `readyTimeoutMs` budget is exhausted. +5. `server.stop()` runs `daemon --profile stop`. + +The server is intentionally transparent: new daemon env vars or CLI flags never require a wrapper release — pass them through `env`, `extraProfileCreateArgs`, or `extraDaemonStartArgs`. + +## Requirements + +- **Node.js ≥ 22** — uses global `fetch` and `AbortSignal.timeout`. +- **`uv` / `uvx`** on `PATH` — used to download and run the Hindsight daemon. Install via [docs.astral.sh/uv](https://docs.astral.sh/uv/). + +## Install + +```bash +npm install @vectorize-io/hindsight-all @vectorize-io/hindsight-client +``` + +## Example + +```ts +import { HindsightServer, consoleLogger } from '@vectorize-io/hindsight-all'; +import { HindsightClient } from '@vectorize-io/hindsight-client'; + +const server = new HindsightServer({ + profile: 'my-app', + port: 9077, + env: { + HINDSIGHT_API_LLM_PROVIDER: 'anthropic', + HINDSIGHT_API_LLM_API_KEY: process.env.ANTHROPIC_API_KEY, + HINDSIGHT_API_LLM_MODEL: 'claude-sonnet-4-20250514', + HINDSIGHT_EMBED_DAEMON_IDLE_TIMEOUT: '0', + }, + logger: consoleLogger, +}); + +await server.start(); + +const client = new HindsightClient({ baseUrl: server.getBaseUrl() }); +await client.retain('user-123', 'User prefers dark mode.'); +const recall = await client.recall('user-123', 'what are the user preferences?'); + +await server.stop(); +``` + +For a remote Hindsight API, skip the server entirely and point `HindsightClient` directly at the remote URL. + +## `HindsightServerOptions` + +| Option | Type | Default | Description | +|---|---|---|---| +| `profile` | `string` | `"default"` | Profile name passed to `--profile` on every sub-command. | +| `port` | `number` | `8888` | TCP port the daemon listens on. | +| `host` | `string` | `"127.0.0.1"` | Hostname the daemon binds to (used for health checks). | +| `embedVersion` | `string` | `"latest"` | Version of the underlying `hindsight-embed` package to run via `uvx`. | +| `embedPackagePath` | `string` | — | Local checkout path — takes precedence over `embedVersion`. Uses `uv run --directory` instead of `uvx`. | +| `env` | `Record` | `{}` | Environment variables passed to the daemon process **and** written into the profile config via `--env KEY=VALUE`. The preferred way to surface any `HINDSIGHT_API_*` / `HINDSIGHT_EMBED_*` setting. | +| `extraProfileCreateArgs` | `string[]` | `[]` | Extra args appended verbatim to `profile create`. | +| `extraDaemonStartArgs` | `string[]` | `[]` | Extra args appended verbatim to `daemon start`. | +| `platformCpuWorkaround` | `boolean` | `true` on macOS | Auto-set `HINDSIGHT_API_EMBEDDINGS_LOCAL_FORCE_CPU=1` and `HINDSIGHT_API_RERANKER_LOCAL_FORCE_CPU=1` to avoid Metal/MPS crashes. Caller-supplied `env` values win over the auto-applied ones. | +| `readyTimeoutMs` | `number` | `30000` | Max time to wait for `/health` to return 200. | +| `readyPollIntervalMs` | `number` | `1000` | Polling interval while waiting for `/health`. | +| `logger` | `Logger` | silent | Pluggable logger (`debug`/`info`/`warn`/`error`). `consoleLogger` and `silentLogger` helpers are exported. | + +## Server methods + +| Method | Returns | Description | +|---|---|---| +| `start()` | `Promise` | Configure profile, spawn the daemon, wait for `/health`. Idempotent — safe to re-run. | +| `stop()` | `Promise` | Stop the daemon. Never throws; logs and resolves even on failure. | +| `checkHealth()` | `Promise` | One-shot `/health` probe with a 2 s timeout. | +| `getBaseUrl()` | `string` | `http://host:port` — pass this straight to `HindsightClient`. | +| `getProfile()` | `string` | The profile name this server operates on. | + +For memory operations (retain, recall, reflect, bank management) use [`@vectorize-io/hindsight-client`](./nodejs.md). diff --git a/skills/hindsight-docs/references/sdks/hindsight-all.md b/skills/hindsight-docs/references/sdks/hindsight-all.md new file mode 100644 index 00000000..ee9dde50 --- /dev/null +++ b/skills/hindsight-docs/references/sdks/hindsight-all.md @@ -0,0 +1,146 @@ +--- +sidebar_position: 2 +--- + +# Programmatic API (Python) + +The `hindsight-all` Python package lets your code spawn and manage a local Hindsight daemon without deploying any server infrastructure. It bundles the Hindsight API server, embedded PostgreSQL, and the Python client into one install — `pip install hindsight-all` and you can start a fully-functional Hindsight instance from a few lines of Python. + +The daemon runs as a **separate OS process** on `127.0.0.1` (not in your Python process memory). Your code talks to it over HTTP via the bundled `HindsightClient`. + +If you already have a Hindsight server running and just need a client, use [Python Client (hindsight-client)](./python.md) instead. + +## How it works + +`hindsight-all` exposes two primary APIs: + +- **`HindsightServer`** — explicit lifecycle. Use it as a context manager when you want deterministic startup/shutdown (e.g. in tests). +- **`HindsightEmbedded`** — auto-managed. Starts a daemon on first use, reuses it across calls, shuts it down after an idle timeout. Easiest for application code that doesn't want to think about lifecycle. + +Both end up talking to the same underlying daemon via the same `HindsightClient` HTTP interface — the difference is only how the server process is managed. + +## Installation + +```bash +pip install hindsight-all +``` + +The `hindsight-all` wheel bundles `hindsight-api-slim`, `hindsight-client`, and `hindsight-embed` as dependencies, so one `pip install` gets you everything. + +## `HindsightServer` — explicit lifecycle + +Use `HindsightServer` as a context manager when you want the server to start immediately, run for the duration of a block, and shut down cleanly afterwards. Ideal for tests and short-lived scripts. + +```python +import os +from hindsight import HindsightServer, HindsightClient + +with HindsightServer( + llm_provider="openai", + llm_model="gpt-4o-mini", + llm_api_key=os.environ["OPENAI_API_KEY"], +) as server: + client = HindsightClient(base_url=server.url) + + client.retain(bank_id="my-bank", content="Alice works at Google") + results = client.recall(bank_id="my-bank", query="What does Alice do?") + for r in results: + print(r.text) + + answer = client.reflect(bank_id="my-bank", query="Tell me about Alice") + print(answer.text) +# Server is stopped here +``` + +## `HindsightEmbedded` — auto-managed + +`HindsightEmbedded` is the simplest way to use Hindsight in Python. It automatically manages a background daemon for you — starts on first use, stays alive across calls, shuts down after an idle timeout. + +```python +from hindsight import HindsightEmbedded +import os + +# Server starts automatically on first call +client = HindsightEmbedded( + profile="myapp", # Profile for data isolation + llm_provider="openai", + llm_model="gpt-4o-mini", + llm_api_key=os.environ["OPENAI_API_KEY"], +) + +# Use immediately - no manual server management needed +client.retain(bank_id="my-bank", content="Alice works at Google") +results = client.recall(bank_id="my-bank", query="What does Alice do?") + +# Server continues running (auto-stops after idle timeout) +# Or explicitly stop it: +client.close(stop_daemon=True) +``` + +### What's a Profile? + +A profile is an isolated Hindsight environment. Each profile gets its own embedded PostgreSQL database (stored in `~/.pg0/instances/hindsight-embed-{profile}/`) and its own API server. Use different profiles to separate environments (dev/prod), applications, or users. + +### When to use which + +| Use case | Pick | +|---|---| +| Tests, short-lived scripts, deterministic startup/shutdown | `HindsightServer` (context manager) | +| Long-running application, auto-start on first use, don't want to manage lifecycle | `HindsightEmbedded` | +| Existing Hindsight server running elsewhere | [`hindsight-client`](./python.md) directly | + +## API namespaces + +Both `HindsightEmbedded` and `HindsightClient` expose organized API namespaces for bank management, mental models, directives, and memories: + +```python +from hindsight import HindsightEmbedded +import os + +embedded = HindsightEmbedded( + profile="myapp", + llm_provider="openai", + llm_api_key=os.environ["OPENAI_API_KEY"], +) + +# Core operations +embedded.retain(bank_id="test", content="Hello") +results = embedded.recall(bank_id="test", query="Hello") + +# Bank management +embedded.banks.create(bank_id="test", name="Test Bank", mission="Help users") +embedded.banks.set_mission(bank_id="test", mission="Updated mission") +embedded.banks.delete(bank_id="test") + +# Mental models +embedded.mental_models.create( + bank_id="test", + name="User Preferences", + content="User prefers dark mode" +) +models = embedded.mental_models.list(bank_id="test") + +# Directives +embedded.directives.create( + bank_id="test", + name="Response Style", + content="Be concise and friendly" +) +directives = embedded.directives.list(bank_id="test") + +# List memories +memories = embedded.memories.list(bank_id="test", type="world", limit=50) +``` + +API namespaces ensure the daemon is running before each call, so daemon crashes are handled gracefully: + +```python +# ✅ GOOD - Uses API namespace (daemon restarts handled) +embedded.banks.create(bank_id="test", name="Test") + +# ❌ BAD - Direct client access (daemon crashes NOT handled) +client = embedded.client +client.create_bank(bank_id="test", name="Test") # Fails if daemon crashed +``` + +For the full reference of retain/recall/reflect methods and their options (which work the same regardless of how you obtain the client) see the [Python Client page](./python.md). diff --git a/skills/hindsight-docs/references/sdks/python.md b/skills/hindsight-docs/references/sdks/python.md index 1ddfb774..86b4419b 100644 --- a/skills/hindsight-docs/references/sdks/python.md +++ b/skills/hindsight-docs/references/sdks/python.md @@ -4,66 +4,18 @@ sidebar_position: 1 # Python Client -Official Python client for the Hindsight API. +Official HTTP client for the Hindsight API. Use this when you have a Hindsight server already running — locally, in Docker, or as a managed service — and you want a typed Python client to talk to it. -import Tabs from '@theme/Tabs'; -import TabItem from '@theme/TabItem'; +If you want to **embed and run a Hindsight server in your Python process** (no external server required), see [Embedded Python (hindsight-all)](./hindsight-all.md) instead. ## Installation - - - -The `hindsight-all` package includes embedded PostgreSQL, HTTP API server, and client: - -```bash -pip install hindsight-all -``` - - - - -If you already have a Hindsight server running: - ```bash pip install hindsight-client ``` - - - ## Quick Start - - - -```python -import os -from hindsight import HindsightServer, HindsightClient - -with HindsightServer( - llm_provider="openai", - llm_model="gpt-4o-mini", - llm_api_key=os.environ["OPENAI_API_KEY"] -) as server: - client = HindsightClient(base_url=server.url) - - # Retain a memory - client.retain(bank_id="my-bank", content="Alice works at Google") - - # Recall memories - results = client.recall(bank_id="my-bank", query="What does Alice do?") - for r in results: - print(r.text) - - # Reflect - generate response with disposition - answer = client.reflect(bank_id="my-bank", query="Tell me about Alice") - print(answer.text) -``` - - - - ```python from hindsight_client import Hindsight @@ -74,130 +26,36 @@ client.retain(bank_id="my-bank", content="Alice works at Google") # Recall memories results = client.recall(bank_id="my-bank", query="What does Alice do?") -for r in results: +for r in results.results: print(r.text) -# Reflect - generate response with disposition +# Reflect - generate a contextual answer answer = client.reflect(bank_id="my-bank", query="Tell me about Alice") print(answer.text) ``` - - - -## Embedded Client (Easiest Option) - -`HindsightEmbedded` provides the simplest way to use Hindsight in Python. It automatically manages a background server for you - no manual setup required: - -```python -from hindsight import HindsightEmbedded -import os - -# Server starts automatically on first use -client = HindsightEmbedded( - profile="myapp", # Profile for data isolation - llm_provider="openai", - llm_model="gpt-4o-mini", - llm_api_key=os.environ["OPENAI_API_KEY"], -) - -# Use immediately - no manual server management needed -client.retain(bank_id="my-bank", content="Alice works at Google") -results = client.recall(bank_id="my-bank", query="What does Alice do?") - -# Server continues running (auto-stops after idle timeout) -# Or explicitly stop it: -client.close(stop_daemon=True) -``` - -**What's a Profile?** - -A profile is an isolated Hindsight environment. Each profile gets its own PostgreSQL database (stored in `~/.pg0/instances/hindsight-embed-{profile}/`) and its own API server. Use different profiles to separate environments (dev/prod), applications, or users. - -**When to Use HindsightEmbedded** - -Use `HindsightEmbedded` when you want the server to start automatically and manage itself. Use `HindsightServer` when you need explicit control over server lifecycle (e.g., testing where you want immediate startup/shutdown). - -**Advanced Operations** - -`HindsightEmbedded` provides organized API namespaces for advanced operations. Each method call automatically ensures the daemon is running: - -```python -from hindsight import HindsightEmbedded -import os - -embedded = HindsightEmbedded( - profile="myapp", - llm_provider="openai", - llm_api_key=os.environ["OPENAI_API_KEY"], -) - -# Core operations (automatically proxied) -embedded.retain(bank_id="test", content="Hello") -results = embedded.recall(bank_id="test", query="Hello") - -# Bank management -embedded.banks.create(bank_id="test", name="Test Bank", mission="Help users") -embedded.banks.set_mission(bank_id="test", mission="Updated mission") -embedded.banks.delete(bank_id="test") - -# Mental models -embedded.mental_models.create( - bank_id="test", - name="User Preferences", - content="User prefers dark mode" -) -models = embedded.mental_models.list(bank_id="test") -embedded.mental_models.update(bank_id="test", mental_model_id="...", content="New content") - -# Directives -embedded.directives.create( - bank_id="test", - name="Response Style", - content="Be concise and friendly" -) -directives = embedded.directives.list(bank_id="test") - -# List memories -memories = embedded.memories.list(bank_id="test", type="world", limit=50) -``` - -**Why Use API Namespaces?** - -API namespaces (`banks`, `mental_models`, `directives`, `memories`) ensure the daemon is running before each call. This handles daemon crashes gracefully: - -```python -# ✅ GOOD - Uses API namespace (daemon restarts handled) -embedded.banks.create(bank_id="test", name="Test") - -# ❌ BAD - Direct client access (daemon crashes NOT handled) -client = embedded.client -client.create_bank(bank_id="test", name="Test") # Fails if daemon crashed -``` - ## Client Initialization ```python -from hindsight import HindsightClient +from hindsight_client import Hindsight -client = HindsightClient( +client = Hindsight( base_url="http://localhost:8888", # Hindsight API URL timeout=30.0, # Request timeout in seconds + # api_key="your-api-key", # Optional bearer token ) # Core operations client.retain(bank_id="test", content="Hello world") results = client.recall(bank_id="test", query="Hello") -# Organized API access (same as HindsightEmbedded) +# Organized API namespaces client.banks.create(bank_id="test", name="Test Bank") models = client.mental_models.list(bank_id="test") directives = client.directives.list(bank_id="test") memories = client.memories.list(bank_id="test") ``` -Both `HindsightClient` and `HindsightEmbedded` provide the same organized API namespaces (`banks`, `mental_models`, `directives`, `memories`) for consistent developer experience. - ## Core Operations ### Retain (Store Memory) diff --git a/uv.lock b/uv.lock index 0159c52c..98345ccd 100644 --- a/uv.lock +++ b/uv.lock @@ -1669,7 +1669,7 @@ requires-dist = [ { name = "flashrank", marker = "extra == 'local-ml'", specifier = ">=0.2.0" }, { name = "google-auth", specifier = ">=2.0.0" }, { name = "google-genai", specifier = ">=1.0.0" }, - { name = "greenlet", specifier = ">=3.2.4" }, + { name = "greenlet", specifier = ">=3.2.4,<3.4.0" }, { name = "hindsight-api-slim", extras = ["local-ml", "embedded-db"], marker = "extra == 'all'" }, { name = "httpx", specifier = ">=0.27.0" }, { name = "huggingface-hub", marker = "extra == 'local-llm'", specifier = ">=0.20.0" },