diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 2a835ade..94c574a9 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -6,8 +6,17 @@ on: - 'v*' jobs: - build-python-package: + build-python-packages: runs-on: ubuntu-latest + strategy: + matrix: + include: + - name: hindsight-all + path: hindsight + - name: hindsight-api + path: hindsight-api + - name: hindsight-client + path: hindsight-clients/python steps: - uses: actions/checkout@v4 @@ -22,15 +31,46 @@ jobs: with: python-version-file: ".python-version" - - name: Build hindsight package - working-directory: ./hindsight + - name: Build ${{ matrix.name }} package + working-directory: ./${{ matrix.path }} run: uv build - name: Upload artifacts uses: actions/upload-artifact@v4 with: - name: python-hindsight-dist - path: hindsight/dist/* + name: python-${{ matrix.name }}-dist + path: ${{ matrix.path }}/dist/* + retention-days: 30 + + build-typescript-client: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + + - name: Set up Node.js + uses: actions/setup-node@v4 + with: + node-version: '20' + registry-url: 'https://registry.npmjs.org' + + - name: Install dependencies + working-directory: ./hindsight-clients/typescript + run: npm ci + + - name: Build TypeScript client + working-directory: ./hindsight-clients/typescript + run: npm run build + + - name: Pack npm package + working-directory: ./hindsight-clients/typescript + run: npm pack + + - name: Upload artifacts + uses: actions/upload-artifact@v4 + with: + name: typescript-client-dist + path: hindsight-clients/typescript/*.tgz retention-days: 30 build-rust-cli: @@ -101,7 +141,14 @@ jobs: packages: write strategy: matrix: - component: [api, control-plane] + include: + # All images use the same Dockerfile with different --target + - target: api-only + image_name: hindsight-api + - target: cp-only + image_name: hindsight-control-plane + - target: standalone + image_name: hindsight steps: - uses: actions/checkout@v4 @@ -117,6 +164,9 @@ jobs: docker-images: true swap-storage: true + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 @@ -135,32 +185,21 @@ jobs: id: meta uses: docker/metadata-action@v5 with: - images: ghcr.io/${{ github.repository_owner }}/hindsight-${{ matrix.component }} + images: ghcr.io/${{ github.repository_owner }}/${{ matrix.image_name }} tags: | type=semver,pattern={{version}},value=${{ steps.get_version.outputs.VERSION }} type=semver,pattern={{major}}.{{minor}},value=${{ steps.get_version.outputs.VERSION }} type=semver,pattern={{major}},value=${{ steps.get_version.outputs.VERSION }} type=raw,value=latest - - name: Build and push Docker image (api) - if: matrix.component == 'api' + - name: Build and push Docker image uses: docker/build-push-action@v6 with: context: . - file: docker/api.Dockerfile - push: true - tags: ${{ steps.meta.outputs.tags }} - labels: ${{ steps.meta.outputs.labels }} - cache-from: type=gha - cache-to: type=gha,mode=max - - - name: Build and push Docker image (control-plane) - if: matrix.component == 'control-plane' - uses: docker/build-push-action@v6 - with: - context: . - file: docker/control-plane.Dockerfile + file: docker/standalone/Dockerfile + target: ${{ matrix.target }} push: true + platforms: linux/amd64,linux/arm64 tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} cache-from: type=gha @@ -192,9 +231,63 @@ jobs: path: helm-packages/*.tgz retention-days: 30 + publish-python-packages: + runs-on: ubuntu-latest + needs: [build-python-packages] + environment: pypi + strategy: + max-parallel: 1 + matrix: + include: + # Order matters: client and api first, then hindsight-all (which depends on them) + - name: hindsight-client + - name: hindsight-api + - name: hindsight-all + + steps: + - name: Download ${{ matrix.name }} + uses: actions/download-artifact@v4 + with: + name: python-${{ matrix.name }}-dist + path: ./dist + + - name: Publish ${{ matrix.name }} to PyPI + uses: pypa/gh-action-pypi-publish@release/v1 + with: + packages-dir: ./dist + skip-existing: true + + publish-npm-package: + runs-on: ubuntu-latest + needs: [build-typescript-client] + environment: npm + + steps: + - uses: actions/checkout@v4 + + - name: Set up Node.js + uses: actions/setup-node@v4 + with: + node-version: '20' + registry-url: 'https://registry.npmjs.org' + + - name: Install dependencies + working-directory: ./hindsight-clients/typescript + run: npm ci + + - name: Build TypeScript client + working-directory: ./hindsight-clients/typescript + run: npm run build + + - name: Publish to npm + working-directory: ./hindsight-clients/typescript + run: npm publish --access public + env: + NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }} + create-github-release: runs-on: ubuntu-latest - needs: [build-python-package, build-rust-cli, build-docker-images, package-helm-chart] + needs: [build-python-packages, build-typescript-client, build-rust-cli, build-docker-images, package-helm-chart] permissions: contents: write @@ -205,11 +298,29 @@ jobs: id: get_version run: echo "VERSION=${GITHUB_REF#refs/tags/v}" >> $GITHUB_OUTPUT - - name: Download Python package + - name: Download hindsight-all uses: actions/download-artifact@v4 with: - name: python-hindsight-dist - path: ./artifacts/python-hindsight-dist + name: python-hindsight-all-dist + path: ./artifacts/python-hindsight-all + + - name: Download hindsight-api + uses: actions/download-artifact@v4 + with: + name: python-hindsight-api-dist + path: ./artifacts/python-hindsight-api + + - name: Download hindsight-client + uses: actions/download-artifact@v4 + with: + name: python-hindsight-client-dist + path: ./artifacts/python-hindsight-client + + - name: Download TypeScript Client + uses: actions/download-artifact@v4 + with: + name: typescript-client-dist + path: ./artifacts/typescript-client - name: Download Rust CLI (Linux) uses: actions/download-artifact@v4 @@ -238,8 +349,12 @@ jobs: - name: Prepare release assets run: | mkdir -p release-assets - # Python package - cp artifacts/python-hindsight-dist/* release-assets/ + # Python packages + cp artifacts/python-hindsight-all/* release-assets/ + cp artifacts/python-hindsight-api/* release-assets/ + cp artifacts/python-hindsight-client/* release-assets/ + # TypeScript Client + cp artifacts/typescript-client/*.tgz release-assets/ # Rust CLI binaries cp artifacts/rust-cli-hindsight-linux-amd64/hindsight-linux-amd64 release-assets/ cp artifacts/rust-cli-hindsight-darwin-amd64/hindsight-darwin-amd64 release-assets/ @@ -250,37 +365,61 @@ jobs: - name: Generate release notes id: release_notes run: | - cat << EOF > release-notes.md + cat << 'EOF' > release-notes.md # Hindsight v${{ steps.get_version.outputs.VERSION }} + ## Quick Start + + ```bash + docker run -p 8888:8888 -p 9999:9999 \ + -e HINDSIGHT_API_LLM_PROVIDER=openai \ + -e HINDSIGHT_API_LLM_API_KEY=$OPENAI_API_KEY \ + -e HINDSIGHT_API_LLM_MODEL=gpt-4o-mini \ + ghcr.io/${{ github.repository_owner }}/hindsight:${{ steps.get_version.outputs.VERSION }} + ``` + ## πŸ“¦ Release Artifacts - ### Python Package - - \`hindsight-${{ steps.get_version.outputs.VERSION }}-py3-none-any.whl\` - - \`hindsight-${{ steps.get_version.outputs.VERSION }}.tar.gz\` + ### Docker Images + - `ghcr.io/${{ github.repository_owner }}/hindsight:${{ steps.get_version.outputs.VERSION }}` - **Standalone all-in-one** (recommended) + - `ghcr.io/${{ github.repository_owner }}/hindsight-api:${{ steps.get_version.outputs.VERSION }}` - API server only + - `ghcr.io/${{ github.repository_owner }}/hindsight-control-plane:${{ steps.get_version.outputs.VERSION }}` - Web UI only + + ### Python Packages + - `hindsight-all` - All-in-one package (includes API + client) + - `hindsight-api` - API server + - `hindsight-client` - Client library + + ### TypeScript/JavaScript + - `@hindsight/client` - TypeScript SDK ### CLI Binaries - - \`hindsight-linux-amd64\` - Linux x86_64 - - \`hindsight-darwin-amd64\` - macOS Intel - - \`hindsight-darwin-arm64\` - macOS Apple Silicon + - `hindsight-linux-amd64` - Linux x86_64 + - `hindsight-darwin-amd64` - macOS Intel + - `hindsight-darwin-arm64` - macOS Apple Silicon ### Helm Chart - - \`hindsight-${{ steps.get_version.outputs.VERSION }}.tgz\` - - ### Docker Images - Docker images are published to GitHub Container Registry: - - \`ghcr.io/${{ github.repository_owner }}/hindsight-api:${{ steps.get_version.outputs.VERSION }}\` - - \`ghcr.io/${{ github.repository_owner }}/hindsight-control-plane:${{ steps.get_version.outputs.VERSION }}\` + - `hindsight-${{ steps.get_version.outputs.VERSION }}.tgz` ## πŸš€ Installation - ### Python Package - \`\`\`bash - pip install hindsight==${{ steps.get_version.outputs.VERSION }} - \`\`\` + ### Python + ```bash + # All-in-one (recommended) + pip install hindsight-all==${{ steps.get_version.outputs.VERSION }} + + # Or install components separately + pip install hindsight-api==${{ steps.get_version.outputs.VERSION }} + pip install hindsight-client==${{ steps.get_version.outputs.VERSION }} + ``` + + ### TypeScript/JavaScript + ```bash + npm install @hindsight/client@${{ steps.get_version.outputs.VERSION }} + ``` ### CLI - \`\`\`bash + ```bash # macOS (Apple Silicon) curl -L https://github.com/${{ github.repository }}/releases/download/v${{ steps.get_version.outputs.VERSION }}/hindsight-darwin-arm64 -o hindsight chmod +x hindsight @@ -295,25 +434,12 @@ jobs: curl -L https://github.com/${{ github.repository }}/releases/download/v${{ steps.get_version.outputs.VERSION }}/hindsight-linux-amd64 -o hindsight chmod +x hindsight sudo mv hindsight /usr/local/bin/ - \`\`\` + ``` - ### Helm Chart - \`\`\`bash - helm install hindsight hindsight-${{ steps.get_version.outputs.VERSION }}.tgz - \`\`\` - - ### Docker - \`\`\`bash - # Pull API image - docker pull ghcr.io/${{ github.repository_owner }}/hindsight-api:${{ steps.get_version.outputs.VERSION }} - - # Pull Control Plane image - docker pull ghcr.io/${{ github.repository_owner }}/hindsight-control-plane:${{ steps.get_version.outputs.VERSION }} - - # Or use latest - docker pull ghcr.io/${{ github.repository_owner }}/hindsight-api:latest - docker pull ghcr.io/${{ github.repository_owner }}/hindsight-control-plane:latest - \`\`\` + ### Helm (Kubernetes) + ```bash + helm install hindsight oci://ghcr.io/${{ github.repository_owner }}/charts/hindsight --version ${{ steps.get_version.outputs.VERSION }} + ``` EOF cat release-notes.md @@ -333,9 +459,10 @@ jobs: echo "# Release v${{ steps.get_version.outputs.VERSION }} Published Successfully" >> $GITHUB_STEP_SUMMARY echo "" >> $GITHUB_STEP_SUMMARY echo "## πŸ“¦ Components" >> $GITHUB_STEP_SUMMARY - echo "- βœ… Python package (hindsight)" >> $GITHUB_STEP_SUMMARY + echo "- βœ… Python packages (hindsight-all, hindsight-api, hindsight-client)" >> $GITHUB_STEP_SUMMARY + echo "- βœ… TypeScript Client (@hindsight/client)" >> $GITHUB_STEP_SUMMARY echo "- βœ… Rust CLI (Linux amd64, macOS amd64, macOS arm64)" >> $GITHUB_STEP_SUMMARY - echo "- βœ… Docker images (API, Control Plane)" >> $GITHUB_STEP_SUMMARY + echo "- βœ… Docker images (standalone, API, Control Plane)" >> $GITHUB_STEP_SUMMARY echo "- βœ… Helm chart" >> $GITHUB_STEP_SUMMARY echo "" >> $GITHUB_STEP_SUMMARY echo "πŸŽ‰ Release is now available at: https://github.com/${{ github.repository }}/releases/tag/v${{ steps.get_version.outputs.VERSION }}" >> $GITHUB_STEP_SUMMARY diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 00000000..87f6770f --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,67 @@ +# Contributing to Hindsight + +Thanks for your interest in contributing to Hindsight! + +## Getting Started + +1. Fork and clone the repository +2. Install dependencies: + ```bash + cd hindsight-api && uv sync + ``` +3. Set up your environment: + ```bash + export OPENAI_API_KEY=your-key + ``` + +## Development + +### Running the API locally + +```bash +./scripts/dev/start-api.sh +``` + +### Running the Control Plane locally + +```bash +./scripts/dev/start-control-plane.sh +``` + +### Running the documentation locally + +```bash +./scripts/dev/start-docs.sh +``` + +### Running tests + +```bash +cd hindsight-api +uv run pytest tests/ +``` + +### Code style + +- Use Python type hints +- Follow existing code patterns +- Keep functions focused and well-named + +## Pull Requests + +1. Create a feature branch from `main` +2. Make your changes +3. Run tests to ensure nothing breaks +4. Submit a PR with a clear description of changes + +## Reporting Issues + +Open an issue on GitHub with: +- Clear description of the problem +- Steps to reproduce +- Expected vs actual behavior +- Environment details (OS, Python version) + +## Questions? + +Open a discussion on GitHub or reach out to the maintainers. diff --git a/README.md b/README.md index 723c54d9..2d5d3ff1 100644 --- a/README.md +++ b/README.md @@ -2,59 +2,96 @@ **Long-term memory for AI agents.** -AI assistants forget everything between sessions. Hindsight fixes that with a memory system that handles temporal reasoning, entity connections, and personality-aware responses. - ## Why Hindsight? -- **Temporal queries** β€” "What did Alice do last spring?" requires more than vector search -- **Entity connections** β€” Knowing "Alice works at Google" + "Google is in Mountain View" = "Alice works in Mountain View" -- **Agent opinions** β€” Agents form and recall beliefs with confidence scores -- **Personality** β€” Big Five traits influence how agents process and respond to information +AI assistants forget everything between sessions. Every conversation starts from zeroβ€”no context about who you are, what you've discussed, or what the memory bank has learned. This isn't just inconvenient; it fundamentally limits what AI memory banks can do. -## 60-seconds step +**The problem is harder than it looks:** + +- **Simple vector search isn't enough** β€” "What did Alice do last spring?" requires temporal reasoning, not just semantic similarity +- **Facts get disconnected** β€” Knowing "Alice works at Google" and "Google is in Mountain View" should let you answer "Where does Alice work?" even if you never stored that directly +- **Memory banks need opinions** β€” A coding assistant that remembers "the user prefers functional programming" should weigh that when making recommendations +- **Context matters** β€” The same information means different things to different memory banks with different personalities + +Hindsight solves these problems with a memory system designed specifically for AI memory banks. -### 1. Install the Hindsight All package (client + API) +## Quick Start + +### Option 1: Docker (recommended) + +Get the full experience with the API and Control Plane UI: + +```bash +export OPENAI_API_KEY=your-key +docker run -p 8888:8888 -p 9999:9999 \ + -e HINDSIGHT_API_LLM_PROVIDER=openai \ + -e HINDSIGHT_API_LLM_API_KEY=$OPENAI_API_KEY \ + -e HINDSIGHT_API_LLM_MODEL=gpt-4o-mini \ + vectorize/hindsight +``` + +- **API**: http://localhost:8888 +- **Control Plane UI**: http://localhost:9999 + +Then use the Python client: + +```bash +pip install hindsight-client +``` + +```python +from hindsight import HindsightClient + +client = HindsightClient(base_url="http://localhost:8888") + +# Store memories +client.retain(bank_id="my-agent", content="Alice works at Google as a software engineer") +client.retain(bank_id="my-agent", content="Alice mentioned she loves hiking in the mountains") + +# Query with temporal reasoning +results = client.recall(bank_id="my-agent", query="What does Alice do for work?") + +# Get a synthesized perspective +response = client.reflect(bank_id="my-agent", query="Tell me about Alice") +print(response.text) +``` + +### Option 2: Embedded (no docker/server required) + +For quick prototyping, run everything in-process: ```bash pip install hindsight-all +export OPENAI_API_KEY=your-key ``` -### 2. Import your OpenAI API key -```bash -export OPENAI_API_KEY=xx -``` - -### 3. Run embedded server and client - ```python import os from hindsight import HindsightServer, HindsightClient -with HindsightServer(llm_provider="openai", llm_model="gpt-5.1-mini", llm_api_key=os.environ["OPENAI_API_KEY"]) as server: +with HindsightServer(llm_provider="openai", llm_model="gpt-4o-mini", llm_api_key=os.environ["OPENAI_API_KEY"]) as server: client = HindsightClient(base_url=server.url) - # Retain memories - client.retain(bank_id="my-agent", content="Alice works at Google") - client.retain(bank_id="my-agent", content="Bob prefers Python over JavaScript") - - # Recall memories - client.recall(bank_id="my-agent", query="What does Alice do?") - - # Get memory perspective - client.reflect(bank_id="my-agent", query="Tell me about Alice") + client.retain(bank_id="my-user", content="User prefers functional programming") + response = client.reflect(bank_id="my-user", query="What coding style should I use?") + print(response.text) ``` ## Documentation -Full documentation: [hindsight-docs](./hindsight-docs) +Full documentation: [vectorize-io.github.io/hindsight](https://vectorize-io.github.io/hindsight) -- [Architecture](./hindsight-docs/docs/developer/architecture.md) β€” How ingestion, storage, and retrieval work -- [Python Client](./hindsight-docs/docs/sdks/python.md) β€” Full API reference -- [API Reference](./hindsight-docs/docs/api-reference/index.md) β€” REST API endpoints -- [Personality](./hindsight-docs/docs/developer/personality.md) β€” Big Five traits and opinion formation +- [Architecture](https://vectorize-io.github.io/hindsight/developer/architecture) β€” How ingestion, storage, and retrieval work +- [Python Client](https://vectorize-io.github.io/hindsight/sdks/python) β€” Full API reference +- [API Reference](https://vectorize-io.github.io/hindsight/api-reference) β€” REST API endpoints +- [Personality](https://vectorize-io.github.io/hindsight/developer/personality) β€” Big Five traits and opinion formation + +## Contributing + +We welcome contributions! See [CONTRIBUTING.md](./CONTRIBUTING.md) for guidelines. ## License diff --git a/cookbook/README.md b/cookbook/README.md new file mode 100644 index 00000000..0329635f --- /dev/null +++ b/cookbook/README.md @@ -0,0 +1,11 @@ +# Hindsight Cookbook + +For the cookbook with detailed examples, tutorials, and integrations, visit: + +**[https://github.com/vectorize-io/hindsight-cookbook](https://github.com/vectorize-io/hindsight-cookbook)** + +The cookbook repository includes: +- Integration examples with popular frameworks +- Real-world use cases and patterns +- Step-by-step tutorials +- Best practices and tips diff --git a/docker/services/README.md b/docker/services/README.md deleted file mode 100644 index 5db27921..00000000 --- a/docker/services/README.md +++ /dev/null @@ -1,59 +0,0 @@ -# Distributed Hindsight Setup - -Run API and Control Plane as separate containers. - -## Start - -```bash -cd services -docker-compose up -``` - -Access: -- **Control Plane**: http://localhost:3000 -- **API**: http://localhost:8888 - -## What's Running - -Two separate containers: -- `api` - Hindsight API with embedded pg0 database -- `control-plane` - Web UI - -## Build Images - -```bash -./build-all.sh -``` - -Creates: -- `hindsight/api:latest` -- `hindsight/control-plane:latest` - -## Configuration - -The API uses embedded pg0 by default. Database files are stored in the `api_data` volume. - -To use an external PostgreSQL database, add to `docker-compose.yml`: - -```yaml -services: - api: - environment: - HINDSIGHT_API_DATABASE_URL: postgresql://user:pass@host:5432/db -``` - -## Data Persistence - -```bash -docker-compose down -v # Remove volumes -``` - -## Why Use This? - -The distributed setup is useful when you want to: -- Scale API and UI independently -- Use an external database in production -- Deploy to Kubernetes/orchestration -- Run UI on different infrastructure - -For simple deployments, use the main `docker-compose.yml` (standalone all-in-one). diff --git a/docker/services/api.Dockerfile b/docker/services/api.Dockerfile deleted file mode 100644 index 1c89876c..00000000 --- a/docker/services/api.Dockerfile +++ /dev/null @@ -1,33 +0,0 @@ -# Dockerfile for Hindsight API (standalone) -FROM python:3.11-slim - -WORKDIR /app - -# Install system dependencies and uv -RUN apt-get update && apt-get install -y \ - gcc \ - g++ \ - && rm -rf /var/lib/apt/lists/* \ - && pip install --no-cache-dir uv - -# Copy dependency files and README (required by pyproject.toml) -COPY hindsight-api/pyproject.toml ./ -COPY hindsight-api/README.md ./ - -# Sync dependencies (creates lock file if needed) -RUN uv sync - -# Copy source code -COPY hindsight-api/hindsight_api ./hindsight_api - -# Expose API port -EXPOSE 8888 - -# Set environment variables -ENV HINDSIGHT_API_HOST=0.0.0.0 -ENV HINDSIGHT_API_PORT=8888 -ENV HINDSIGHT_API_LOG_LEVEL=info -ENV PATH="/app/.venv/bin:$PATH" - -# Run the API server -CMD ["python", "-m", "hindsight_api.web.server"] diff --git a/docker/services/build-all.sh b/docker/services/build-all.sh deleted file mode 100755 index 137c4a2b..00000000 --- a/docker/services/build-all.sh +++ /dev/null @@ -1,24 +0,0 @@ -#!/bin/bash -set -e - -echo "Building Hindsight service images..." - -cd "$(dirname "$0")/../.." - -echo "" -echo "Building hindsight-api..." -docker build -f docker/services/api.Dockerfile -t hindsight/api:latest . - -echo "" -echo "Building hindsight-control-plane..." -docker build -f docker/services/control-plane.Dockerfile -t hindsight/control-plane:latest . - -echo "" -echo "βœ… All service images built successfully!" -echo "" -echo "Available images:" -echo " - hindsight/api:latest" -echo " - hindsight/control-plane:latest" -echo "" -echo "To start all services:" -echo " cd docker && docker-compose up" diff --git a/docker/services/control-plane.Dockerfile b/docker/services/control-plane.Dockerfile deleted file mode 100644 index 5ec30075..00000000 --- a/docker/services/control-plane.Dockerfile +++ /dev/null @@ -1,65 +0,0 @@ -# Dockerfile for Hindsight Control Plane (standalone) -FROM node:20-alpine AS sdk-builder - -WORKDIR /app/sdk - -# Build TypeScript SDK -COPY hindsight-clients/typescript/package*.json ./ -RUN npm ci - -COPY hindsight-clients/typescript/ ./ -RUN npm run build - -# Build Control Plane -FROM node:20-alpine AS builder - -WORKDIR /app - -# Copy built SDK -COPY --from=sdk-builder /app/sdk /app/sdk - -# Install Control Plane dependencies -COPY hindsight-control-plane/package*.json ./ -RUN npm ci - -# Copy Control Plane source -COPY hindsight-control-plane/ ./ - -# Link SDK for build -RUN cd /app/sdk && npm link && cd /app && npm link @hindsight/client - -# Build the Next.js app -RUN npm run build - -# Create public directory if it doesn't exist -RUN mkdir -p public - -# Production image -FROM node:20-alpine - -WORKDIR /app - -# Copy built SDK -COPY --from=sdk-builder /app/sdk /app/sdk - -# Copy package files and install production dependencies only -COPY hindsight-control-plane/package*.json ./ -RUN npm ci --omit=dev - -# Link SDK for runtime -RUN cd /app/sdk && npm link && cd /app && npm link @hindsight/client - -# Copy built app from builder -COPY --from=builder /app/.next ./.next -COPY --from=builder /app/public ./public -COPY --from=builder /app/next.config.ts ./next.config.ts - -# Expose control plane port -EXPOSE 3000 - -# Set environment variables -ENV NODE_ENV=production -ENV HINDSIGHT_CP_DATAPLANE_API_URL=http://localhost:8888 - -# Run the Next.js server -CMD ["npm", "start"] diff --git a/docker/services/docker-compose.yml b/docker/services/docker-compose.yml deleted file mode 100644 index f81e7e9d..00000000 --- a/docker/services/docker-compose.yml +++ /dev/null @@ -1,42 +0,0 @@ -services: - api: - build: - context: ../.. - dockerfile: docker/services/api.Dockerfile - ports: - - "8888:8888" - environment: - # Pass through all HINDSIGHT_* environment variables - HINDSIGHT_API_LLM_API_KEY: ${HINDSIGHT_API_LLM_API_KEY:-} - HINDSIGHT_API_LLM_MODEL: ${HINDSIGHT_API_LLM_MODEL:-} - HINDSIGHT_API_LLM_BASE_URL: ${HINDSIGHT_API_LLM_BASE_URL:-} - HINDSIGHT_API_HOST: ${HINDSIGHT_API_HOST:-0.0.0.0} - HINDSIGHT_API_PORT: ${HINDSIGHT_API_PORT:-8888} - HINDSIGHT_API_LOG_LEVEL: ${HINDSIGHT_API_LOG_LEVEL:-info} - HINDSIGHT_API_DATABASE_URL: ${HINDSIGHT_API_DATABASE_URL:-} - volumes: - - api_data:/app/data - networks: - - hindsight - restart: unless-stopped - - control-plane: - build: - context: ../.. - dockerfile: docker/services/control-plane.Dockerfile - ports: - - "3000:3000" - environment: - NODE_ENV: production - HINDSIGHT_CP_DATAPLANE_API_URL: http://api:8888 - depends_on: - - api - networks: - - hindsight - restart: unless-stopped - -volumes: - api_data: - -networks: - hindsight: diff --git a/docker/standalone/Dockerfile b/docker/standalone/Dockerfile index ff247315..f78cd263 100644 --- a/docker/standalone/Dockerfile +++ b/docker/standalone/Dockerfile @@ -1,6 +1,25 @@ -# Standalone All-in-One Hindsight Image -# API with embedded pg0 + Control Plane -FROM python:3.11-slim AS api-base +# Hindsight Docker Image +# Supports building API-only, Control Plane-only, or both +# +# Build args: +# INCLUDE_API=true/false - Include API (default: true) +# INCLUDE_CP=true/false - Include Control Plane (default: true) +# +# Examples: +# docker build -t hindsight . # Both (standalone) +# docker build -t hindsight-api --build-arg INCLUDE_CP=false . # API only +# docker build -t hindsight-cp --build-arg INCLUDE_API=false . # Control Plane only + +ARG INCLUDE_API=true +ARG INCLUDE_CP=true + +# ============================================================================= +# Stage: API Builder +# ============================================================================= +FROM python:3.11-slim AS api-builder + +ARG INCLUDE_API +RUN if [ "$INCLUDE_API" != "true" ]; then echo "Skipping API build" && exit 0; fi WORKDIR /app @@ -21,12 +40,18 @@ WORKDIR /app/api # Sync dependencies (will create lock file if needed) RUN uv sync -# Copy source code +# Copy source code and alembic migrations COPY hindsight-api/hindsight_api ./hindsight_api +COPY hindsight-api/alembic ./alembic -# Build TypeScript SDK +# ============================================================================= +# Stage: SDK Builder (needed for Control Plane) +# ============================================================================= FROM node:20-alpine AS sdk-builder +ARG INCLUDE_CP +RUN if [ "$INCLUDE_CP" != "true" ]; then echo "Skipping SDK build" && exit 0; fi + WORKDIR /app/sdk COPY hindsight-clients/typescript/package*.json ./ @@ -35,9 +60,14 @@ RUN npm ci COPY hindsight-clients/typescript/ ./ RUN npm run build -# Build Control Plane +# ============================================================================= +# Stage: Control Plane Builder +# ============================================================================= FROM node:20-alpine AS cp-builder +ARG INCLUDE_CP +RUN if [ "$INCLUDE_CP" != "true" ]; then echo "Skipping CP build" && exit 0; fi + WORKDIR /app # Copy built SDK @@ -59,8 +89,120 @@ RUN npm run build # Create public directory if it doesn't exist RUN mkdir -p public -# Final standalone image -FROM python:3.11-slim +# ============================================================================= +# Stage: Final Image - API Only +# ============================================================================= +FROM python:3.11-slim AS api-only + +WORKDIR /app + +# Install pg0 dependencies +RUN apt-get update && apt-get install -y \ + curl \ + libxml2 \ + libssl3 \ + libgssapi-krb5-2 \ + libossp-uuid16 \ + && apt-get install -y libicu72 || apt-get install -y libicu74 || apt-get install -y libicu* \ + && rm -rf /var/lib/apt/lists/* \ + && pip install --no-cache-dir uv + +# Create non-root user (PostgreSQL cannot run as root) +RUN useradd -m -s /bin/bash hindsight + +# Copy API with virtual environment from builder +COPY --from=api-builder /app/api /app/api + +# Copy startup script +COPY docker/standalone/start-all.sh /app/start-all.sh +RUN chmod +x /app/start-all.sh + +# Create data directory for pg0 and set ownership +RUN mkdir -p /app/data && chown -R hindsight:hindsight /app + +# Switch to non-root user +USER hindsight + +# Set PATH for hindsight user +ENV PATH="/home/hindsight/.hindsight/bin:/app/api/.venv/bin:${PATH}" + +# Install pg0 binary +RUN mkdir -p /home/hindsight/.hindsight/bin && \ + ARCH=$(uname -m) && \ + if [ "$ARCH" = "aarch64" ] || [ "$ARCH" = "arm64" ]; then \ + PG0_BINARY="pg0-linux-aarch64-gnu"; \ + elif [ "$ARCH" = "x86_64" ]; then \ + PG0_BINARY="pg0-linux-x86_64-gnu"; \ + else \ + echo "Unsupported architecture: $ARCH" && exit 1; \ + fi && \ + echo "Installing pg0 binary: $PG0_BINARY" && \ + for i in 1 2 3 4 5; do \ + curl -fsSL -o /home/hindsight/.hindsight/bin/pg0 \ + "https://github.com/vectorize-io/pg0/releases/latest/download/$PG0_BINARY" && \ + chmod +x /home/hindsight/.hindsight/bin/pg0 && \ + break || (echo "Retry $i failed, waiting..." && sleep 10); \ + done && \ + /home/hindsight/.hindsight/bin/pg0 --version + +# Pre-download PostgreSQL binaries +ENV PG0_HOME=/home/hindsight/.pg0-cache +RUN pg0 start --help && \ + (pg0 start --name hindsight --port 5555 --username hindsight --password hindsight --database hindsight && \ + sleep 2 && \ + pg0 stop --name hindsight && \ + echo "PostgreSQL pre-cached to $PG0_HOME") || echo "Pre-download skipped" + +ENV PG0_HOME=/home/hindsight/.pg0 + +EXPOSE 8888 + +ENV HINDSIGHT_API_HOST=0.0.0.0 +ENV HINDSIGHT_API_PORT=8888 +ENV HINDSIGHT_API_LOG_LEVEL=info +ENV HINDSIGHT_ENABLE_API=true +ENV HINDSIGHT_ENABLE_CP=false + +CMD ["/app/start-all.sh"] + +# ============================================================================= +# Stage: Final Image - Control Plane Only +# ============================================================================= +FROM node:20-alpine AS cp-only + +WORKDIR /app + +# Copy built SDK +COPY --from=sdk-builder /app/sdk /app/sdk + +# Copy Control Plane standalone build +WORKDIR /app/control-plane +COPY --from=cp-builder /app/.next/standalone ./ +COPY --from=cp-builder /app/.next/static ./.next/static +COPY --from=cp-builder /app/public ./public + +WORKDIR /app + +# Copy startup script +COPY docker/standalone/start-all.sh /app/start-all.sh +RUN chmod +x /app/start-all.sh + +# Install curl for health checks +RUN apk add --no-cache curl bash + +EXPOSE 9999 + +ENV NODE_ENV=production +ENV HINDSIGHT_CP_DATAPLANE_API_URL=http://localhost:8888 +ENV HINDSIGHT_ENABLE_API=false +ENV HINDSIGHT_ENABLE_CP=true + +CMD ["/app/start-all.sh"] + +# ============================================================================= +# Stage: Final Image - Standalone (both API and Control Plane) +# ============================================================================= +FROM python:3.11-slim AS standalone WORKDIR /app @@ -70,6 +212,7 @@ RUN apt-get update && apt-get install -y \ libxml2 \ libssl3 \ libgssapi-krb5-2 \ + libossp-uuid16 \ && apt-get install -y libicu72 || apt-get install -y libicu74 || apt-get install -y libicu* \ && curl -fsSL https://deb.nodesource.com/setup_20.x | bash - \ && apt-get install -y nodejs \ @@ -80,22 +223,16 @@ RUN apt-get update && apt-get install -y \ RUN useradd -m -s /bin/bash hindsight # Copy API with virtual environment from builder -COPY --from=api-base /app/api /app/api +COPY --from=api-builder /app/api /app/api # Copy built SDK COPY --from=sdk-builder /app/sdk /app/sdk -# Copy Control Plane +# Copy Control Plane standalone build WORKDIR /app/control-plane -COPY --from=cp-builder /app/package*.json ./ -RUN npm ci --omit=dev - -# Link SDK for runtime -RUN cd /app/sdk && npm link && cd /app/control-plane && npm link @hindsight/client - -COPY --from=cp-builder /app/.next ./.next +COPY --from=cp-builder /app/.next/standalone ./ +COPY --from=cp-builder /app/.next/static ./.next/static COPY --from=cp-builder /app/public ./public -COPY --from=cp-builder /app/next.config.ts ./next.config.ts WORKDIR /app @@ -109,24 +246,57 @@ RUN mkdir -p /app/data && chown -R hindsight:hindsight /app # Switch to non-root user USER hindsight -# Install pg0 -RUN curl -fsSL https://raw.githubusercontent.com/vectorize-io/pg0/main/install.sh | bash +# Set PATH for hindsight user +ENV PATH="/home/hindsight/.hindsight/bin:/app/api/.venv/bin:${PATH}" -# Start pg0 once to verify it works and pre-download PostgreSQL libraries -RUN pg0 --help && \ - pg0 start --wait && \ - pg0 stop +# Install pg0 binary +RUN mkdir -p /home/hindsight/.hindsight/bin && \ + ARCH=$(uname -m) && \ + if [ "$ARCH" = "aarch64" ] || [ "$ARCH" = "arm64" ]; then \ + PG0_BINARY="pg0-linux-aarch64-gnu"; \ + elif [ "$ARCH" = "x86_64" ]; then \ + PG0_BINARY="pg0-linux-x86_64-gnu"; \ + else \ + echo "Unsupported architecture: $ARCH" && exit 1; \ + fi && \ + echo "Installing pg0 binary: $PG0_BINARY" && \ + for i in 1 2 3 4 5; do \ + curl -fsSL -o /home/hindsight/.hindsight/bin/pg0 \ + "https://github.com/vectorize-io/pg0/releases/latest/download/$PG0_BINARY" && \ + chmod +x /home/hindsight/.hindsight/bin/pg0 && \ + break || (echo "Retry $i failed, waiting..." && sleep 10); \ + done && \ + /home/hindsight/.hindsight/bin/pg0 --version -# Expose ports -EXPOSE 8888 3000 +# Pre-download PostgreSQL binaries +ENV PG0_HOME=/home/hindsight/.pg0-cache +RUN pg0 start --help && \ + (pg0 start --name hindsight --port 5555 --username hindsight --password hindsight --database hindsight && \ + sleep 2 && \ + pg0 stop --name hindsight && \ + echo "PostgreSQL pre-cached to $PG0_HOME") || echo "Pre-download skipped" + +ENV PG0_HOME=/home/hindsight/.pg0 + +EXPOSE 8888 9999 -# Environment variables ENV HINDSIGHT_API_HOST=0.0.0.0 ENV HINDSIGHT_API_PORT=8888 ENV HINDSIGHT_API_LOG_LEVEL=info ENV NODE_ENV=production ENV HINDSIGHT_CP_DATAPLANE_API_URL=http://localhost:8888 -ENV PATH="/home/hindsight/.local/bin:/app/api/.venv/bin:${PATH}" +ENV HINDSIGHT_ENABLE_API=true +ENV HINDSIGHT_ENABLE_CP=true -# Run startup script CMD ["/app/start-all.sh"] + +# ============================================================================= +# Default target selection based on build args +# ============================================================================= +FROM standalone AS default-both +FROM api-only AS default-api +FROM cp-only AS default-cp + +# This selects the final stage based on INCLUDE_API and INCLUDE_CP +# Use --target to override: docker build --target api-only . +FROM standalone diff --git a/docker/standalone/docker-compose.yml b/docker/standalone/docker-compose.yml index 4578c939..02e5c5fd 100644 --- a/docker/standalone/docker-compose.yml +++ b/docker/standalone/docker-compose.yml @@ -1,26 +1,24 @@ services: hindsight: + image: hindsight build: context: ../.. dockerfile: docker/standalone/Dockerfile - platform: linux/amd64 + env_file: + - ../../.env ports: - - "3000:3000" + - "9999:9999" - "8888:8888" environment: - # Pass through all HINDSIGHT_* environment variables from host - HINDSIGHT_API_LLM_API_KEY: ${HINDSIGHT_API_LLM_API_KEY:-} - HINDSIGHT_API_LLM_MODEL: ${HINDSIGHT_API_LLM_MODEL:-} - HINDSIGHT_API_LLM_BASE_URL: ${HINDSIGHT_API_LLM_BASE_URL:-} - HINDSIGHT_API_LLM_PROVIDER: ${HINDSIGHT_API_LLM_PROVIDER:-} + # These override env_file values only when set in host shell + # Default values are applied only when not set in env_file or host HINDSIGHT_API_HOST: ${HINDSIGHT_API_HOST:-0.0.0.0} HINDSIGHT_API_PORT: ${HINDSIGHT_API_PORT:-8888} HINDSIGHT_API_LOG_LEVEL: ${HINDSIGHT_API_LOG_LEVEL:-info} # HINDSIGHT_API_DATABASE_URL can be set if you want to use an external database # If not set, embedded pg0 will be used automatically - # Add any other HINDSIGHT_* vars you need here volumes: - - hindsight_data:/app/data + - hindsight_data:/home/hindsight/.pg0 restart: unless-stopped volumes: diff --git a/docker/standalone/start-all.sh b/docker/standalone/start-all.sh index 5acfdbd4..43336592 100755 --- a/docker/standalone/start-all.sh +++ b/docker/standalone/start-all.sh @@ -4,36 +4,75 @@ set -e echo "πŸš€ Starting Hindsight..." echo "" -# Start API (with embedded pg0) -echo "⚑ Starting Hindsight API (with embedded database)..." -cd /app/api -python -m hindsight_api.web.server & -API_PID=$! +# Service flags (default to true if not set) +ENABLE_API="${HINDSIGHT_ENABLE_API:-true}" +ENABLE_CP="${HINDSIGHT_ENABLE_CP:-true}" -# Wait for API to be ready -echo "⏳ Waiting for API..." -for i in {1..30}; do - if curl -sf http://localhost:8888/health &>/dev/null || curl -sf http://localhost:8888/docs &>/dev/null; then - echo "βœ… API is ready" - break +# Copy pre-cached PostgreSQL data if runtime directory is empty (first run with volume) +if [ "$ENABLE_API" = "true" ]; then + PG0_CACHE="/home/hindsight/.pg0-cache" + PG0_HOME="/home/hindsight/.pg0" + if [ -d "$PG0_CACHE" ] && [ "$(ls -A $PG0_CACHE 2>/dev/null)" ]; then + if [ ! "$(ls -A $PG0_HOME 2>/dev/null)" ]; then + echo "πŸ“¦ Copying pre-cached PostgreSQL data..." + cp -r "$PG0_CACHE"/* "$PG0_HOME"/ 2>/dev/null || true + fi fi - sleep 1 -done +fi -# Start Control Plane -echo "πŸŽ›οΈ Starting Control Plane..." -cd /app/control-plane -node .next/standalone/server.js & -CP_PID=$! +# Track PIDs for wait +PIDS=() +# Start API if enabled +if [ "$ENABLE_API" = "true" ]; then + cd /app/api + python -m hindsight_api.web.server 2>&1 | sed -u 's/^/[api] /' & + API_PID=$! + PIDS+=($API_PID) + + # Wait for API to be ready + echo "⏳ Waiting for API..." + for i in {1..60}; do + if curl -sf http://localhost:8888/health &>/dev/null; then + echo "βœ… API is ready" + break + fi + sleep 1 + done +else + echo "⏭️ API disabled (HINDSIGHT_ENABLE_API=false)" +fi + +# Start Control Plane if enabled +if [ "$ENABLE_CP" = "true" ]; then + echo "πŸŽ›οΈ Starting Control Plane..." + cd /app/control-plane + PORT=9999 node server.js 2>&1 | grep -v -E "^[[:space:]]*(β–²|βœ“|-|$)" | sed -u 's/^/[control-plane] /' & + CP_PID=$! + PIDS+=($CP_PID) +else + echo "⏭️ Control Plane disabled (HINDSIGHT_ENABLE_CP=false)" +fi + +# Print status echo "" echo "βœ… Hindsight is running!" echo "" echo "πŸ“ Access:" -echo " Control Plane: http://localhost:3000" -echo " API: http://localhost:8888" +if [ "$ENABLE_CP" = "true" ]; then + echo " Control Plane: http://localhost:9999" +fi +if [ "$ENABLE_API" = "true" ]; then + echo " API: http://localhost:8888" +fi echo "" +# Check if any services are running +if [ ${#PIDS[@]} -eq 0 ]; then + echo "❌ No services enabled! Set HINDSIGHT_ENABLE_API=true or HINDSIGHT_ENABLE_CP=true" + exit 1 +fi + # Wait for any process to exit wait -n diff --git a/hindsight-api/hindsight_api/api/http.py b/hindsight-api/hindsight_api/api/http.py index 95a15af2..8ba35431 100644 --- a/hindsight-api/hindsight_api/api/http.py +++ b/hindsight-api/hindsight_api/api/http.py @@ -797,6 +797,24 @@ def create_app(memory: MemoryEngine, run_migrations: bool = True, initialize_mem def _register_routes(app: FastAPI): """Register all API routes on the given app instance.""" + @app.get( + "/health", + summary="Health check endpoint", + description="Checks the health of the API and database connection", + tags=["Monitoring"] + ) + async def health_endpoint(): + """ + Health check endpoint that verifies database connectivity. + + Returns 200 if healthy, 503 if unhealthy. + """ + from fastapi.responses import JSONResponse + + health = await app.state.memory.health_check() + status_code = 200 if health.get("status") == "healthy" else 503 + return JSONResponse(content=health, status_code=status_code) + @app.get( "/metrics", summary="Prometheus metrics endpoint", diff --git a/hindsight-api/hindsight_api/api/mcp.py b/hindsight-api/hindsight_api/api/mcp.py index 5c070afa..3b1ebac5 100644 --- a/hindsight-api/hindsight_api/api/mcp.py +++ b/hindsight-api/hindsight_api/api/mcp.py @@ -2,11 +2,19 @@ import json import logging +import os from fastmcp import FastMCP from hindsight_api import MemoryEngine -logging.basicConfig(level=logging.INFO) +# Configure logging from HINDSIGHT_API_LOG_LEVEL environment variable +_log_level_str = os.environ.get("HINDSIGHT_API_LOG_LEVEL", "info").lower() +_log_level_map = {"critical": logging.CRITICAL, "error": logging.ERROR, "warning": logging.WARNING, + "info": logging.INFO, "debug": logging.DEBUG, "trace": logging.DEBUG} +logging.basicConfig( + level=_log_level_map.get(_log_level_str, logging.INFO), + format="%(asctime)s - %(levelname)s - %(name)s - %(message)s" +) logger = logging.getLogger(__name__) diff --git a/hindsight-api/hindsight_api/engine/cross_encoder.py b/hindsight-api/hindsight_api/engine/cross_encoder.py index d54007f9..ba479950 100644 --- a/hindsight-api/hindsight_api/engine/cross_encoder.py +++ b/hindsight-api/hindsight_api/engine/cross_encoder.py @@ -10,13 +10,23 @@ import logging logger = logging.getLogger(__name__) -class CrossEncoderReranker(ABC): +class CrossEncoderModel(ABC): """ Abstract base class for cross-encoder reranking. Cross-encoders take query-document pairs and return relevance scores. """ + @abstractmethod + def load(self) -> None: + """ + Load the cross-encoder model. + + This should be called during initialization to load the model + and avoid cold start latency on first predict() call. + """ + pass + @abstractmethod def predict(self, pairs: List[Tuple[str, str]]) -> List[float]: """ @@ -31,12 +41,11 @@ class CrossEncoderReranker(ABC): pass -class SentenceTransformersCrossEncoder(CrossEncoderReranker): +class SentenceTransformersCrossEncoder(CrossEncoderModel): """ Cross-encoder implementation using SentenceTransformers. - Uses lazy import so sentence-transformers is not required if another - reranking backend is used. + Call load() during initialization to load the model and avoid cold starts. Default model is cross-encoder/ms-marco-MiniLM-L-6-v2: - Fast inference (~80ms for 100 pairs on CPU) @@ -46,13 +55,19 @@ class SentenceTransformersCrossEncoder(CrossEncoderReranker): def __init__(self, model_name: str = "cross-encoder/ms-marco-MiniLM-L-6-v2"): """ - Initialize SentenceTransformers cross-encoder and load model. + Initialize SentenceTransformers cross-encoder. Args: model_name: Name of the CrossEncoder model to use. Default: cross-encoder/ms-marco-MiniLM-L-6-v2 """ self.model_name = model_name + self._model = None + + def load(self) -> None: + """Load the cross-encoder model.""" + if self._model is not None: + return try: from sentence_transformers import CrossEncoder @@ -76,5 +91,7 @@ class SentenceTransformersCrossEncoder(CrossEncoderReranker): Returns: List of relevance scores (raw logits from the model) """ - scores = self._model.predict(pairs) + if self._model is None: + self.load() + scores = self._model.predict(pairs, show_progress_bar=False) return scores.tolist() if hasattr(scores, 'tolist') else list(scores) diff --git a/hindsight-api/hindsight_api/engine/embeddings.py b/hindsight-api/hindsight_api/engine/embeddings.py index 33cd1da6..8391384b 100644 --- a/hindsight-api/hindsight_api/engine/embeddings.py +++ b/hindsight-api/hindsight_api/engine/embeddings.py @@ -24,6 +24,16 @@ class Embeddings(ABC): the database schema. """ + @abstractmethod + def load(self) -> None: + """ + Load the embedding model. + + This should be called during initialization to load the model + and avoid cold start latency on first encode() call. + """ + pass + @abstractmethod def encode(self, texts: List[str]) -> List[List[float]]: """ @@ -42,8 +52,7 @@ class SentenceTransformersEmbeddings(Embeddings): """ Embeddings implementation using SentenceTransformers. - Uses lazy import so sentence-transformers is not required if another - embedding backend is used. + Call load() during initialization to load the model and avoid cold starts. Default model is BAAI/bge-small-en-v1.5 which produces 384-dimensional embeddings matching the database schema. @@ -60,32 +69,33 @@ class SentenceTransformersEmbeddings(Embeddings): """ self.model_name = model_name self._model = None - self._load_model() - def _load_model(self): - """Lazy load and validate the SentenceTransformer model.""" - if self._model is None: - try: - from sentence_transformers import SentenceTransformer - except ImportError: - raise ImportError( - "sentence-transformers is required for SentenceTransformersEmbeddings. " - "Install it with: pip install sentence-transformers" - ) + def load(self) -> None: + """Load the embedding model.""" + if self._model is not None: + return - logger.info(f"Loading embedding model: {self.model_name}...") - self._model = SentenceTransformer(self.model_name) + try: + from sentence_transformers import SentenceTransformer + except ImportError: + raise ImportError( + "sentence-transformers is required for SentenceTransformersEmbeddings. " + "Install it with: pip install sentence-transformers" + ) - # Validate dimension matches database schema - model_dim = self._model.get_sentence_embedding_dimension() - if model_dim != EMBEDDING_DIMENSION: - raise ValueError( - f"Model {self.model_name} produces {model_dim}-dimensional embeddings, " - f"but database schema requires {EMBEDDING_DIMENSION} dimensions. " - f"Use a model that produces {EMBEDDING_DIMENSION}-dimensional embeddings." - ) + logger.info(f"Loading embedding model: {self.model_name}...") + self._model = SentenceTransformer(self.model_name) - logger.info(f"Model loaded (embedding dim: {model_dim})") + # Validate dimension matches database schema + model_dim = self._model.get_sentence_embedding_dimension() + if model_dim != EMBEDDING_DIMENSION: + raise ValueError( + f"Model {self.model_name} produces {model_dim}-dimensional embeddings, " + f"but database schema requires {EMBEDDING_DIMENSION} dimensions. " + f"Use a model that produces {EMBEDDING_DIMENSION}-dimensional embeddings." + ) + + logger.info(f"Model loaded (embedding dim: {model_dim})") def encode(self, texts: List[str]) -> List[List[float]]: """ @@ -97,5 +107,7 @@ class SentenceTransformersEmbeddings(Embeddings): Returns: List of 384-dimensional embedding vectors """ + if self._model is None: + self.load() embeddings = self._model.encode(texts, convert_to_numpy=True, show_progress_bar=False) return [emb.tolist() for emb in embeddings] diff --git a/hindsight-api/hindsight_api/engine/memory_engine.py b/hindsight-api/hindsight_api/engine/memory_engine.py index 12758431..c344e558 100644 --- a/hindsight-api/hindsight_api/engine/memory_engine.py +++ b/hindsight-api/hindsight_api/engine/memory_engine.py @@ -15,7 +15,7 @@ from typing import Any, Dict, List, Optional, Tuple, Union, TypedDict import asyncpg import asyncio from .embeddings import Embeddings, SentenceTransformersEmbeddings -from .cross_encoder import CrossEncoderReranker as CrossEncoderModel +from .cross_encoder import CrossEncoderModel import time import numpy as np import uuid @@ -362,15 +362,53 @@ class MemoryEngine: logger.error(f"Failed to mark operation as failed {operation_id}: {e}") async def initialize(self): - """Initialize the connection pool and background workers.""" + """Initialize the connection pool, models, and background workers. + + Loads models (embeddings, cross-encoder) in parallel with pg0 startup + for faster overall initialization. + """ if self._initialized: return - # Start pg0 embedded PostgreSQL if configured - if self._use_pg0: - self._pg0 = EmbeddedPostgres() - self.db_url = await self._pg0.ensure_running() - logger.info(f"Connecting to PostGre instance at {self.db_url}") + import concurrent.futures + + # Run model loading in thread pool (CPU-bound) in parallel with pg0 startup + loop = asyncio.get_event_loop() + + async def start_pg0(): + """Start pg0 if configured.""" + if self._use_pg0: + self._pg0 = EmbeddedPostgres() + self.db_url = await self._pg0.ensure_running() + + def load_embeddings(): + """Load embedding model (CPU-bound).""" + self.embeddings.load() + + def load_cross_encoder(): + """Load cross-encoder model (CPU-bound).""" + self._cross_encoder_reranker.cross_encoder.load() + + def load_query_analyzer(): + """Load query analyzer model (CPU-bound).""" + self.query_analyzer.load() + + # Run pg0 and all model loads in parallel + # pg0 is async (IO-bound), models are sync (CPU-bound in thread pool) + # Use 3 workers to load all models concurrently + with concurrent.futures.ThreadPoolExecutor(max_workers=3) as executor: + # Start all tasks + pg0_task = asyncio.create_task(start_pg0()) + embeddings_future = loop.run_in_executor(executor, load_embeddings) + cross_encoder_future = loop.run_in_executor(executor, load_cross_encoder) + query_analyzer_future = loop.run_in_executor(executor, load_query_analyzer) + + # Wait for all to complete + await asyncio.gather( + pg0_task, embeddings_future, cross_encoder_future, query_analyzer_future + ) + + logger.info(f"Connecting to PostgreSQL at {self.db_url}") # Create connection pool # For read-heavy workloads with many parallel think/search operations, @@ -414,6 +452,24 @@ class MemoryEngine: return await _retry_with_backoff(acquire) + async def health_check(self) -> dict: + """ + Perform a health check by querying the database. + + Returns: + dict with status and optional error message + """ + try: + pool = await self._get_pool() + async with pool.acquire() as conn: + result = await conn.fetchval("SELECT 1") + if result == 1: + return {"status": "healthy", "database": "connected"} + else: + return {"status": "unhealthy", "database": "unexpected response"} + except Exception as e: + return {"status": "unhealthy", "database": "error", "error": str(e)} + async def close(self): """Close the connection pool and shutdown background workers.""" logger.info("close() started") @@ -503,8 +559,15 @@ class MemoryEngine: if not texts: return [] - time_lower = event_date - timedelta(hours=time_window_hours) - time_upper = event_date + timedelta(hours=time_window_hours) + # Handle edge cases where event_date is at datetime boundaries + try: + time_lower = event_date - timedelta(hours=time_window_hours) + except OverflowError: + time_lower = datetime.min + try: + time_upper = event_date + timedelta(hours=time_window_hours) + except OverflowError: + time_upper = datetime.max # Fetch ALL existing facts in time window ONCE (much faster than N queries) import time as time_mod diff --git a/hindsight-api/hindsight_api/engine/query_analyzer.py b/hindsight-api/hindsight_api/engine/query_analyzer.py index 417cdfeb..6966cbbb 100644 --- a/hindsight-api/hindsight_api/engine/query_analyzer.py +++ b/hindsight-api/hindsight_api/engine/query_analyzer.py @@ -46,6 +46,16 @@ class QueryAnalyzer(ABC): information like temporal constraints, entities, etc. """ + @abstractmethod + def load(self) -> None: + """ + Load the query analyzer model. + + This should be called during initialization to load the model + and avoid cold start latency on first analyze() call. + """ + pass + @abstractmethod def analyze( self, query: str, reference_date: Optional[datetime] = None @@ -94,21 +104,29 @@ class TransformerQueryAnalyzer(QueryAnalyzer): self._model = None self._tokenizer = None - def _load_model(self): - """Lazy load the T5 model for temporal extraction.""" - if self._model is None: - try: - from transformers import AutoTokenizer, AutoModelForSeq2SeqLM - except ImportError: - raise ImportError( - "transformers is required for TransformerQueryAnalyzer. " - "Install it with: pip install transformers" - ) + def load(self) -> None: + """Load the T5 model for temporal extraction.""" + if self._model is not None: + return - self._tokenizer = AutoTokenizer.from_pretrained(self.model_name) - self._model = AutoModelForSeq2SeqLM.from_pretrained(self.model_name) - self._model.to(self.device) - self._model.eval() + try: + from transformers import AutoTokenizer, AutoModelForSeq2SeqLM + except ImportError: + raise ImportError( + "transformers is required for TransformerQueryAnalyzer. " + "Install it with: pip install transformers" + ) + + logger.info(f"Loading query analyzer model: {self.model_name}...") + self._tokenizer = AutoTokenizer.from_pretrained(self.model_name) + self._model = AutoModelForSeq2SeqLM.from_pretrained(self.model_name) + self._model.to(self.device) + self._model.eval() + logger.info("Query analyzer model loaded") + + def _load_model(self): + """Lazy load the T5 model for temporal extraction (calls load()).""" + self.load() def analyze( self, query: str, reference_date: Optional[datetime] = None diff --git a/hindsight-api/hindsight_api/engine/retain/link_utils.py b/hindsight-api/hindsight_api/engine/retain/link_utils.py index 9324b8d8..e6f21c39 100644 --- a/hindsight-api/hindsight_api/engine/retain/link_utils.py +++ b/hindsight-api/hindsight_api/engine/retain/link_utils.py @@ -5,11 +5,107 @@ Link creation utilities for temporal, semantic, and entity links. import time import logging from typing import List -from datetime import timedelta +from datetime import timedelta, datetime, timezone logger = logging.getLogger(__name__) +def _normalize_datetime(dt): + """Normalize datetime to be timezone-aware (UTC) for consistent comparison.""" + if dt is None: + return None + if dt.tzinfo is None: + # Naive datetime - assume UTC + return dt.replace(tzinfo=timezone.utc) + return dt + + +def compute_temporal_links( + new_units: dict, + candidates: list, + time_window_hours: int = 24, +) -> list: + """ + Compute temporal links between new units and candidate neighbors. + + This is a pure function that takes query results and returns link tuples, + making it easy to test without database access. + + Args: + new_units: Dict mapping unit_id (str) to event_date (datetime) + candidates: List of dicts with 'id' and 'event_date' keys (candidate neighbors) + time_window_hours: Time window in hours for temporal links + + Returns: + List of tuples: (from_unit_id, to_unit_id, 'temporal', weight, None) + """ + if not new_units: + return [] + + links = [] + for unit_id, unit_event_date in new_units.items(): + # Normalize unit_event_date for consistent comparison + unit_event_date_norm = _normalize_datetime(unit_event_date) + + # Calculate time window bounds with overflow protection + try: + time_lower = unit_event_date_norm - timedelta(hours=time_window_hours) + except OverflowError: + time_lower = datetime.min.replace(tzinfo=timezone.utc) + try: + time_upper = unit_event_date_norm + timedelta(hours=time_window_hours) + except OverflowError: + time_upper = datetime.max.replace(tzinfo=timezone.utc) + + # Filter candidates within this unit's time window + matching_neighbors = [ + (row['id'], row['event_date']) + for row in candidates + if time_lower <= _normalize_datetime(row['event_date']) <= time_upper + ][:10] # Limit to top 10 + + for recent_id, recent_event_date in matching_neighbors: + # Calculate temporal proximity weight + time_diff_hours = abs((unit_event_date_norm - _normalize_datetime(recent_event_date)).total_seconds() / 3600) + weight = max(0.3, 1.0 - (time_diff_hours / time_window_hours)) + links.append((unit_id, str(recent_id), 'temporal', weight, None)) + + return links + + +def compute_temporal_query_bounds( + new_units: dict, + time_window_hours: int = 24, +) -> tuple: + """ + Compute the min/max date bounds for querying temporal neighbors. + + Args: + new_units: Dict mapping unit_id (str) to event_date (datetime) + time_window_hours: Time window in hours + + Returns: + Tuple of (min_date, max_date) with overflow protection + """ + if not new_units: + return None, None + + # Normalize all dates to be timezone-aware to avoid comparison issues + all_dates = [_normalize_datetime(d) for d in new_units.values()] + + try: + min_date = min(all_dates) - timedelta(hours=time_window_hours) + except OverflowError: + min_date = datetime.min.replace(tzinfo=timezone.utc) + + try: + max_date = max(all_dates) + timedelta(hours=time_window_hours) + except OverflowError: + max_date = datetime.max.replace(tzinfo=timezone.utc) + + return min_date, max_date + + def _log(log_buffer, message, level='info'): """Helper to log to buffer if available, otherwise use logger.""" if log_buffer is not None: @@ -267,10 +363,8 @@ async def create_temporal_links_batch_per_fact( _log(log_buffer, f" [7.1] Fetch event_dates for {len(unit_ids)} units: {time_mod.time() - fetch_dates_start:.3f}s") # Fetch ALL potential temporal neighbors in ONE query (much faster!) - # Get time range across all units - all_dates = list(new_units.values()) - min_date = min(all_dates) - timedelta(hours=time_window_hours) - max_date = max(all_dates) + timedelta(hours=time_window_hours) + # Get time range across all units with overflow protection + min_date, max_date = compute_temporal_query_bounds(new_units, time_window_hours) fetch_neighbors_start = time_mod.time() all_candidates = await conn.fetch( @@ -291,24 +385,7 @@ async def create_temporal_links_batch_per_fact( # Filter and create links in memory (much faster than N queries) link_gen_start = time_mod.time() - links = [] - for unit_id, unit_event_date in new_units.items(): - # Filter candidates within this unit's time window - time_lower = unit_event_date - timedelta(hours=time_window_hours) - time_upper = unit_event_date + timedelta(hours=time_window_hours) - - matching_neighbors = [ - (row['id'], row['event_date']) - for row in all_candidates - if time_lower <= row['event_date'] <= time_upper - ][:10] # Limit to top 10 - - for recent_id, recent_event_date in matching_neighbors: - # Calculate temporal proximity weight - time_diff_hours = abs((unit_event_date - recent_event_date).total_seconds() / 3600) - weight = max(0.3, 1.0 - (time_diff_hours / time_window_hours)) - links.append((unit_id, str(recent_id), 'temporal', weight, None)) - + links = compute_temporal_links(new_units, all_candidates, time_window_hours) _log(log_buffer, f" [7.3] Generate {len(links)} temporal links: {time_mod.time() - link_gen_start:.3f}s") if links: diff --git a/hindsight-api/hindsight_api/engine/search/reranking.py b/hindsight-api/hindsight_api/engine/search/reranking.py index a501f0f7..7a59bf55 100644 --- a/hindsight-api/hindsight_api/engine/search/reranking.py +++ b/hindsight-api/hindsight_api/engine/search/reranking.py @@ -23,9 +23,11 @@ class CrossEncoderReranker: Args: cross_encoder: CrossEncoderReranker instance. If None, uses default SentenceTransformersCrossEncoder with ms-marco-MiniLM-L-6-v2 + (loaded lazily for faster startup) """ if cross_encoder is None: from hindsight_api.engine.cross_encoder import SentenceTransformersCrossEncoder + # Model is loaded lazily - call ensure_loaded() during initialize() cross_encoder = SentenceTransformersCrossEncoder() self.cross_encoder = cross_encoder diff --git a/hindsight-api/hindsight_api/pg0.py b/hindsight-api/hindsight_api/pg0.py index 265fead2..d2e67a5a 100644 --- a/hindsight-api/hindsight_api/pg0.py +++ b/hindsight-api/hindsight_api/pg0.py @@ -3,10 +3,10 @@ import json import logging import os import platform +import re import shutil import stat import subprocess -import sys from pathlib import Path from typing import Optional @@ -14,8 +14,7 @@ import httpx logger = logging.getLogger(__name__) -DEFAULT_DATA_DIR = Path(os.environ.get("HINDSIGHT_API_PG0_DATA_DIR", Path.home() / ".hindsight" / "pg_data")) -DEFAULT_INSTALL_DIR = Path.home() / ".hindsight" / "bin" +# pg0 configuration BINARY_NAME = "pg0" DEFAULT_PORT = 5555 DEFAULT_USERNAME = "hindsight" @@ -65,9 +64,7 @@ def get_download_url( version: str = "latest", repo: str = "vectorize-io/pg0", ) -> str: - """ - """ - # Check for direct URL override + """Get the download URL for pg0 binary.""" binary_name = get_platform_binary_name() if version == "latest": @@ -76,17 +73,32 @@ def get_download_url( return f"https://github.com/{repo}/releases/download/{version}/{binary_name}" +def _find_pg0_binary() -> Optional[Path]: + """Find pg0 binary in PATH or default install location.""" + # First check PATH + pg0_in_path = shutil.which("pg0") + if pg0_in_path: + return Path(pg0_in_path) + + # Fall back to default install location + default_path = Path.home() / ".hindsight" / "bin" / "pg0" + if default_path.exists() and os.access(default_path, os.X_OK): + return default_path + + return None + + class EmbeddedPostgres: """ - Manages an embedded PostgreSQL server instance. + Manages an embedded PostgreSQL server instance using pg0. This class handles: - - Downloading and installing the embedded-postgres CLI + - Finding or downloading the pg0 CLI - Starting/stopping the PostgreSQL server - Getting the connection URI Example: - pg = EmbeddedPostgres(data_dir="~/.myapp/data") + pg = EmbeddedPostgres() await pg.ensure_installed() await pg.start() uri = await pg.get_uri() @@ -96,8 +108,6 @@ class EmbeddedPostgres: def __init__( self, - data_dir: Optional[Path] = None, - install_dir: Optional[Path] = None, version: str = "latest", port: int = DEFAULT_PORT, username: str = DEFAULT_USERNAME, @@ -109,17 +119,13 @@ class EmbeddedPostgres: Initialize the embedded PostgreSQL manager. Args: - data_dir: Directory to store PostgreSQL data. Defaults to ~/.hindsight/pg_data - install_dir: Directory to install the CLI binary. Defaults to ~/.hindsight/bin - version: Version of embedded-postgres to use. Defaults to "latest" + version: Version of pg0 to download if not found. Defaults to "latest" port: Port to listen on. Defaults to 5555 username: Username for the database. Defaults to "hindsight" password: Password for the database. Defaults to "hindsight" database: Database name to create. Defaults to "hindsight" name: Instance name for pg0. Defaults to "hindsight" """ - self.data_dir = Path(data_dir or DEFAULT_DATA_DIR).expanduser() - self.install_dir = Path(install_dir or DEFAULT_INSTALL_DIR).expanduser() self.version = version self.port = port self.username = username @@ -127,35 +133,42 @@ class EmbeddedPostgres: self.database = database self.name = name - # Binary path - binary_name = "pg0.exe" if platform.system() == "Windows" else "pg0" - self.binary_path = self.install_dir / binary_name + # Will be set when binary is found/installed + self._binary_path: Optional[Path] = _find_pg0_binary() - self._process: Optional[subprocess.Popen] = None + @property + def binary_path(self) -> Path: + """Get the path to the pg0 binary.""" + if self._binary_path is None: + # Default install location + return Path.home() / ".hindsight" / "bin" / "pg0" + return self._binary_path def is_installed(self) -> bool: - """Check if the embedded-postgres CLI is installed.""" - return self.binary_path.exists() and os.access(self.binary_path, os.X_OK) + """Check if pg0 is available (in PATH or installed).""" + self._binary_path = _find_pg0_binary() + return self._binary_path is not None async def ensure_installed(self) -> None: """ - Ensure the embedded-postgres CLI is installed. + Ensure pg0 is available. - Downloads and installs the binary if not already present. + First checks PATH, then default location, then downloads if needed. """ if self.is_installed(): - logger.info(f"pg0 already installed at {self.binary_path}") + logger.debug(f"pg0 found at {self._binary_path}") return - logger.info("Installing pg0 CLI...") + logger.info("pg0 not found, downloading...") # Log platform information binary_name = get_platform_binary_name() logger.info(f"Detected platform: system={platform.system()}, machine={platform.machine()}") - logger.info(f"Will download binary: {binary_name}") - # Create install directory - self.install_dir.mkdir(parents=True, exist_ok=True) + # Install to default location + install_dir = Path.home() / ".hindsight" / "bin" + install_dir.mkdir(parents=True, exist_ok=True) + install_path = install_dir / "pg0" # Download the binary download_url = get_download_url(self.version) @@ -167,85 +180,115 @@ class EmbeddedPostgres: response.raise_for_status() # Write binary to disk - with open(self.binary_path, "wb") as f: + with open(install_path, "wb") as f: f.write(response.content) # Make executable on Unix if platform.system() != "Windows": - st = os.stat(self.binary_path) - os.chmod(self.binary_path, st.st_mode | stat.S_IEXEC) + st = os.stat(install_path) + os.chmod(install_path, st.st_mode | stat.S_IEXEC) - logger.info(f"Installed pg0 to {self.binary_path}") + self._binary_path = install_path + logger.info(f"Installed pg0 to {install_path}") except httpx.HTTPError as e: raise RuntimeError(f"Failed to download pg0: {e}") from e def _run_command(self, *args: str, capture_output: bool = True) -> subprocess.CompletedProcess: - """Run an embedded-postgres command synchronously.""" + """Run a pg0 command synchronously.""" + cmd = [str(self.binary_path), *args] + return subprocess.run(cmd, capture_output=capture_output, text=True) + + async def _run_command_async(self, *args: str, timeout: int = 120) -> tuple[int, str, str]: + """Run a pg0 command asynchronously.""" cmd = [str(self.binary_path), *args] - return subprocess.run( - cmd, - capture_output=capture_output, - text=True, - ) + def run_sync(): + try: + result = subprocess.run( + cmd, + stdin=subprocess.DEVNULL, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + timeout=timeout, + ) + return result.returncode, result.stdout, result.stderr + except subprocess.TimeoutExpired: + return 1, "", "Command timed out" - async def _run_command_async(self, *args: str) -> tuple[int, str, str]: - """Run an embedded-postgres command asynchronously.""" - cmd = [str(self.binary_path), *args] + loop = asyncio.get_event_loop() + return await loop.run_in_executor(None, run_sync) - process = await asyncio.create_subprocess_exec( - *cmd, - stdout=asyncio.subprocess.PIPE, - stderr=asyncio.subprocess.PIPE, - ) + def _extract_uri_from_output(self, output: str) -> Optional[str]: + """Extract the PostgreSQL URI from pg0 start output.""" + match = re.search(r"Connection URI:\s*(postgresql://[^\s]+)", output) + if match: + return match.group(1) + return None - stdout, stderr = await process.communicate() - return process.returncode, stdout.decode(), stderr.decode() - - async def start(self) -> str: + async def start(self, max_retries: int = 3, retry_delay: float = 2.0) -> str: """ - Start the PostgreSQL server. + Start the PostgreSQL server with retry logic. + + Args: + max_retries: Maximum number of start attempts (default: 3) + retry_delay: Initial delay between retries in seconds (default: 2.0) Returns: The connection URI for the started server. Raises: - RuntimeError: If the server fails to start. + RuntimeError: If the server fails to start after all retries. """ if not self.is_installed(): raise RuntimeError("pg0 is not installed. Call ensure_installed() first.") - # Create data directory - self.data_dir.mkdir(parents=True, exist_ok=True) + logger.info(f"Starting embedded PostgreSQL (name: {self.name}, port: {self.port})...") - logger.info(f"Starting embedded PostgreSQL (name: {self.name}, data: {self.data_dir}, install: {self.install_dir}, port: {self.port})...") + last_error = None + for attempt in range(1, max_retries + 1): + returncode, stdout, stderr = await self._run_command_async( + "start", + "--name", self.name, + "--port", str(self.port), + "--username", self.username, + "--password", self.password, + "--database", self.database, + timeout=300, + ) - returncode, stdout, stderr = await self._run_command_async( - "start", - "--name", self.name, - "--port", str(self.port), - "--username", self.username, - "--password", self.password, - "--database", self.database, - "--data-dir", self.data_dir.as_posix() - ) + # Try to extract URI from output + uri = self._extract_uri_from_output(stdout) + if uri: + logger.info(f"PostgreSQL started on port {self.port}") + return uri - if returncode != 0: - raise RuntimeError(f"Failed to start PostgreSQL: {stderr}") + # Check if pg0 info can find the running instance + try: + uri = await self.get_uri() + logger.info(f"PostgreSQL started on port {self.port}") + return uri + except RuntimeError: + pass - logger.info("Embedded PostgreSQL started") + # Start failed, log and retry + last_error = stderr or f"pg0 start returned exit code {returncode}" + if attempt < max_retries: + delay = retry_delay * (2 ** (attempt - 1)) + logger.warning(f"pg0 start attempt {attempt}/{max_retries} failed: {last_error.strip()}") + logger.info(f"Retrying in {delay:.1f}s...") + await asyncio.sleep(delay) + else: + logger.warning(f"pg0 start attempt {attempt}/{max_retries} failed: {last_error.strip()}") - # Get and return the URI - return await self.get_uri() + # All retries exhausted - use constructed URI as fallback + uri = f"postgresql://{self.username}:{self.password}@localhost:{self.port}/{self.database}" + logger.warning(f"All pg0 start attempts failed, using constructed URI: {uri}") + return uri async def stop(self) -> None: - """ - Stop the PostgreSQL server. - - Raises: - RuntimeError: If the server fails to stop. - """ + """Stop the PostgreSQL server.""" if not self.is_installed(): return @@ -254,7 +297,6 @@ class EmbeddedPostgres: returncode, stdout, stderr = await self._run_command_async("stop", "--name", self.name) if returncode != 0: - # Don't raise if server wasn't running if "not running" in stderr.lower(): return raise RuntimeError(f"Failed to stop PostgreSQL: {stderr}") @@ -262,20 +304,13 @@ class EmbeddedPostgres: logger.info("Embedded PostgreSQL stopped") async def _get_info(self) -> dict: - """ - Get info from pg0 using the `info -o json` command. - - Returns: - Dictionary with 'running' (bool) and 'uri' (str) keys. - - Raises: - RuntimeError: If unable to get info. - """ + """Get info from pg0 using the `info -o json` command.""" if not self.is_installed(): raise RuntimeError("pg0 is not installed.") returncode, stdout, stderr = await self._run_command_async( - "info", "--name", self.name, "-o", "json") + "info", "--name", self.name, "-o", "json" + ) if returncode != 0: raise RuntimeError(f"Failed to get PostgreSQL info: {stderr}") @@ -286,15 +321,7 @@ class EmbeddedPostgres: raise RuntimeError(f"Failed to parse pg0 info output: {e}") async def get_uri(self) -> str: - """ - Get the connection URI for the PostgreSQL server. - - Returns: - PostgreSQL connection URI (e.g., postgresql://user:pass@localhost:5432/db) - - Raises: - RuntimeError: If unable to get the URI or server is not running. - """ + """Get the connection URI for the PostgreSQL server.""" info = await self._get_info() uri = info.get("uri") if not uri: @@ -302,12 +329,7 @@ class EmbeddedPostgres: return uri async def status(self) -> dict: - """ - Get the status of the PostgreSQL server. - - Returns: - Dictionary with status information including 'running' boolean and 'uri'. - """ + """Get the status of the PostgreSQL server.""" if not self.is_installed(): return {"installed": False, "running": False} @@ -317,16 +339,9 @@ class EmbeddedPostgres: "installed": True, "running": info.get("running", False), "uri": info.get("uri"), - "data_dir": str(self.data_dir), - "binary_path": str(self.binary_path), } except RuntimeError: - return { - "installed": True, - "running": False, - "data_dir": str(self.data_dir), - "binary_path": str(self.binary_path), - } + return {"installed": True, "running": False} async def is_running(self) -> bool: """Check if the PostgreSQL server is currently running.""" @@ -355,59 +370,42 @@ class EmbeddedPostgres: return await self.start() def uninstall(self) -> None: - """Remove the embedded-postgres binary.""" - if self.binary_path.exists(): - self.binary_path.unlink() - logger.info(f"Removed {self.binary_path}") + """Remove the pg0 binary (only if we installed it).""" + default_path = Path.home() / ".hindsight" / "bin" / "pg0" + if default_path.exists(): + default_path.unlink() + logger.info(f"Removed {default_path}") def clear_data(self) -> None: """Remove all PostgreSQL data (destructive!).""" - if self.data_dir.exists(): - shutil.rmtree(self.data_dir) - logger.info(f"Removed data directory {self.data_dir}") + result = self._run_command("drop", "--name", self.name, "--force") + if result.returncode == 0: + logger.info(f"Dropped pg0 instance {self.name}") + else: + logger.warning(f"Failed to drop pg0 instance {self.name}: {result.stderr}") -# Convenience functions for simple usage +# Convenience functions _default_instance: Optional[EmbeddedPostgres] = None -def get_embedded_postgres( - data_dir: Optional[Path] = None, - install_dir: Optional[Path] = None, -) -> EmbeddedPostgres: - """ - Get or create the default EmbeddedPostgres instance. - - Args: - data_dir: Override default data directory - install_dir: Override default install directory - - Returns: - EmbeddedPostgres instance - """ +def get_embedded_postgres() -> EmbeddedPostgres: + """Get or create the default EmbeddedPostgres instance.""" global _default_instance - if _default_instance is None or data_dir or install_dir: - _default_instance = EmbeddedPostgres( - data_dir=data_dir, - install_dir=install_dir, - ) + if _default_instance is None: + _default_instance = EmbeddedPostgres() return _default_instance -async def start_embedded_postgres( - data_dir: Optional[Path] = None, -) -> str: +async def start_embedded_postgres() -> str: """ Quick start function for embedded PostgreSQL. Downloads, installs, and starts PostgreSQL in one call. - Args: - data_dir: Directory to store PostgreSQL data - Returns: Connection URI string @@ -415,7 +413,7 @@ async def start_embedded_postgres( db_url = await start_embedded_postgres() conn = await asyncpg.connect(db_url) """ - pg = get_embedded_postgres(data_dir=data_dir) + pg = get_embedded_postgres() return await pg.ensure_running() @@ -424,4 +422,4 @@ async def stop_embedded_postgres() -> None: global _default_instance if _default_instance: - await _default_instance.stop() \ No newline at end of file + await _default_instance.stop() diff --git a/hindsight-api/hindsight_api/web/server.py b/hindsight-api/hindsight_api/web/server.py index 5c002f7f..fe7f9148 100644 --- a/hindsight-api/hindsight_api/web/server.py +++ b/hindsight-api/hindsight_api/web/server.py @@ -79,7 +79,10 @@ app = create_app( if __name__ == "__main__": import uvicorn - logging.basicConfig(level=logging.INFO) + # Get log level from environment variable (default: info) + env_log_level = os.environ.get("HINDSIGHT_API_LOG_LEVEL", "info").lower() + if env_log_level not in ["critical", "error", "warning", "info", "debug", "trace"]: + env_log_level = "info" # Parse CLI arguments parser = argparse.ArgumentParser(description="Memory Graph API Server") @@ -87,8 +90,8 @@ if __name__ == "__main__": parser.add_argument("--port", type=int, default=8888, help="Port to bind to (default: 8888)") parser.add_argument("--reload", action="store_true", help="Enable auto-reload on code changes") parser.add_argument("--workers", type=int, default=1, help="Number of worker processes (default: 1)") - parser.add_argument("--log-level", default="info", choices=["critical", "error", "warning", "info", "debug", "trace"], - help="Log level (default: info)") + parser.add_argument("--log-level", default=env_log_level, choices=["critical", "error", "warning", "info", "debug", "trace"], + help=f"Log level (default: {env_log_level}, from HINDSIGHT_API_LOG_LEVEL)") parser.add_argument("--access-log", action="store_true", help="Enable access log") parser.add_argument("--no-access-log", dest="access_log", action="store_false", help="Disable access log") parser.add_argument("--proxy-headers", action="store_true", help="Enable X-Forwarded-Proto, X-Forwarded-For headers") @@ -99,6 +102,21 @@ if __name__ == "__main__": args = parser.parse_args() + # Configure Python logging based on log level + log_level_map = { + "critical": logging.CRITICAL, + "error": logging.ERROR, + "warning": logging.WARNING, + "info": logging.INFO, + "debug": logging.DEBUG, + "trace": logging.DEBUG, # Python doesn't have TRACE, use DEBUG + } + logging.basicConfig( + level=log_level_map.get(args.log_level, logging.INFO), + format="%(asctime)s - %(levelname)s - %(name)s - %(message)s" + ) + logging.info(f"Starting Hindsight API on {args.host}:{args.port}") + app_ref = "hindsight_api.web.server:app" # Prepare uvicorn config diff --git a/hindsight-api/tests/test_link_utils.py b/hindsight-api/tests/test_link_utils.py new file mode 100644 index 00000000..969ca70f --- /dev/null +++ b/hindsight-api/tests/test_link_utils.py @@ -0,0 +1,256 @@ +"""Tests for link_utils datetime handling and temporal link computation.""" +import pytest +from datetime import datetime, timezone, timedelta + +from hindsight_api.engine.retain.link_utils import ( + _normalize_datetime, + compute_temporal_links, + compute_temporal_query_bounds, +) + + +class TestNormalizeDatetime: + """Tests for the _normalize_datetime helper function.""" + + def test_none_returns_none(self): + """Test that None input returns None.""" + assert _normalize_datetime(None) is None + + def test_naive_datetime_becomes_utc(self): + """Test that naive datetimes are converted to UTC.""" + naive_dt = datetime(2024, 6, 15, 10, 30, 0) + result = _normalize_datetime(naive_dt) + + assert result.tzinfo is not None + assert result.tzinfo == timezone.utc + assert result.year == 2024 + assert result.month == 6 + assert result.day == 15 + assert result.hour == 10 + assert result.minute == 30 + + def test_aware_datetime_unchanged(self): + """Test that timezone-aware datetimes are returned unchanged.""" + aware_dt = datetime(2024, 6, 15, 10, 30, 0, tzinfo=timezone.utc) + result = _normalize_datetime(aware_dt) + + assert result == aware_dt + assert result.tzinfo == timezone.utc + + def test_mixed_datetimes_can_be_compared(self): + """Test that normalized naive and aware datetimes can be compared.""" + naive_dt = datetime(2024, 6, 15, 10, 30, 0) + aware_dt = datetime(2024, 6, 15, 10, 30, 0, tzinfo=timezone.utc) + + normalized_naive = _normalize_datetime(naive_dt) + normalized_aware = _normalize_datetime(aware_dt) + + # Should be able to compare without TypeError + assert normalized_naive == normalized_aware + + +class TestComputeTemporalQueryBounds: + """Tests for compute_temporal_query_bounds function.""" + + def test_empty_units_returns_none(self): + """Test that empty input returns (None, None).""" + min_date, max_date = compute_temporal_query_bounds({}) + assert min_date is None + assert max_date is None + + def test_single_unit_normal_date(self): + """Test bounds for a single unit with normal date.""" + units = {"unit-1": datetime(2024, 6, 15, 12, 0, 0, tzinfo=timezone.utc)} + min_date, max_date = compute_temporal_query_bounds(units, time_window_hours=24) + + assert min_date == datetime(2024, 6, 14, 12, 0, 0, tzinfo=timezone.utc) + assert max_date == datetime(2024, 6, 16, 12, 0, 0, tzinfo=timezone.utc) + + def test_multiple_units(self): + """Test bounds span across multiple units.""" + units = { + "unit-1": datetime(2024, 6, 10, 12, 0, 0, tzinfo=timezone.utc), + "unit-2": datetime(2024, 6, 15, 12, 0, 0, tzinfo=timezone.utc), + "unit-3": datetime(2024, 6, 20, 12, 0, 0, tzinfo=timezone.utc), + } + min_date, max_date = compute_temporal_query_bounds(units, time_window_hours=24) + + # min should be Jun 10 - 24h = Jun 9 + assert min_date == datetime(2024, 6, 9, 12, 0, 0, tzinfo=timezone.utc) + # max should be Jun 20 + 24h = Jun 21 + assert max_date == datetime(2024, 6, 21, 12, 0, 0, tzinfo=timezone.utc) + + def test_mixed_naive_and_aware_datetimes(self): + """Test that mixed naive/aware datetimes work correctly.""" + units = { + "unit-1": datetime(2024, 6, 10, 12, 0, 0), # naive + "unit-2": datetime(2024, 6, 15, 12, 0, 0, tzinfo=timezone.utc), # aware + } + # Should not raise TypeError + min_date, max_date = compute_temporal_query_bounds(units, time_window_hours=24) + + assert min_date is not None + assert max_date is not None + assert min_date.tzinfo is not None + assert max_date.tzinfo is not None + + def test_overflow_near_datetime_min(self): + """Test overflow protection near datetime.min.""" + units = {"unit-1": datetime(1, 1, 2, 0, 0, tzinfo=timezone.utc)} + min_date, max_date = compute_temporal_query_bounds(units, time_window_hours=48) + + # Should handle overflow gracefully + assert min_date == datetime.min.replace(tzinfo=timezone.utc) + assert max_date is not None + + def test_overflow_near_datetime_max(self): + """Test overflow protection near datetime.max.""" + units = {"unit-1": datetime(9999, 12, 30, 0, 0, tzinfo=timezone.utc)} + min_date, max_date = compute_temporal_query_bounds(units, time_window_hours=48) + + # Should handle overflow gracefully + assert min_date is not None + assert max_date == datetime.max.replace(tzinfo=timezone.utc) + + +class TestComputeTemporalLinks: + """Tests for compute_temporal_links function.""" + + def test_empty_units_returns_empty(self): + """Test that empty input returns empty list.""" + links = compute_temporal_links({}, []) + assert links == [] + + def test_no_candidates_returns_empty(self): + """Test that no candidates means no links.""" + units = {"unit-1": datetime(2024, 6, 15, 12, 0, 0, tzinfo=timezone.utc)} + links = compute_temporal_links(units, []) + assert links == [] + + def test_candidate_within_window_creates_link(self): + """Test that candidates within time window create links.""" + units = {"unit-1": datetime(2024, 6, 15, 12, 0, 0, tzinfo=timezone.utc)} + candidates = [ + {"id": "candidate-1", "event_date": datetime(2024, 6, 15, 10, 0, 0, tzinfo=timezone.utc)}, + ] + + links = compute_temporal_links(units, candidates, time_window_hours=24) + + assert len(links) == 1 + assert links[0][0] == "unit-1" + assert links[0][1] == "candidate-1" + assert links[0][2] == "temporal" + assert links[0][4] is None + # Weight should be high since they're close (2 hours apart) + assert links[0][3] > 0.9 + + def test_candidate_outside_window_no_link(self): + """Test that candidates outside time window don't create links.""" + units = {"unit-1": datetime(2024, 6, 15, 12, 0, 0, tzinfo=timezone.utc)} + candidates = [ + {"id": "candidate-1", "event_date": datetime(2024, 6, 10, 12, 0, 0, tzinfo=timezone.utc)}, + ] + + links = compute_temporal_links(units, candidates, time_window_hours=24) + + assert len(links) == 0 + + def test_weight_decreases_with_distance(self): + """Test that weight decreases as time difference increases.""" + units = {"unit-1": datetime(2024, 6, 15, 12, 0, 0, tzinfo=timezone.utc)} + candidates = [ + {"id": "close", "event_date": datetime(2024, 6, 15, 11, 0, 0, tzinfo=timezone.utc)}, # 1 hour + {"id": "far", "event_date": datetime(2024, 6, 14, 18, 0, 0, tzinfo=timezone.utc)}, # 18 hours + ] + + links = compute_temporal_links(units, candidates, time_window_hours=24) + + assert len(links) == 2 + close_link = next(l for l in links if l[1] == "close") + far_link = next(l for l in links if l[1] == "far") + + assert close_link[3] > far_link[3] + + def test_max_10_links_per_unit(self): + """Test that at most 10 links are created per unit.""" + units = {"unit-1": datetime(2024, 6, 15, 12, 0, 0, tzinfo=timezone.utc)} + # Create 15 candidates all within window + candidates = [ + {"id": f"candidate-{i}", "event_date": datetime(2024, 6, 15, 11, 0, 0, tzinfo=timezone.utc)} + for i in range(15) + ] + + links = compute_temporal_links(units, candidates, time_window_hours=24) + + assert len(links) == 10 + + def test_multiple_units_multiple_candidates(self): + """Test with multiple units and candidates.""" + units = { + "unit-1": datetime(2024, 6, 15, 12, 0, 0, tzinfo=timezone.utc), + "unit-2": datetime(2024, 6, 20, 12, 0, 0, tzinfo=timezone.utc), + } + candidates = [ + {"id": "c1", "event_date": datetime(2024, 6, 15, 10, 0, 0, tzinfo=timezone.utc)}, # near unit-1 + {"id": "c2", "event_date": datetime(2024, 6, 20, 10, 0, 0, tzinfo=timezone.utc)}, # near unit-2 + {"id": "c3", "event_date": datetime(2024, 6, 17, 12, 0, 0, tzinfo=timezone.utc)}, # between, near neither + ] + + links = compute_temporal_links(units, candidates, time_window_hours=24) + + # unit-1 should link to c1 only + # unit-2 should link to c2 only + unit1_links = [l for l in links if l[0] == "unit-1"] + unit2_links = [l for l in links if l[0] == "unit-2"] + + assert len(unit1_links) == 1 + assert unit1_links[0][1] == "c1" + + assert len(unit2_links) == 1 + assert unit2_links[0][1] == "c2" + + def test_mixed_naive_and_aware_datetimes(self): + """Test that mixed naive/aware datetimes work correctly.""" + units = {"unit-1": datetime(2024, 6, 15, 12, 0, 0)} # naive + candidates = [ + {"id": "c1", "event_date": datetime(2024, 6, 15, 10, 0, 0, tzinfo=timezone.utc)}, # aware + ] + + # Should not raise TypeError + links = compute_temporal_links(units, candidates, time_window_hours=24) + assert len(links) == 1 + + def test_overflow_near_datetime_min(self): + """Test overflow protection when unit date is near datetime.min.""" + units = {"unit-1": datetime(1, 1, 2, 0, 0, tzinfo=timezone.utc)} + candidates = [ + {"id": "c1", "event_date": datetime(1, 1, 1, 12, 0, 0, tzinfo=timezone.utc)}, + ] + + # Should not raise OverflowError + links = compute_temporal_links(units, candidates, time_window_hours=48) + assert len(links) == 1 + + def test_overflow_near_datetime_max(self): + """Test overflow protection when unit date is near datetime.max.""" + units = {"unit-1": datetime(9999, 12, 30, 0, 0, tzinfo=timezone.utc)} + candidates = [ + {"id": "c1", "event_date": datetime(9999, 12, 31, 12, 0, 0, tzinfo=timezone.utc)}, + ] + + # Should not raise OverflowError + links = compute_temporal_links(units, candidates, time_window_hours=48) + assert len(links) == 1 + + def test_weight_minimum_is_0_3(self): + """Test that weight doesn't go below 0.3.""" + units = {"unit-1": datetime(2024, 6, 15, 12, 0, 0, tzinfo=timezone.utc)} + candidates = [ + # 23 hours apart - should be just within 24h window but low weight + {"id": "c1", "event_date": datetime(2024, 6, 14, 13, 0, 0, tzinfo=timezone.utc)}, + ] + + links = compute_temporal_links(units, candidates, time_window_hours=24) + + assert len(links) == 1 + assert links[0][3] >= 0.3 diff --git a/hindsight-control-plane/package-lock.json b/hindsight-control-plane/package-lock.json index c1b98642..0c9d4cb5 100644 --- a/hindsight-control-plane/package-lock.json +++ b/hindsight-control-plane/package-lock.json @@ -12,7 +12,9 @@ "@hindsight/client": "file:../hindsight-clients/typescript", "@radix-ui/react-checkbox": "^1.3.3", "@radix-ui/react-dialog": "^1.1.15", + "@radix-ui/react-label": "^2.1.8", "@radix-ui/react-popover": "^1.1.15", + "@radix-ui/react-radio-group": "^1.3.8", "@radix-ui/react-select": "^2.2.6", "@radix-ui/react-slot": "^1.2.4", "@tailwindcss/postcss": "^4.1.17", @@ -1586,6 +1588,52 @@ } } }, + "node_modules/@radix-ui/react-label": { + "version": "2.1.8", + "resolved": "https://registry.npmjs.org/@radix-ui/react-label/-/react-label-2.1.8.tgz", + "integrity": "sha512-FmXs37I6hSBVDlO4y764TNz1rLgKwjJMQ0EGte6F3Cb3f4bIuHB/iLa/8I9VKkmOy+gNHq8rql3j686ACVV21A==", + "license": "MIT", + "dependencies": { + "@radix-ui/react-primitive": "2.1.4" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", + "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-label/node_modules/@radix-ui/react-primitive": { + "version": "2.1.4", + "resolved": "https://registry.npmjs.org/@radix-ui/react-primitive/-/react-primitive-2.1.4.tgz", + "integrity": "sha512-9hQc4+GNVtJAIEPEqlYqW5RiYdrr8ea5XQ0ZOnD6fgru+83kqT15mq2OCcbe8KnjRZl5vF3ks69AKz3kh1jrhg==", + "license": "MIT", + "dependencies": { + "@radix-ui/react-slot": "1.2.4" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", + "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, "node_modules/@radix-ui/react-popover": { "version": "1.1.15", "resolved": "https://registry.npmjs.org/@radix-ui/react-popover/-/react-popover-1.1.15.tgz", @@ -1762,6 +1810,69 @@ } } }, + "node_modules/@radix-ui/react-radio-group": { + "version": "1.3.8", + "resolved": "https://registry.npmjs.org/@radix-ui/react-radio-group/-/react-radio-group-1.3.8.tgz", + "integrity": "sha512-VBKYIYImA5zsxACdisNQ3BjCBfmbGH3kQlnFVqlWU4tXwjy7cGX8ta80BcrO+WJXIn5iBylEH3K6ZTlee//lgQ==", + "license": "MIT", + "dependencies": { + "@radix-ui/primitive": "1.1.3", + "@radix-ui/react-compose-refs": "1.1.2", + "@radix-ui/react-context": "1.1.2", + "@radix-ui/react-direction": "1.1.1", + "@radix-ui/react-presence": "1.1.5", + "@radix-ui/react-primitive": "2.1.3", + "@radix-ui/react-roving-focus": "1.1.11", + "@radix-ui/react-use-controllable-state": "1.2.2", + "@radix-ui/react-use-previous": "1.1.1", + "@radix-ui/react-use-size": "1.1.1" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", + "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-roving-focus": { + "version": "1.1.11", + "resolved": "https://registry.npmjs.org/@radix-ui/react-roving-focus/-/react-roving-focus-1.1.11.tgz", + "integrity": "sha512-7A6S9jSgm/S+7MdtNDSb+IU859vQqJ/QAtcYQcfFC6W8RS4IxIZDldLR0xqCFZ6DCyrQLjLPsxtTNch5jVA4lA==", + "license": "MIT", + "dependencies": { + "@radix-ui/primitive": "1.1.3", + "@radix-ui/react-collection": "1.1.7", + "@radix-ui/react-compose-refs": "1.1.2", + "@radix-ui/react-context": "1.1.2", + "@radix-ui/react-direction": "1.1.1", + "@radix-ui/react-id": "1.1.1", + "@radix-ui/react-primitive": "2.1.3", + "@radix-ui/react-use-callback-ref": "1.1.1", + "@radix-ui/react-use-controllable-state": "1.2.2" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", + "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, "node_modules/@radix-ui/react-select": { "version": "2.2.6", "resolved": "https://registry.npmjs.org/@radix-ui/react-select/-/react-select-2.2.6.tgz", diff --git a/hindsight-control-plane/package.json b/hindsight-control-plane/package.json index 073d4706..0327cddf 100644 --- a/hindsight-control-plane/package.json +++ b/hindsight-control-plane/package.json @@ -16,7 +16,9 @@ "@hindsight/client": "file:../hindsight-clients/typescript", "@radix-ui/react-checkbox": "^1.3.3", "@radix-ui/react-dialog": "^1.1.15", + "@radix-ui/react-label": "^2.1.8", "@radix-ui/react-popover": "^1.1.15", + "@radix-ui/react-radio-group": "^1.3.8", "@radix-ui/react-select": "^2.2.6", "@radix-ui/react-slot": "^1.2.4", "@tailwindcss/postcss": "^4.1.17", diff --git a/hindsight-control-plane/src/app/api/banks/route.ts b/hindsight-control-plane/src/app/api/banks/route.ts index 4f0a91ee..50fc571c 100644 --- a/hindsight-control-plane/src/app/api/banks/route.ts +++ b/hindsight-control-plane/src/app/api/banks/route.ts @@ -13,3 +13,31 @@ export async function GET() { ); } } + +export async function POST(request: Request) { + try { + const body = await request.json(); + const { bank_id } = body; + + if (!bank_id) { + return NextResponse.json( + { error: 'bank_id is required' }, + { status: 400 } + ); + } + + const response = await sdk.createOrUpdateBank({ + client: lowLevelClient, + path: { bank_id }, + body: {}, + }); + + return NextResponse.json(response.data, { status: 201 }); + } catch (error) { + console.error('Error creating bank:', error); + return NextResponse.json( + { error: 'Failed to create bank' }, + { status: 500 } + ); + } +} diff --git a/hindsight-control-plane/src/components/bank-selector.tsx b/hindsight-control-plane/src/components/bank-selector.tsx index 30908fcc..9acc80e0 100644 --- a/hindsight-control-plane/src/components/bank-selector.tsx +++ b/hindsight-control-plane/src/components/bank-selector.tsx @@ -4,6 +4,7 @@ import * as React from 'react'; import { Suspense } from 'react'; import { useRouter, useSearchParams } from 'next/navigation'; import { useBank } from '@/lib/bank-context'; +import { client } from '@/lib/api'; import { Button } from '@/components/ui/button'; import { Command, @@ -18,19 +19,105 @@ import { PopoverContent, PopoverTrigger, } from '@/components/ui/popover'; -import { Check, ChevronsUpDown } from 'lucide-react'; +import { + Dialog, + DialogContent, + DialogHeader, + DialogTitle, + DialogFooter, +} from '@/components/ui/dialog'; +import { Input } from '@/components/ui/input'; +import { Check, ChevronsUpDown, Plus, FileText } from 'lucide-react'; +import { Textarea } from '@/components/ui/textarea'; +import { Checkbox } from '@/components/ui/checkbox'; import { cn } from '@/lib/utils'; function BankSelectorInner() { const router = useRouter(); const searchParams = useSearchParams(); - const { currentBank, setCurrentBank, banks } = useBank(); + const { currentBank, setCurrentBank, banks, loadBanks } = useBank(); const [open, setOpen] = React.useState(false); + const [createDialogOpen, setCreateDialogOpen] = React.useState(false); + const [newBankId, setNewBankId] = React.useState(''); + const [isCreating, setIsCreating] = React.useState(false); + const [createError, setCreateError] = React.useState(null); + + // Document creation state + const [docDialogOpen, setDocDialogOpen] = React.useState(false); + const [docContent, setDocContent] = React.useState(''); + const [docContext, setDocContext] = React.useState(''); + const [docEventDate, setDocEventDate] = React.useState(''); + const [docDocumentId, setDocDocumentId] = React.useState(''); + const [docAsync, setDocAsync] = React.useState(false); + const [isCreatingDoc, setIsCreatingDoc] = React.useState(false); + const [docError, setDocError] = React.useState(null); const sortedBanks = React.useMemo(() => { return [...banks].sort((a, b) => a.localeCompare(b)); }, [banks]); + const handleCreateBank = async () => { + if (!newBankId.trim()) return; + + setIsCreating(true); + setCreateError(null); + + try { + await client.createBank(newBankId.trim()); + await loadBanks(); + setCreateDialogOpen(false); + setNewBankId(''); + // Navigate to the new bank + setCurrentBank(newBankId.trim()); + router.push(`/banks/${newBankId.trim()}?view=data`); + } catch (error) { + setCreateError(error instanceof Error ? error.message : 'Failed to create bank'); + } finally { + setIsCreating(false); + } + }; + + const handleCreateDocument = async () => { + if (!currentBank || !docContent.trim()) return; + + setIsCreatingDoc(true); + setDocError(null); + + try { + const item: any = { content: docContent }; + if (docContext) item.context = docContext; + if (docEventDate) item.event_date = docEventDate; + + const params: any = { + bank_id: currentBank, + items: [item], + }; + + if (docDocumentId) params.document_id = docDocumentId; + + if (docAsync) { + await client.retain({ ...params, async: true }); + } else { + await client.retain(params); + } + + // Reset form and close dialog + setDocDialogOpen(false); + setDocContent(''); + setDocContext(''); + setDocEventDate(''); + setDocDocumentId(''); + setDocAsync(false); + + // Navigate to documents view to see the new document + router.push(`/banks/${currentBank}?view=documents`); + } catch (error) { + setDocError(error instanceof Error ? error.message : 'Failed to create document'); + } finally { + setIsCreatingDoc(false); + } + }; + return (
@@ -81,6 +168,163 @@ function BankSelectorInner() { + + + + {currentBank && ( + + )} + + + + + Create New Memory Bank + +
+ setNewBankId(e.target.value)} + onKeyDown={(e) => { + if (e.key === 'Enter' && !isCreating) { + handleCreateBank(); + } + }} + autoFocus + /> + {createError && ( +

{createError}

+ )} +
+ + + + + + + + + + + Add New Document +

+ Add a new document to memory bank: {currentBank} +

+ +
+
+ +