diff --git a/docker/docker-compose/timescale/.dockerignore b/docker/docker-compose/timescale/.dockerignore new file mode 100644 index 00000000..a6e3050f --- /dev/null +++ b/docker/docker-compose/timescale/.dockerignore @@ -0,0 +1,16 @@ +# Git +.git +.gitignore +.gitattributes + +# Docker +docker-compose.yaml +.dockerignore + +# Documentation +README.md +*.md + +# Environment +.env +.env.example diff --git a/docker/docker-compose/timescale/.env.example b/docker/docker-compose/timescale/.env.example new file mode 100644 index 00000000..6003665f --- /dev/null +++ b/docker/docker-compose/timescale/.env.example @@ -0,0 +1,25 @@ +# PostgreSQL Configuration +HINDSIGHT_DB_USER=hindsight_user +HINDSIGHT_DB_PASSWORD=change-me-to-secure-password +HINDSIGHT_DB_NAME=hindsight_db + +# Hindsight Version +HINDSIGHT_VERSION=latest + +# LLM Configuration +HINDSIGHT_API_LLM_PROVIDER=openai +OPENAI_API_KEY=your-openai-api-key-here + +# Alternative LLM providers (uncomment and configure as needed): +# HINDSIGHT_API_LLM_PROVIDER=anthropic +# ANTHROPIC_API_KEY=your-anthropic-api-key + +# HINDSIGHT_API_LLM_PROVIDER=gemini +# GEMINI_API_KEY=your-gemini-api-key + +# HINDSIGHT_API_LLM_PROVIDER=groq +# GROQ_API_KEY=your-groq-api-key + +# Vector and Text Search (already configured in docker-compose.yaml) +# HINDSIGHT_API_VECTOR_EXTENSION=pgvectorscale +# HINDSIGHT_API_TEXT_SEARCH_EXTENSION=pg_textsearch diff --git a/docker/docker-compose/timescale/Dockerfile b/docker/docker-compose/timescale/Dockerfile new file mode 100644 index 00000000..d75d1d2b --- /dev/null +++ b/docker/docker-compose/timescale/Dockerfile @@ -0,0 +1,55 @@ +# PostgreSQL with pgvector, pgvectorscale, and pg_textsearch extensions +# All three extensions from Timescale/pgvector for high-performance vector and text search +# Note: Requires PostgreSQL 16+ +FROM postgres:17 + +# Install build dependencies and Rust toolchain +RUN apt-get update && apt-get install -y \ + build-essential \ + git \ + postgresql-server-dev-17 \ + libpq-dev \ + cmake \ + curl \ + pkg-config \ + libssl-dev \ + && rm -rf /var/lib/apt/lists/* + +# Install Rust toolchain (required for pgvectorscale) +RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y +ENV PATH="/root/.cargo/bin:${PATH}" + +# Install pgvector (required by pgvectorscale) +RUN cd /tmp && \ + git clone --branch v0.8.0 https://github.com/pgvector/pgvector.git && \ + cd pgvector && \ + make && \ + make install && \ + rm -rf /tmp/pgvector + +# Install cargo-pgrx (PostgreSQL extension framework for Rust) +RUN cargo install cargo-pgrx --version 0.12.5 --locked && \ + cargo pgrx init --pg17 /usr/bin/pg_config + +# Install pgvectorscale (DiskANN index support) +RUN cd /tmp && \ + git clone --branch 0.5.1 https://github.com/timescale/pgvectorscale.git && \ + cd pgvectorscale/pgvectorscale && \ + cargo pgrx install --release && \ + rm -rf /tmp/pgvectorscale + +# Install pg_textsearch (BM25 text search) +RUN cd /tmp && \ + git clone https://github.com/timescale/pg_textsearch.git && \ + cd pg_textsearch && \ + make && \ + make install && \ + rm -rf /tmp/pg_textsearch + +# Clean up build dependencies (keep runtime dependencies) +RUN apt-get purge -y --auto-remove git cmake curl && \ + rm -rf /root/.cargo/registry /root/.cargo/git + +# Ensure extensions are preloaded (pg_textsearch requires preloading) +RUN echo "shared_preload_libraries = 'pg_textsearch'" >> /usr/share/postgresql/postgresql.conf.sample + diff --git a/docker/docker-compose/timescale/README.md b/docker/docker-compose/timescale/README.md new file mode 100644 index 00000000..82104a4d --- /dev/null +++ b/docker/docker-compose/timescale/README.md @@ -0,0 +1,101 @@ +# Hindsight with Timescale Extensions + +This Docker Compose setup provides a complete Hindsight deployment with **Timescale extensions**: +- **pgvectorscale** - DiskANN algorithm for disk-based scalable vector search +- **pg_textsearch** - High-performance BM25 text search + +Both extensions are from [Timescale](https://github.com/timescale) and provide production-grade performance. + +## Prerequisites + +- Docker and Docker Compose installed +- OpenAI API key (or another LLM provider) + +## Quick Start + +```bash +# Set environment variables +export HINDSIGHT_DB_PASSWORD="your-secure-password" +export OPENAI_API_KEY="your-openai-api-key" + +# Build and start +docker compose -f docker/docker-compose/timescale/docker-compose.yaml up -d --build + +# Check logs + +docker compose -f docker/docker-compose/timescale/docker-compose.yaml logs -f +``` + +**Access:** +- API: http://localhost:8888 +- Control Plane: http://localhost:9999 + +## Stop and Clean Up + +```bash +# Stop services +docker compose -f docker/docker-compose/timescale/docker-compose.yaml down + +# Remove volumes (deletes all data) +docker compose -f docker/docker-compose/timescale/docker-compose.yaml down -v +``` + +## Configuration + +### Environment Variables + +| Variable | Description | Default | +|----------|-------------|---------| +| `HINDSIGHT_DB_PASSWORD` | PostgreSQL password | `hindsight_password` | +| `HINDSIGHT_DB_USER` | PostgreSQL username | `hindsight_user` | +| `HINDSIGHT_DB_NAME` | Database name | `hindsight_db` | +| `HINDSIGHT_VERSION` | Hindsight Docker image version | `latest` | +| `OPENAI_API_KEY` | OpenAI API key | (required) | +| `HINDSIGHT_API_LLM_PROVIDER` | LLM provider | `openai` | + +### Why Timescale Extensions? + +**pgvectorscale (DiskANN):** +- 28x lower p95 latency vs dedicated vector databases +- 16x higher query throughput at 99% recall +- 60-75% cost reduction (disk is cheaper than RAM) +- Best for large datasets (10M+ vectors) + +**pg_textsearch (BM25):** +- High-performance keyword retrieval +- Native BM25 ranking algorithm +- Optimized for full-text search + +## Troubleshooting + +### Extensions not installed + +Check if extensions are available: + +```bash +docker exec -it hindsight-db-timescale psql -U hindsight_user -d hindsight_db -c "\dx" +``` + +You should see: +- `vector` (pgvector) +- `vectorscale` (pgvectorscale/DiskANN) +- `pg_textsearch` (BM25 search) + +### Build fails + +If the Docker build fails during pgvectorscale compilation: + +1. Ensure you have sufficient memory (recommended: 4GB+) +2. Check Docker build logs for Rust compilation errors +3. Try building with more resources: `docker compose build --no-cache --memory 4g` + +### Port conflicts + +If port 5438 is already in use, modify the `ports` section in docker-compose.yaml. + +## Learn More + +- [pgvectorscale GitHub](https://github.com/timescale/pgvectorscale) +- [pg_textsearch GitHub](https://github.com/timescale/pg_textsearch) +- [HNSW vs DiskANN](https://www.tigerdata.com/learn/hnsw-vs-diskann) +- [Hindsight Documentation](https://hindsight.dev) diff --git a/docker/docker-compose/timescale/docker-compose.yaml b/docker/docker-compose/timescale/docker-compose.yaml new file mode 100644 index 00000000..fe1c7441 --- /dev/null +++ b/docker/docker-compose/timescale/docker-compose.yaml @@ -0,0 +1,108 @@ +name: hindsight +# Docker Compose file for Hindsight with Timescale extensions +# - pgvectorscale: DiskANN vector search (disk-based, scalable) +# - pg_textsearch: BM25 text search (high-performance keyword retrieval) +# +# Quick start: +# docker compose -f docker/docker-compose/timescale/docker-compose.yaml up -d --build +# +# Required environment variables: +# - HINDSIGHT_DB_PASSWORD: Password for the PostgreSQL user +# - OPENAI_API_KEY (or configure another LLM provider) +# +# Optional environment variables with defaults: +# - HINDSIGHT_VERSION: Hindsight application version (default: latest) +# - HINDSIGHT_DB_USER: PostgreSQL user (default: hindsight_user) +# - HINDSIGHT_DB_NAME: PostgreSQL database name (default: hindsight_db) + +services: + db: + # Custom PostgreSQL image with Timescale extensions (pgvectorscale + pg_textsearch) + build: + context: . + dockerfile: Dockerfile + container_name: hindsight-db-timescale + restart: always + # Expose PostgreSQL port (using 5438 to avoid conflicts with other setups) + ports: + - "5438:5432" + environment: + POSTGRES_USER: ${HINDSIGHT_DB_USER:-hindsight_user} + POSTGRES_PASSWORD: ${HINDSIGHT_DB_PASSWORD:-hindsight_password} + POSTGRES_DB: ${HINDSIGHT_DB_NAME:-hindsight_db} + volumes: + - pg_data:/var/lib/postgresql/data + networks: + - hindsight-net + # Health check to ensure database is ready + healthcheck: + test: ["CMD-SHELL", "pg_isready -U hindsight_user"] + interval: 5s + timeout: 5s + retries: 5 + + timescale-init: + build: + context: . + dockerfile: Dockerfile + depends_on: + db: + condition: service_healthy + environment: + - PGPASSWORD=${HINDSIGHT_DB_PASSWORD:-hindsight_password} + command: > + bash -c " + echo 'PostgreSQL is ready - creating hindsight_db database'; + psql -h hindsight-db-timescale -p 5432 -U hindsight_user -c 'CREATE DATABASE hindsight_db;' 2>/dev/null || echo 'Database already exists'; + echo 'Installing Timescale extensions...'; + echo '1/3: Installing pgvector (required by pgvectorscale)...'; + psql -h hindsight-db-timescale -p 5432 -U hindsight_user -d hindsight_db -c 'CREATE EXTENSION IF NOT EXISTS vector CASCADE;'; + echo '2/3: Installing pgvectorscale (DiskANN vector search)...'; + psql -h hindsight-db-timescale -p 5432 -U hindsight_user -d hindsight_db -c 'CREATE EXTENSION IF NOT EXISTS vectorscale CASCADE;'; + echo '3/3: Installing pg_textsearch (BM25 text search)...'; + psql -h hindsight-db-timescale -p 5432 -U hindsight_user -d hindsight_db -c 'CREATE EXTENSION IF NOT EXISTS pg_textsearch CASCADE;'; + echo ''; + echo '✅ Timescale extensions installed successfully'; + echo ''; + echo 'Installed extensions:'; + psql -h hindsight-db-timescale -p 5432 -U hindsight_user -d hindsight_db -c \"\\dx\" | grep -E '(vector|vectorscale|pg_textsearch)'; + " + restart: "no" + networks: + - hindsight-net + + hindsight: + image: ghcr.io/vectorize-io/hindsight:${HINDSIGHT_VERSION:-latest} + container_name: hindsight-app-timescale + ports: + - "8888:8888" + - "9999:9999" + environment: + # LLM Configuration + HINDSIGHT_API_LLM_PROVIDER: ${HINDSIGHT_API_LLM_PROVIDER:-openai} + HINDSIGHT_API_LLM_API_KEY: ${OPENAI_API_KEY:-your-api-key} + + # Database Configuration + HINDSIGHT_API_DATABASE_URL: postgresql://${HINDSIGHT_DB_USER:-hindsight_user}:${HINDSIGHT_DB_PASSWORD:-hindsight_password}@db:5432/${HINDSIGHT_DB_NAME:-hindsight_db} + + # Timescale Extensions + # pgvectorscale: DiskANN algorithm for disk-based scalable vector search + HINDSIGHT_API_VECTOR_EXTENSION: pgvectorscale + # pg_textsearch: High-performance BM25 text search + HINDSIGHT_API_TEXT_SEARCH_EXTENSION: pg_textsearch + + depends_on: + db: + condition: service_healthy + timescale-init: + condition: service_completed_successfully + networks: + - hindsight-net + + +networks: + hindsight-net: + driver: bridge + +volumes: + pg_data: diff --git a/hindsight-api/hindsight_api/alembic/versions/5a366d414dce_initial_schema.py b/hindsight-api/hindsight_api/alembic/versions/5a366d414dce_initial_schema.py index 2fd978a2..7deb872d 100644 --- a/hindsight-api/hindsight_api/alembic/versions/5a366d414dce_initial_schema.py +++ b/hindsight-api/hindsight_api/alembic/versions/5a366d414dce_initial_schema.py @@ -24,14 +24,27 @@ depends_on: str | Sequence[str] | None = None def _detect_vector_extension() -> str: """ - Detect or validate vector extension: 'vchord' or 'pgvector'. + Detect or validate vector extension: 'pgvector', 'vchord', or 'pgvectorscale'. Respects HINDSIGHT_API_VECTOR_EXTENSION env var if set. """ conn = op.get_bind() vector_extension = os.getenv("HINDSIGHT_API_VECTOR_EXTENSION", "pgvector").lower() # Validate configured extension is installed - if vector_extension == "vchord": + if vector_extension == "pgvectorscale": + # pgvectorscale requires pgvector + pgvector_check = conn.execute(text("SELECT 1 FROM pg_extension WHERE extname = 'vector'")).scalar() + if not pgvector_check: + raise RuntimeError( + "pgvectorscale requires pgvector. Install with: CREATE EXTENSION vector; CREATE EXTENSION vectorscale CASCADE;" + ) + vectorscale_check = conn.execute(text("SELECT 1 FROM pg_extension WHERE extname = 'vectorscale'")).scalar() + if not vectorscale_check: + raise RuntimeError( + "Configured vector extension 'pgvectorscale' not found. Install it with: CREATE EXTENSION vectorscale CASCADE;" + ) + return "pgvectorscale" + elif vector_extension == "vchord": vchord_check = conn.execute(text("SELECT 1 FROM pg_extension WHERE extname = 'vchord'")).scalar() if not vchord_check: raise RuntimeError( @@ -46,7 +59,9 @@ def _detect_vector_extension() -> str: ) return "pgvector" else: - raise ValueError(f"Invalid HINDSIGHT_API_VECTOR_EXTENSION: {vector_extension}. Must be 'pgvector' or 'vchord'") + raise ValueError( + f"Invalid HINDSIGHT_API_VECTOR_EXTENSION: {vector_extension}. Must be 'pgvector', 'vchord', or 'pgvectorscale'" + ) def _detect_text_search_extension() -> str: @@ -289,7 +304,14 @@ def upgrade() -> None: # Create vector index - conditional based on available extension vector_ext = _detect_vector_extension() - if vector_ext == "vchord": + if vector_ext == "pgvectorscale": + # Use DiskANN index for pgvectorscale (disk-based, scalable) + op.execute(""" + CREATE INDEX idx_memory_units_embedding ON memory_units + USING diskann (embedding vector_cosine_ops) + WITH (num_neighbors = 50) + """) + elif vector_ext == "vchord": # Use vchordrq index for vchord (supports high-dimensional embeddings) op.execute(""" CREATE INDEX idx_memory_units_embedding ON memory_units diff --git a/hindsight-api/hindsight_api/alembic/versions/n9i0j1k2l3m4_learnings_and_pinned_reflections.py b/hindsight-api/hindsight_api/alembic/versions/n9i0j1k2l3m4_learnings_and_pinned_reflections.py index 37b10be1..837401a9 100644 --- a/hindsight-api/hindsight_api/alembic/versions/n9i0j1k2l3m4_learnings_and_pinned_reflections.py +++ b/hindsight-api/hindsight_api/alembic/versions/n9i0j1k2l3m4_learnings_and_pinned_reflections.py @@ -31,14 +31,27 @@ def _get_schema_prefix() -> str: def _detect_vector_extension() -> str: """ - Detect or validate vector extension: 'vchord' or 'pgvector'. + Detect or validate vector extension: 'pgvector', 'vchord', or 'pgvectorscale'. Respects HINDSIGHT_API_VECTOR_EXTENSION env var if set. """ conn = op.get_bind() vector_extension = os.getenv("HINDSIGHT_API_VECTOR_EXTENSION", "pgvector").lower() # Validate configured extension is installed - if vector_extension == "vchord": + if vector_extension == "pgvectorscale": + # pgvectorscale requires pgvector + pgvector_check = conn.execute(text("SELECT 1 FROM pg_extension WHERE extname = 'vector'")).scalar() + if not pgvector_check: + raise RuntimeError( + "pgvectorscale requires pgvector. Install with: CREATE EXTENSION vector; CREATE EXTENSION vectorscale CASCADE;" + ) + vectorscale_check = conn.execute(text("SELECT 1 FROM pg_extension WHERE extname = 'vectorscale'")).scalar() + if not vectorscale_check: + raise RuntimeError( + "Configured vector extension 'pgvectorscale' not found. Install it with: CREATE EXTENSION vectorscale CASCADE;" + ) + return "pgvectorscale" + elif vector_extension == "vchord": vchord_check = conn.execute(text("SELECT 1 FROM pg_extension WHERE extname = 'vchord'")).scalar() if not vchord_check: raise RuntimeError( @@ -53,7 +66,9 @@ def _detect_vector_extension() -> str: ) return "pgvector" else: - raise ValueError(f"Invalid HINDSIGHT_API_VECTOR_EXTENSION: {vector_extension}. Must be 'pgvector' or 'vchord'") + raise ValueError( + f"Invalid HINDSIGHT_API_VECTOR_EXTENSION: {vector_extension}. Must be 'pgvector', 'vchord', or 'pgvectorscale'" + ) def _detect_text_search_extension() -> str: @@ -134,7 +149,13 @@ def upgrade() -> None: op.execute(f"CREATE INDEX idx_learnings_bank_id ON {schema}learnings(bank_id)") # Create vector index based on detected extension - if vector_ext == "vchord": + if vector_ext == "pgvectorscale": + op.execute(f""" + CREATE INDEX idx_learnings_embedding ON {schema}learnings + USING diskann (embedding vector_cosine_ops) + WITH (num_neighbors = 50) + """) + elif vector_ext == "vchord": op.execute(f""" CREATE INDEX idx_learnings_embedding ON {schema}learnings USING vchordrq (embedding vector_l2_ops) @@ -201,7 +222,13 @@ def upgrade() -> None: op.execute(f"CREATE INDEX idx_pinned_reflections_bank_id ON {schema}pinned_reflections(bank_id)") # Create vector index based on detected extension - if vector_ext == "vchord": + if vector_ext == "pgvectorscale": + op.execute(f""" + CREATE INDEX idx_pinned_reflections_embedding ON {schema}pinned_reflections + USING diskann (embedding vector_cosine_ops) + WITH (num_neighbors = 50) + """) + elif vector_ext == "vchord": op.execute(f""" CREATE INDEX idx_pinned_reflections_embedding ON {schema}pinned_reflections USING vchordrq (embedding vector_l2_ops) diff --git a/hindsight-api/hindsight_api/banner.py b/hindsight-api/hindsight_api/banner.py index ca9bf147..2eb5285a 100644 --- a/hindsight-api/hindsight_api/banner.py +++ b/hindsight-api/hindsight_api/banner.py @@ -86,6 +86,8 @@ def print_startup_info( reranker_provider: str, mcp_enabled: bool = False, version: str | None = None, + vector_extension: str | None = None, + text_search_extension: str | None = None, ): """Print styled startup information.""" print(color_start("Starting Hindsight API...")) @@ -96,6 +98,8 @@ def print_startup_info( print(f" {dim('LLM:')} {color(f'{llm_provider} / {llm_model}', 0.6)}") print(f" {dim('Embeddings:')} {color(embeddings_provider, 0.8)}") print(f" {dim('Reranker:')} {color(reranker_provider, 1.0)}") + extensions = f"{vector_extension or 'default'} (vector) / {text_search_extension or 'default'} (text)" + print(f" {dim('Extensions:')} {color(extensions, 0.4)}") if mcp_enabled: print(f" {dim('MCP:')} {color_end('enabled at /mcp')}") print() diff --git a/hindsight-api/hindsight_api/config.py b/hindsight-api/hindsight_api/config.py index 7857e691..48531015 100644 --- a/hindsight-api/hindsight_api/config.py +++ b/hindsight-api/hindsight_api/config.py @@ -342,8 +342,8 @@ DEFAULT_RERANKER_FLASHRANK_CACHE_DIR = None # Use default cache directory DEFAULT_EMBEDDINGS_COHERE_MODEL = "embed-english-v3.0" DEFAULT_RERANKER_COHERE_MODEL = "rerank-english-v3.0" -# Vector extension (pgvector vs vchord) -DEFAULT_VECTOR_EXTENSION = "pgvector" # Options: "pgvector", "vchord" +# Vector extension (pgvector, vchord, or pgvectorscale) +DEFAULT_VECTOR_EXTENSION = "pgvector" # Options: "pgvector", "vchord", "pgvectorscale" # Text search extension (native PostgreSQL, vchord BM25, or Timescale pg_textsearch) DEFAULT_TEXT_SEARCH_EXTENSION = "native" # Options: "native", "vchord", "pg_textsearch" @@ -733,7 +733,7 @@ class HindsightConfig: def validate(self) -> None: """Validate configuration values and raise errors for invalid combinations.""" # Validate vector_extension - valid_extensions = ("pgvector", "vchord") + valid_extensions = ("pgvector", "vchord", "pgvectorscale") if self.vector_extension not in valid_extensions: raise ValueError( f"Invalid vector_extension: {self.vector_extension}. Must be one of: {', '.join(valid_extensions)}" diff --git a/hindsight-api/hindsight_api/main.py b/hindsight-api/hindsight_api/main.py index 77317f0f..1ae2eda0 100644 --- a/hindsight-api/hindsight_api/main.py +++ b/hindsight-api/hindsight_api/main.py @@ -379,6 +379,8 @@ def main(): reranker_provider=config.reranker_provider, mcp_enabled=config.mcp_enabled, version=__version__, + vector_extension=config.vector_extension, + text_search_extension=config.text_search_extension, ) # Start idle checker in daemon mode diff --git a/hindsight-api/hindsight_api/migrations.py b/hindsight-api/hindsight_api/migrations.py index 45d94fee..05c8994a 100644 --- a/hindsight-api/hindsight_api/migrations.py +++ b/hindsight-api/hindsight_api/migrations.py @@ -35,20 +35,38 @@ MIGRATION_LOCK_ID = 123456789 def _detect_vector_extension(conn, vector_extension: str = "pgvector") -> str: """ - Validate vector extension: 'vchord' or 'pgvector'. + Validate vector extension: 'pgvector', 'vchord', or 'pgvectorscale'. Args: conn: SQLAlchemy connection object - vector_extension: Configured extension ("pgvector" or "vchord") + vector_extension: Configured extension ("pgvector", "vchord", or "pgvectorscale") Returns: - "vchord" or "pgvector" + "pgvector", "vchord", or "pgvectorscale" Raises: RuntimeError: If configured extension is not installed """ # Verify the configured extension is installed - if vector_extension == "vchord": + if vector_extension == "pgvectorscale": + # pgvectorscale requires pgvector to be installed first + pgvector_check = conn.execute(text("SELECT 1 FROM pg_extension WHERE extname = 'vector'")).scalar() + if not pgvector_check: + raise RuntimeError( + "pgvectorscale requires pgvector to be installed. " + "Install it with: CREATE EXTENSION vector; CREATE EXTENSION vectorscale CASCADE;" + ) + + # Check for vectorscale extension + vectorscale_check = conn.execute(text("SELECT 1 FROM pg_extension WHERE extname = 'vectorscale'")).scalar() + if not vectorscale_check: + raise RuntimeError( + "Configured vector extension 'pgvectorscale' not found. " + "Install it with: CREATE EXTENSION vectorscale CASCADE;" + ) + logger.debug("Using configured vector extension: pgvectorscale (DiskANN)") + return "pgvectorscale" + elif vector_extension == "vchord": vchord_check = conn.execute(text("SELECT 1 FROM pg_extension WHERE extname = 'vchord'")).scalar() if not vchord_check: raise RuntimeError( @@ -65,7 +83,9 @@ def _detect_vector_extension(conn, vector_extension: str = "pgvector") -> str: logger.debug("Using configured vector extension: pgvector") return "pgvector" else: - raise ValueError(f"Invalid vector_extension: {vector_extension}. Must be 'pgvector' or 'vchord'") + raise ValueError( + f"Invalid vector_extension: {vector_extension}. Must be 'pgvector', 'vchord', or 'pgvectorscale'" + ) def _get_schema_lock_id(schema: str) -> int: @@ -277,6 +297,48 @@ def run_migrations( "Please install it with: CREATE EXTENSION vector;" ) from e + # If using pgvectorscale, ensure vectorscale extension is also installed + vector_extension = os.getenv("HINDSIGHT_API_VECTOR_EXTENSION", "pgvector").lower() + if vector_extension == "pgvectorscale": + logger.debug("Checking pgvectorscale (vectorscale) extension availability...") + + vectorscale_check = conn.execute( + text("SELECT 1 FROM pg_extension WHERE extname = 'vectorscale'") + ).scalar() + + if vectorscale_check: + logger.info("pgvectorscale extension already installed") + else: + # Extension doesn't exist - try to install + logger.info("pgvectorscale extension not found, attempting to install...") + try: + conn.execute(text("CREATE EXTENSION vectorscale CASCADE")) + conn.commit() + logger.info("pgvectorscale extension installed successfully") + except Exception as e: + # Installation failed - check one more time in case another process installed it + conn.rollback() + vectorscale_recheck = conn.execute( + text("SELECT 1 FROM pg_extension WHERE extname = 'vectorscale'") + ).fetchone() + + if vectorscale_recheck: + logger.warning( + "Could not install pgvectorscale extension (permission denied?), " + "but extension exists. Continuing..." + ) + else: + # Extension truly doesn't exist and we can't install it + logger.error( + f"pgvectorscale extension is not installed and cannot be installed: {e}. " + f"Please ensure pgvectorscale is installed by a database administrator. " + f"See: https://github.com/timescale/pgvectorscale#installation" + ) + raise RuntimeError( + "pgvectorscale extension is required but not installed. " + "Please install it with: CREATE EXTENSION vectorscale CASCADE;" + ) from e + # Run migrations while holding the lock _run_migrations_internal(database_url, script_location, schema=schema) finally: @@ -475,7 +537,17 @@ def ensure_embedding_dimension( conn.commit() # Recreate index with appropriate type based on detected extension - if vector_ext == "vchord": + if vector_ext == "pgvectorscale": + conn.execute( + text(f""" + CREATE INDEX IF NOT EXISTS idx_memory_units_embedding_diskann + ON {schema_name}.memory_units + USING diskann (embedding vector_cosine_ops) + WITH (num_neighbors = 50) + """) + ) + logger.info(f"Created DiskANN index for {required_dimension}-dimensional embeddings") + elif vector_ext == "vchord": conn.execute( text(f""" CREATE INDEX IF NOT EXISTS idx_memory_units_embedding_vchordrq @@ -537,7 +609,12 @@ def ensure_vector_extension( ] # Determine target index type - target_index_type = "vchordrq" if target_ext == "vchord" else "hnsw" + if target_ext == "pgvectorscale": + target_index_type = "diskann" + elif target_ext == "vchord": + target_index_type = "vchordrq" + else: + target_index_type = "hnsw" mismatched_tables = [] tables_with_data = [] @@ -576,7 +653,9 @@ def ensure_vector_extension( continue indexdef = current_index_info[0].lower() - if "vchordrq" in indexdef: + if "diskann" in indexdef: + current_index_type = "diskann" + elif "vchordrq" in indexdef: current_index_type = "vchordrq" elif "hnsw" in indexdef: current_index_type = "hnsw" @@ -609,13 +688,18 @@ def ensure_vector_extension( # If there's data in any mismatched table, raise error if tables_with_data: table_list = ", ".join([f"{table}({count} rows)" for table, count in tables_with_data]) + # Map index type back to extension name for error message + current_ext_name = {"diskann": "pgvectorscale", "vchordrq": "vchord", "hnsw": "pgvector"}.get( + current_index_type, current_index_type + ) + raise RuntimeError( f"Cannot change vector extension from {current_index_type} to {target_index_type}: " f"the following tables contain data: {table_list}. " f"To change vector extension, you must either:\n" f" 1. Re-embed all data: DELETE FROM {schema_name}.memory_units; " f"DELETE FROM {schema_name}.learnings; DELETE FROM {schema_name}.pinned_reflections; then restart\n" - f" 2. Use the current vector extension (set HINDSIGHT_API_VECTOR_EXTENSION='{current_index_type.replace('vchordrq', 'vchord').replace('hnsw', 'pgvector')}')" + f" 2. Use the current vector extension (set HINDSIGHT_API_VECTOR_EXTENSION='{current_ext_name}')" ) # Tables are empty, safe to recreate indexes @@ -628,7 +712,17 @@ def ensure_vector_extension( conn.execute(text(f"DROP INDEX IF EXISTS {schema_name}.{index_name}")) # Create new index with appropriate type - if target_ext == "vchord": + if target_ext == "pgvectorscale": + logger.info(f"Creating DiskANN index on {table_name}") + conn.execute( + text(f""" + CREATE INDEX IF NOT EXISTS {index_name} + ON {schema_name}.{table_name} + USING diskann (embedding vector_cosine_ops) + WITH (num_neighbors = 50) + """) + ) + elif target_ext == "vchord": logger.info(f"Creating vchordrq index on {table_name}") conn.execute( text(f""" diff --git a/hindsight-docs/docs/developer/configuration.md b/hindsight-docs/docs/developer/configuration.md index e02f094b..9fc83dc0 100644 --- a/hindsight-docs/docs/developer/configuration.md +++ b/hindsight-docs/docs/developer/configuration.md @@ -61,31 +61,61 @@ hindsight-admin run-db-migration --schema tenant_acme | Variable | Description | Default | |----------|-------------|---------| -| `HINDSIGHT_API_VECTOR_EXTENSION` | Vector extension to use: `auto`, `pgvector`, or `vchord` | `auto` | +| `HINDSIGHT_API_VECTOR_EXTENSION` | Vector index algorithm: `pgvector`, `vchord`, or `pgvectorscale` | `pgvector` | -Hindsight supports two PostgreSQL vector extensions: -- **pgvector**: Standard extension, works well for most embeddings (up to ~2000 dimensions) -- **vchord**: Optimized for high-dimensional embeddings (3000+ dimensions), includes BM25 search +Hindsight supports three PostgreSQL vector extensions: -When set to `auto` (default), Hindsight automatically detects which extension is installed, preferring vchord if both are available. +#### **pgvector** (HNSW - default) +- In-memory index using Hierarchical Navigable Small World algorithm +- Works well for most embeddings and dataset sizes +- Fast for small-medium datasets (<10M vectors) +- Higher memory usage for large datasets +- Most widely deployed and supported + +#### **pgvectorscale** (DiskANN - recommended for scale) ⭐ +- Disk-based index using StreamingDiskANN algorithm (by Timescale) +- **28x lower p95 latency** and **16x higher throughput** vs dedicated vector DBs +- **60-75% cost reduction** at scale (SSDs cheaper than RAM) +- Superior filtering performance with streaming retrieval model +- Optimized for large datasets (10M+ vectors) +- Requires both `pgvector` and `vectorscale` extensions +- **Installation:** `CREATE EXTENSION vector; CREATE EXTENSION vectorscale CASCADE;` + +#### **vchord** (vchordrq) +- Alternative high-performance vector index +- Optimized for high-dimensional embeddings (3000+ dimensions) +- Includes integrated BM25 search capabilities +- Requires `vchord` extension + +**When to use pgvectorscale (DiskANN):** +- Large datasets (10M+ vectors) ⭐ +- Complex filtering requirements +- Cost-sensitive deployments +- Production workloads requiring high throughput +- When disk I/O is not a bottleneck + +**When to use pgvector (HNSW):** +- Small-medium datasets (<10M vectors) +- Maximum query speed when all data fits in memory +- Simple nearest-neighbor queries without filters +- Standard PostgreSQL deployment preference **When to use vchord:** -- Using high-dimensional embeddings (e.g., `text-embedding-3-large` with 3072 dimensions) -- Need better performance with large embedding dimensions -- Want to use vchord's BM25 search capabilities - -**When to use pgvector:** -- Using standard embedding dimensions (384-1536) -- Prefer the widely-adopted pgvector extension -- Simpler deployment (pgvector is more commonly available) +- High-dimensional embeddings (3000+ dimensions) +- Want integrated BM25 search +- Already using vchord for text search **Switching extensions:** If you need to switch from one extension to another: -1. Set `HINDSIGHT_API_VECTOR_EXTENSION` to your desired extension (`pgvector` or `vchord`) +1. Set `HINDSIGHT_API_VECTOR_EXTENSION` to your desired extension (`pgvector`, `vchord`, or `pgvectorscale`) 2. If your database has existing data, you'll get an error with migration instructions 3. For empty databases, indexes will be automatically recreated on startup +**Learn more:** +- [HNSW vs. DiskANN comparison](https://www.tigerdata.com/learn/hnsw-vs-diskann) +- [pgvectorscale GitHub](https://github.com/timescale/pgvectorscale) + ### Text Search Extension | Variable | Description | Default |