From 50fa2ed090480c999acd5c326baeea5fbefdae81 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=B2=20Boschi?= Date: Mon, 26 Jan 2026 15:25:30 +0100 Subject: [PATCH] ci: add upgrade tests (#200) --- .github/workflows/test.yml | 60 ++++ hindsight-dev/pyproject.toml | 9 +- hindsight-dev/upgrade_tests/__init__.py | 1 + hindsight-dev/upgrade_tests/conftest.py | 110 +++++++ hindsight-dev/upgrade_tests/test_upgrade.py | 303 ++++++++++++++++++ hindsight-dev/upgrade_tests/version_runner.py | 275 ++++++++++++++++ scripts/run-upgrade-tests.sh | 39 +++ uv.lock | 11 + 8 files changed, 807 insertions(+), 1 deletion(-) create mode 100644 hindsight-dev/upgrade_tests/__init__.py create mode 100644 hindsight-dev/upgrade_tests/conftest.py create mode 100644 hindsight-dev/upgrade_tests/test_upgrade.py create mode 100644 hindsight-dev/upgrade_tests/version_runner.py create mode 100755 scripts/run-upgrade-tests.sh diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 348ab35a..1fec88ca 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -875,6 +875,66 @@ jobs: echo "=== API Server Logs ===" cat /tmp/api-server.log || echo "No API server log found" + test-upgrade: + runs-on: ubuntu-latest + env: + HINDSIGHT_API_LLM_PROVIDER: groq + HINDSIGHT_API_LLM_API_KEY: ${{ secrets.GROQ_API_KEY }} + HINDSIGHT_API_LLM_MODEL: openai/gpt-oss-20b + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + UV_INDEX: pytorch=https://download.pytorch.org/whl/cpu + + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 # Full history needed for git clone of tags + + - name: Fetch tags + run: git fetch --tags + + - name: Install uv + uses: astral-sh/setup-uv@v5 + with: + enable-cache: true + prune-cache: false + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version-file: ".python-version" + + - name: Cache HuggingFace models + uses: actions/cache@v4 + with: + path: ~/.cache/huggingface + key: ${{ runner.os }}-huggingface-${{ hashFiles('hindsight-api/pyproject.toml') }} + restore-keys: | + ${{ runner.os }}-huggingface- + + - name: Install hindsight-dev dependencies + working-directory: ./hindsight-dev + run: uv sync --frozen --extra test --index-strategy unsafe-best-match + + - name: Install current hindsight-api + working-directory: ./hindsight-api + run: uv sync --frozen --index-strategy unsafe-best-match + + - name: Pre-download models + working-directory: ./hindsight-api + run: | + uv run python -c " + from sentence_transformers import SentenceTransformer, CrossEncoder + print('Downloading embedding model...') + SentenceTransformer('BAAI/bge-small-en-v1.5') + print('Downloading cross-encoder model...') + CrossEncoder('cross-encoder/ms-marco-MiniLM-L-6-v2') + print('Models downloaded successfully') + " + + - name: Run upgrade tests + working-directory: ./hindsight-dev + run: uv run pytest upgrade_tests/ -v --tb=short + verify-generated-files: runs-on: ubuntu-latest env: diff --git a/hindsight-dev/pyproject.toml b/hindsight-dev/pyproject.toml index fbc526ff..f431c141 100644 --- a/hindsight-dev/pyproject.toml +++ b/hindsight-dev/pyproject.toml @@ -16,8 +16,15 @@ dependencies = [ "pydantic>=2.0.0", ] +[project.optional-dependencies] +test = [ + "pytest>=8.0.0", + "httpx>=0.27.0", + "python-dotenv>=1.0.0", +] + [tool.hatch.build.targets.wheel] -packages = ["hindsight_dev", "benchmarks"] +packages = ["hindsight_dev", "benchmarks", "upgrade_tests"] [tool.uv.sources] hindsight-api = { workspace = true } diff --git a/hindsight-dev/upgrade_tests/__init__.py b/hindsight-dev/upgrade_tests/__init__.py new file mode 100644 index 00000000..8f7456aa --- /dev/null +++ b/hindsight-dev/upgrade_tests/__init__.py @@ -0,0 +1 @@ +# Upgrade and backwards compatibility tests diff --git a/hindsight-dev/upgrade_tests/conftest.py b/hindsight-dev/upgrade_tests/conftest.py new file mode 100644 index 00000000..b27ea43c --- /dev/null +++ b/hindsight-dev/upgrade_tests/conftest.py @@ -0,0 +1,110 @@ +""" +Pytest configuration and fixtures for upgrade tests. +""" + +import asyncio +import logging +import os +from pathlib import Path + +import pytest +from dotenv import load_dotenv + +# Configure logging for tests +logging.basicConfig( + level=logging.INFO, + format="%(asctime)s - %(name)s - %(levelname)s - %(message)s", +) + +# Reduce noise from httpx +logging.getLogger("httpx").setLevel(logging.WARNING) +logging.getLogger("httpcore").setLevel(logging.WARNING) + + +def pytest_configure(config): + """Load environment variables before running tests.""" + # Look for .env in the workspace root + env_file = Path(__file__).parent.parent.parent / ".env" + if env_file.exists(): + load_dotenv(env_file) + + +_pg0_instance = None +_pg0_url = None + + +def _get_or_create_pg0(): + """Get or create the shared pg0 instance for upgrade tests.""" + global _pg0_instance, _pg0_url + from hindsight_api.pg0 import EmbeddedPostgres + + if _pg0_instance is None: + _pg0_instance = EmbeddedPostgres(name="hindsight-upgrade-test", port=5560) + + loop = asyncio.new_event_loop() + try: + _pg0_url = loop.run_until_complete(_pg0_instance.ensure_running()) + finally: + loop.close() + + return _pg0_url + + +def _clean_database(db_url: str): + """Drop all tables in the database to reset state for next test.""" + from sqlalchemy import create_engine, text + + engine = create_engine(db_url) + with engine.connect() as conn: + # Drop all tables in public schema (cascade to handle foreign keys) + tables = conn.execute( + text(""" + SELECT tablename FROM pg_tables + WHERE schemaname = 'public' + AND tablename NOT LIKE 'pg_%' + """) + ).fetchall() + for table in tables: + conn.execute(text(f'DROP TABLE IF EXISTS public."{table[0]}" CASCADE')) + conn.commit() + engine.dispose() + + +@pytest.fixture(scope="function") +def db_url(): + """ + Provide a PostgreSQL connection URL for upgrade tests. + + Uses pg0 (embedded PostgreSQL) for a clean, isolated test database. + The database is cleaned between tests to ensure fresh state for migrations. + """ + url = _get_or_create_pg0() + + # Clean database before each test + _clean_database(url) + + yield url + + # No cleanup after - database is cleaned at start of next test + + +@pytest.fixture(scope="module") +def llm_config(): + """ + Provide LLM configuration from environment. + + Returns a dict with provider, api_key, and model. + """ + return { + "provider": os.getenv("HINDSIGHT_API_LLM_PROVIDER", "groq"), + "api_key": os.getenv("HINDSIGHT_API_LLM_API_KEY") or os.getenv("GROQ_API_KEY"), + "model": os.getenv("HINDSIGHT_API_LLM_MODEL", "llama-3.3-70b-versatile"), + } + + +@pytest.fixture +def unique_bank_id(): + """Generate a unique bank ID for each test.""" + import uuid + + return f"upgrade_test_{uuid.uuid4().hex[:8]}" diff --git a/hindsight-dev/upgrade_tests/test_upgrade.py b/hindsight-dev/upgrade_tests/test_upgrade.py new file mode 100644 index 00000000..96864e83 --- /dev/null +++ b/hindsight-dev/upgrade_tests/test_upgrade.py @@ -0,0 +1,303 @@ +""" +Upgrade and backwards compatibility tests. + +These tests verify that: +1. Data stored in older versions is accessible after upgrade +2. Database migrations run correctly +3. API behavior remains compatible +""" + +import logging + +import httpx +import pytest + +from .version_runner import VersionRunner + +logger = logging.getLogger(__name__) + +# Version upgrade paths to test +# Format: (old_version, new_version) +UPGRADE_PATHS = [ + ("v0.3.0", "HEAD"), +] + + +class TestUpgrade: + """Tests for version upgrades.""" + + @pytest.mark.parametrize("old_version,new_version", UPGRADE_PATHS) + def test_upgrade_preserves_memories(self, db_url, llm_config, unique_bank_id, old_version, new_version): + """ + Verify memories stored in old version are accessible after upgrade. + + Workflow: + 1. Start old version + 2. Store memories via retain + 3. Verify recall works on old version + 4. Stop old version + 5. Start new version (same database - migrations run) + 6. Verify recall returns same data + 7. Verify reflect works + """ + bank_id = unique_bank_id + + # Test data to store + test_memories = [ + {"content": "Alice is a software engineer at TechCorp.", "context": "team introduction"}, + {"content": "Bob manages the infrastructure team and loves Kubernetes.", "context": "team introduction"}, + {"content": "The project deadline is next Friday.", "context": "project planning"}, + ] + + # Phase 1: Store data with old version + logger.info(f"=== Phase 1: Setting up data with {old_version} ===") + + with VersionRunner( + old_version, + db_url, + port=8891, + llm_provider=llm_config["provider"], + llm_api_key=llm_config["api_key"], + llm_model=llm_config["model"], + ) as old: + server = old.start() + client = httpx.Client(base_url=server.url, timeout=60) + + # Store memories + resp = client.post( + f"/v1/default/banks/{bank_id}/memories", + json={"items": test_memories}, + ) + assert resp.status_code == 200, f"Failed to store memories: {resp.text}" + result = resp.json() + assert result["success"] is True + assert result["items_count"] == len(test_memories) + + # Verify recall works on old version + resp = client.post( + f"/v1/default/banks/{bank_id}/memories/recall", + json={"query": "Who works at TechCorp?"}, + ) + assert resp.status_code == 200, f"Recall failed on old version: {resp.text}" + old_results = resp.json()["results"] + assert len(old_results) > 0, "No results from recall on old version" + + # Get stats for comparison + resp = client.get(f"/v1/default/banks/{bank_id}/stats") + assert resp.status_code == 200 + old_stats = resp.json() + logger.info(f"Old version stats: {old_stats}") + + client.close() + + # Phase 2: Verify data with new version + logger.info(f"=== Phase 2: Verifying data with {new_version} ===") + + with VersionRunner( + new_version, + db_url, + port=8892, + llm_provider=llm_config["provider"], + llm_api_key=llm_config["api_key"], + llm_model=llm_config["model"], + ) as new: + server = new.start() + client = httpx.Client(base_url=server.url, timeout=60) + + # Verify recall returns data + resp = client.post( + f"/v1/default/banks/{bank_id}/memories/recall", + json={"query": "Who works at TechCorp?"}, + ) + assert resp.status_code == 200, f"Recall failed on new version: {resp.text}" + new_results = resp.json()["results"] + assert len(new_results) > 0, f"No results from recall after upgrade. Bank: {bank_id}" + + # Verify Alice is found + found_alice = any("Alice" in r.get("text", "") for r in new_results) + assert found_alice, f"Alice not found in results after upgrade: {new_results}" + + # Verify reflect works + resp = client.post( + f"/v1/default/banks/{bank_id}/reflect", + json={"query": "Tell me about the team members"}, + ) + assert resp.status_code == 200, f"Reflect failed after upgrade: {resp.text}" + reflect_result = resp.json() + assert len(reflect_result.get("text", "")) > 0, "Empty reflect response after upgrade" + + # Verify stats are preserved + resp = client.get(f"/v1/default/banks/{bank_id}/stats") + assert resp.status_code == 200 + new_stats = resp.json() + logger.info(f"New version stats: {new_stats}") + + # Stats should be similar (might have small differences due to re-indexing) + assert new_stats["total_nodes"] >= old_stats["total_nodes"], ( + f"Lost nodes after upgrade: {old_stats['total_nodes']} -> {new_stats['total_nodes']}" + ) + + # Cleanup - delete test bank + resp = client.delete(f"/v1/default/banks/{bank_id}") + assert resp.status_code == 200 + + client.close() + + @pytest.mark.parametrize("old_version,new_version", UPGRADE_PATHS) + def test_upgrade_preserves_documents(self, db_url, llm_config, unique_bank_id, old_version, new_version): + """ + Verify documents stored in old version are accessible after upgrade. + """ + bank_id = unique_bank_id + doc_id = "test-document-001" + + # Phase 1: Store document with old version + logger.info(f"=== Phase 1: Storing document with {old_version} ===") + + with VersionRunner( + old_version, + db_url, + port=8893, + llm_provider=llm_config["provider"], + llm_api_key=llm_config["api_key"], + llm_model=llm_config["model"], + ) as old: + server = old.start() + client = httpx.Client(base_url=server.url, timeout=60) + + # Store memory with document + resp = client.post( + f"/v1/default/banks/{bank_id}/memories", + json={ + "items": [ + { + "content": "The quarterly report shows 25% revenue growth.", + "context": "Q1 financial review", + "document_id": doc_id, + } + ] + }, + ) + assert resp.status_code == 200, f"Failed to store document: {resp.text}" + + # Verify document exists + resp = client.get(f"/v1/default/banks/{bank_id}/documents") + assert resp.status_code == 200 + docs = resp.json()["items"] + doc_ids = [d["id"] for d in docs] + assert doc_id in doc_ids, f"Document not found in old version: {doc_ids}" + + client.close() + + # Phase 2: Verify document with new version + logger.info(f"=== Phase 2: Verifying document with {new_version} ===") + + with VersionRunner( + new_version, + db_url, + port=8894, + llm_provider=llm_config["provider"], + llm_api_key=llm_config["api_key"], + llm_model=llm_config["model"], + ) as new: + server = new.start() + client = httpx.Client(base_url=server.url, timeout=60) + + # Verify document still exists + resp = client.get(f"/v1/default/banks/{bank_id}/documents") + assert resp.status_code == 200 + docs = resp.json()["items"] + doc_ids = [d["id"] for d in docs] + assert doc_id in doc_ids, f"Document not found after upgrade: {doc_ids}" + + # Verify document details + resp = client.get(f"/v1/default/banks/{bank_id}/documents/{doc_id}") + assert resp.status_code == 200 + doc_info = resp.json() + assert doc_info["id"] == doc_id + assert doc_info["memory_unit_count"] > 0 + + # Cleanup + resp = client.delete(f"/v1/default/banks/{bank_id}") + assert resp.status_code == 200 + + client.close() + + @pytest.mark.parametrize("old_version,new_version", UPGRADE_PATHS) + def test_upgrade_preserves_bank_profile(self, db_url, llm_config, unique_bank_id, old_version, new_version): + """ + Verify bank profile (disposition) is preserved after upgrade. + """ + bank_id = unique_bank_id + + # Phase 1: Create bank with custom disposition + logger.info(f"=== Phase 1: Creating bank profile with {old_version} ===") + + with VersionRunner( + old_version, + db_url, + port=8895, + llm_provider=llm_config["provider"], + llm_api_key=llm_config["api_key"], + llm_model=llm_config["model"], + ) as old: + server = old.start() + client = httpx.Client(base_url=server.url, timeout=60) + + # Create bank by storing a memory + resp = client.post( + f"/v1/default/banks/{bank_id}/memories", + json={"items": [{"content": "Test memory", "context": "test"}]}, + ) + assert resp.status_code == 200 + + # Set custom disposition + resp = client.put( + f"/v1/default/banks/{bank_id}/profile", + json={ + "disposition": { + "skepticism": 4, + "literalism": 2, + "empathy": 5, + } + }, + ) + assert resp.status_code == 200 + + # Verify profile + resp = client.get(f"/v1/default/banks/{bank_id}/profile") + assert resp.status_code == 200 + old_profile = resp.json() + assert old_profile["disposition"]["skepticism"] == 4 + assert old_profile["disposition"]["literalism"] == 2 + assert old_profile["disposition"]["empathy"] == 5 + + client.close() + + # Phase 2: Verify profile with new version + logger.info(f"=== Phase 2: Verifying profile with {new_version} ===") + + with VersionRunner( + new_version, + db_url, + port=8896, + llm_provider=llm_config["provider"], + llm_api_key=llm_config["api_key"], + llm_model=llm_config["model"], + ) as new: + server = new.start() + client = httpx.Client(base_url=server.url, timeout=60) + + # Verify profile is preserved + resp = client.get(f"/v1/default/banks/{bank_id}/profile") + assert resp.status_code == 200 + new_profile = resp.json() + assert new_profile["disposition"]["skepticism"] == 4, "Skepticism not preserved" + assert new_profile["disposition"]["literalism"] == 2, "Literalism not preserved" + assert new_profile["disposition"]["empathy"] == 5, "Empathy not preserved" + + # Cleanup + resp = client.delete(f"/v1/default/banks/{bank_id}") + assert resp.status_code == 200 + + client.close() diff --git a/hindsight-dev/upgrade_tests/version_runner.py b/hindsight-dev/upgrade_tests/version_runner.py new file mode 100644 index 00000000..41bc46a7 --- /dev/null +++ b/hindsight-dev/upgrade_tests/version_runner.py @@ -0,0 +1,275 @@ +""" +Version runner for upgrade tests. + +Manages running different git versions of the Hindsight API for upgrade testing. +Handles git checkout, venv creation, dependency installation, and server lifecycle. +""" + +import logging +import os +import shutil +import subprocess +import tempfile +import time +from dataclasses import dataclass +from pathlib import Path + +import httpx + +logger = logging.getLogger(__name__) + + +@dataclass +class ServerInfo: + """Information about a running server.""" + + url: str + port: int + version: str + + +class VersionRunner: + """ + Manages running a specific git version of the Hindsight API. + + For "HEAD" or "current", uses the current working directory. + For git tags (e.g., "v0.3.0"), clones the repo at that tag to a temp directory. + """ + + def __init__( + self, + version: str, + db_url: str, + port: int = 8890, + llm_provider: str | None = None, + llm_api_key: str | None = None, + llm_model: str | None = None, + ): + """ + Initialize a version runner. + + Args: + version: Git tag (e.g., "v0.3.0") or "HEAD"/"current" for current code + db_url: PostgreSQL connection URL + port: Port to run the API on + llm_provider: LLM provider (defaults to env var) + llm_api_key: LLM API key (defaults to env var) + llm_model: LLM model (defaults to env var) + """ + self.version = version + self.db_url = db_url + self.port = port + self.llm_provider = llm_provider or os.getenv("HINDSIGHT_API_LLM_PROVIDER", "groq") + self.llm_api_key = llm_api_key or os.getenv("HINDSIGHT_API_LLM_API_KEY") or os.getenv("GROQ_API_KEY") + self.llm_model = llm_model or os.getenv("HINDSIGHT_API_LLM_MODEL", "llama-3.3-70b-versatile") + + self.work_dir: Path | None = None + self.process: subprocess.Popen | None = None + self._temp_dir: str | None = None + self._is_current = version.lower() in ("head", "current") + + def _find_repo_root(self) -> Path: + """Find the git repository root.""" + result = subprocess.run( + ["git", "rev-parse", "--show-toplevel"], + capture_output=True, + text=True, + check=True, + ) + return Path(result.stdout.strip()) + + def setup(self) -> None: + """Checkout version and install dependencies.""" + if self._is_current: + # Use current working directory + self.work_dir = self._find_repo_root() + logger.info(f"Using current code at {self.work_dir}") + return + + # Create temp dir and checkout specific version + self._temp_dir = tempfile.mkdtemp(prefix=f"hindsight-{self.version}-") + self.work_dir = Path(self._temp_dir) + + repo_root = self._find_repo_root() + logger.info(f"Cloning {repo_root} at {self.version} to {self.work_dir}") + + # Shallow clone at specific tag + subprocess.run( + ["git", "clone", "--depth", "1", "--branch", self.version, str(repo_root), str(self.work_dir)], + check=True, + capture_output=True, + ) + + # Create venv and install + venv_path = self.work_dir / ".venv-upgrade-test" + logger.info(f"Creating venv at {venv_path}") + + subprocess.run(["uv", "venv", str(venv_path)], check=True, capture_output=True) + + api_path = self.work_dir / "hindsight-api" + logger.info(f"Installing hindsight-api from {api_path}") + + # Install with uv pip - use --index-strategy for pytorch + subprocess.run( + [ + "uv", + "pip", + "install", + "-e", + str(api_path), + "--python", + str(venv_path / "bin" / "python"), + "--index-strategy", + "unsafe-best-match", + ], + check=True, + capture_output=True, + env={**os.environ, "UV_INDEX": "pytorch=https://download.pytorch.org/whl/cpu"}, + ) + + logger.info(f"Version {self.version} setup complete") + + def _get_venv_path(self) -> Path: + """Get the path to the venv for this version.""" + if self._is_current: + # For current code, the venv is at the workspace root (uv workspace layout) + # Check both possible locations + workspace_venv = self.work_dir / ".venv" + api_venv = self.work_dir / "hindsight-api" / ".venv" + + if (workspace_venv / "bin" / "hindsight-api").exists(): + return workspace_venv + elif (api_venv / "bin" / "hindsight-api").exists(): + return api_venv + else: + # Default to workspace root + return workspace_venv + return self.work_dir / ".venv-upgrade-test" + + def start(self) -> ServerInfo: + """ + Start the API server. + + Returns: + ServerInfo with the URL and port + """ + venv_path = self._get_venv_path() + hindsight_api_bin = venv_path / "bin" / "hindsight-api" + + if not hindsight_api_bin.exists(): + raise RuntimeError(f"hindsight-api binary not found at {hindsight_api_bin}") + + env = os.environ.copy() + env.update( + { + "HINDSIGHT_API_PORT": str(self.port), + "HINDSIGHT_API_DATABASE_URL": self.db_url, + "HINDSIGHT_API_HOST": "127.0.0.1", + "HINDSIGHT_API_LLM_PROVIDER": self.llm_provider, + "HINDSIGHT_API_LLM_API_KEY": self.llm_api_key or "", + "HINDSIGHT_API_LLM_MODEL": self.llm_model, + "PYTHONUNBUFFERED": "1", + } + ) + + logger.info(f"Starting {self.version} API on port {self.port}") + logger.info(f"Database URL: {self.db_url}") + + # Determine working directory + # For HEAD/current, use a temp directory to avoid .env file from workspace root + # (hindsight-api loads .env with override=True which would override our env vars) + if self._is_current: + # Create a temp directory for HEAD to avoid workspace .env + self._head_cwd = tempfile.mkdtemp(prefix="hindsight-head-cwd-") + cwd = self._head_cwd + else: + cwd = str(self.work_dir) + self._head_cwd = None + + # Start the server + self.process = subprocess.Popen( + [str(hindsight_api_bin)], + env=env, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + cwd=cwd, + ) + + self._wait_healthy() + + url = f"http://127.0.0.1:{self.port}" + logger.info(f"Server {self.version} ready at {url}") + + return ServerInfo(url=url, port=self.port, version=self.version) + + def _wait_healthy(self, timeout: int = 120) -> None: + """Wait for /health endpoint to respond.""" + url = f"http://127.0.0.1:{self.port}/health" + deadline = time.time() + timeout + + while time.time() < deadline: + # Check if process is still alive + if self.process and self.process.poll() is not None: + stdout = self.process.stdout.read().decode() if self.process.stdout else "" + raise RuntimeError(f"Server {self.version} exited unexpectedly.\nLogs:\n{stdout}") + + try: + resp = httpx.get(url, timeout=2) + if resp.status_code == 200: + return + except httpx.RequestError: + pass + + time.sleep(1) + + # Timeout - dump logs + if self.process: + self.process.terminate() + try: + stdout, _ = self.process.communicate(timeout=5) + logs = stdout.decode() if stdout else "" + except Exception: + logs = "(failed to read logs)" + raise TimeoutError(f"Server {self.version} not healthy after {timeout}s.\nLogs:\n{logs}") + + def stop(self) -> None: + """Stop the server and cleanup temp directory.""" + if self.process: + logger.info(f"Stopping {self.version} server") + self.process.terminate() + try: + self.process.wait(timeout=10) + except subprocess.TimeoutExpired: + logger.warning(f"Server {self.version} did not stop gracefully, killing") + self.process.kill() + self.process.wait() + self.process = None + + if self._temp_dir and os.path.exists(self._temp_dir): + logger.info(f"Cleaning up {self._temp_dir}") + shutil.rmtree(self._temp_dir, ignore_errors=True) + self._temp_dir = None + + # Clean up HEAD's temp cwd + if hasattr(self, "_head_cwd") and self._head_cwd and os.path.exists(self._head_cwd): + shutil.rmtree(self._head_cwd, ignore_errors=True) + self._head_cwd = None + + def get_logs(self) -> str: + """Get current server logs (if process is running).""" + if self.process and self.process.stdout: + # Non-blocking read of available output + import select + + if hasattr(select, "select"): + readable, _, _ = select.select([self.process.stdout], [], [], 0) + if readable: + return self.process.stdout.read(4096).decode() + return "" + + def __enter__(self) -> "VersionRunner": + self.setup() + return self + + def __exit__(self, *args) -> None: + self.stop() diff --git a/scripts/run-upgrade-tests.sh b/scripts/run-upgrade-tests.sh new file mode 100755 index 00000000..88d84603 --- /dev/null +++ b/scripts/run-upgrade-tests.sh @@ -0,0 +1,39 @@ +#!/bin/bash +# +# Run upgrade tests locally +# +# Usage: +# ./scripts/run-upgrade-tests.sh +# +# Environment variables: +# HINDSIGHT_API_LLM_PROVIDER - LLM provider (default: groq) +# HINDSIGHT_API_LLM_API_KEY - LLM API key (or GROQ_API_KEY) +# HINDSIGHT_API_LLM_MODEL - LLM model +# + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +ROOT_DIR="$(cd "$SCRIPT_DIR/.." && pwd)" + +echo "=== Running Hindsight Upgrade Tests ===" +echo "" + +# Check for LLM API key +if [ -z "${HINDSIGHT_API_LLM_API_KEY:-}" ] && [ -z "${GROQ_API_KEY:-}" ]; then + echo "Warning: No LLM API key found. Set HINDSIGHT_API_LLM_API_KEY or GROQ_API_KEY" +fi + +# Load .env if present +if [ -f "$ROOT_DIR/.env" ]; then + echo "Loading .env file..." + set -a + source "$ROOT_DIR/.env" + set +a +fi + +cd "$ROOT_DIR/hindsight-dev" + +# Run tests +echo "Running upgrade tests..." +uv run pytest upgrade_tests/ -v "$@" diff --git a/uv.lock b/uv.lock index 4f865f54..1803b558 100644 --- a/uv.lock +++ b/uv.lock @@ -1492,6 +1492,13 @@ dependencies = [ { name = "streamlit" }, ] +[package.optional-dependencies] +test = [ + { name = "httpx" }, + { name = "pytest" }, + { name = "python-dotenv" }, +] + [package.dev-dependencies] dev = [ { name = "ruff" }, @@ -1501,12 +1508,16 @@ dev = [ [package.metadata] requires-dist = [ { name = "hindsight-api", editable = "hindsight-api" }, + { name = "httpx", marker = "extra == 'test'", specifier = ">=0.27.0" }, { name = "openai", specifier = ">=1.0.0" }, { name = "pydantic", specifier = ">=2.0.0" }, + { name = "pytest", marker = "extra == 'test'", specifier = ">=8.0.0" }, + { name = "python-dotenv", marker = "extra == 'test'", specifier = ">=1.0.0" }, { name = "python-fasthtml", specifier = ">=0.12.33" }, { name = "rich", specifier = ">=13.0.0" }, { name = "streamlit", specifier = ">=1.51.0" }, ] +provides-extras = ["test"] [package.metadata.requires-dev] dev = [