fleet-memory/docker/standalone/start-all.sh

#!/bin/bash
set -e

# =============================================================================
# Embedded pg0 data integrity check (#675)
#
# When using embedded pg0, check if the data directory has existing PostgreSQL
# data before starting. If the directory exists but appears empty/corrupt
# (e.g., missing PG_VERSION file), log a warning. This helps diagnose data
# loss scenarios where a container restart caused the data directory to be
# wiped despite a volume mount being present.
# =============================================================================
PG0_DATA_DIR="${HOME}/.pg0"
if [ -d "$PG0_DATA_DIR" ]; then
    # Look for actual PostgreSQL data directories (pg0 creates subdirs per instance)
    if compgen -G "$PG0_DATA_DIR"/*/PG_VERSION > /dev/null 2>&1; then
        echo "✅ Existing pg0 data directory detected at $PG0_DATA_DIR"
    elif [ "$(ls -A "$PG0_DATA_DIR" 2>/dev/null)" ]; then
        echo "⚠️  WARNING: pg0 data directory exists at $PG0_DATA_DIR but no PG_VERSION found."
        echo "   This may indicate data corruption or an incomplete previous shutdown."
        echo "   If you see all migrations running from scratch after this, your data may have been lost."
        echo "   See: https://github.com/vectorize-io/hindsight/issues/675"
    fi
fi

# Service flags (default to true if not set)
ENABLE_API="${HINDSIGHT_ENABLE_API:-true}"
ENABLE_CP="${HINDSIGHT_ENABLE_CP:-true}"

# =============================================================================
# Dependency waiting (opt-in via HINDSIGHT_WAIT_FOR_DEPS=true)
#
# Problem: When running with LM Studio, the LLM may take time to load models.
# If Hindsight starts before LM Studio is ready, it fails on LLM verification.
# This wait loop ensures dependencies are ready before starting.
# =============================================================================
if [ "${HINDSIGHT_WAIT_FOR_DEPS:-false}" = "true" ]; then
    LLM_BASE_URL="${HINDSIGHT_API_LLM_BASE_URL:-http://host.docker.internal:1234/v1}"
    MAX_RETRIES="${HINDSIGHT_RETRY_MAX:-0}"  # 0 = infinite
    RETRY_INTERVAL="${HINDSIGHT_RETRY_INTERVAL:-10}"

    # Check if external database is configured (skip check for embedded pg0)
    SKIP_DB_CHECK=false
    if [ -z "${HINDSIGHT_API_DATABASE_URL}" ]; then
        SKIP_DB_CHECK=true
    else
        DB_CHECK_HOST=$(echo "$HINDSIGHT_API_DATABASE_URL" | sed -E 's|.*@([^:/]+):([0-9]+)/.*|\1 \2|')
    fi

    check_db() {
        if $SKIP_DB_CHECK; then
            return 0
        fi
        if command -v pg_isready &> /dev/null; then
            pg_isready -h $(echo $DB_CHECK_HOST | cut -d' ' -f1) -p $(echo $DB_CHECK_HOST | cut -d' ' -f2) &>/dev/null
        else
            python3 -c "import socket; s=socket.socket(); s.settimeout(5); exit(0 if s.connect_ex(('$(echo $DB_CHECK_HOST | cut -d' ' -f1)', $(echo $DB_CHECK_HOST | cut -d' ' -f2))) == 0 else 1)" 2>/dev/null
        fi
    }

    check_llm() {
        curl -sf "${LLM_BASE_URL}/models" --connect-timeout 5 &>/dev/null
    }

    echo "⏳ Waiting for dependencies to be ready..."
    attempt=1

    while true; do
        db_ok=false
        llm_ok=false

        if check_db; then
            db_ok=true
        fi

        if check_llm; then
            llm_ok=true
        fi

        if $db_ok && $llm_ok; then
            echo "✅ Dependencies ready!"
            break
        fi

        if [ "$MAX_RETRIES" -ne 0 ] && [ "$attempt" -ge "$MAX_RETRIES" ]; then
            echo "❌ Max retries ($MAX_RETRIES) reached. Dependencies not available."
            exit 1
        fi

        echo "   Attempt $attempt: DB=$( $db_ok && echo 'ok' || echo 'waiting' ), LLM=$( $llm_ok && echo 'ok' || echo 'waiting' )"
        sleep "$RETRY_INTERVAL"
        ((attempt++))
    done
fi

# =============================================================================
# Graceful shutdown handler (#675)
#
# Docker sends SIGTERM on `docker stop`/`docker restart`. Without a trap, child
# processes (hindsight-api + pg0, control-plane) are killed abruptly. For the
# embedded pg0 database this can cause data loss when the data directory is on
# a Docker volume that gets remounted after restart.
#
# The trap forwards SIGTERM to all tracked child PIDs so that:
#   - hindsight-api receives the signal and can run its shutdown hooks
#   - pg0 gets a clean PostgreSQL shutdown (checkpoint + WAL flush)
#   - The control-plane Node.js process exits cleanly
# =============================================================================
# Guard against concurrent cleanup (e.g., child crash + SIGTERM arriving together)
SHUTTING_DOWN=false

cleanup() {
    if $SHUTTING_DOWN; then return; fi
    SHUTTING_DOWN=true

    echo ""
    echo "🛑 Received shutdown signal, stopping services gracefully..."
    for pid in "${PIDS[@]}"; do
        if kill -0 "$pid" 2>/dev/null; then
            kill -TERM "$pid" 2>/dev/null
        fi
    done
    # Give processes time to shut down cleanly (pg0 needs to flush WAL).
    # NOTE: Docker's default stop_grace_period is 10s. If you use the default,
    # either set stop_grace_period: 30s in your compose file / docker stop -t 30,
    # or Docker will SIGKILL the container before this timeout expires.
    local timeout=30
    for ((i=1; i<=timeout; i++)); do
        local all_stopped=true
        for pid in "${PIDS[@]}"; do
            if kill -0 "$pid" 2>/dev/null; then
                all_stopped=false
                break
            fi
        done
        if $all_stopped; then
            echo "✅ All services stopped cleanly"
            exit 0
        fi
        sleep 1
    done
    # Force kill if still running after timeout
    echo "⚠️  Timeout reached, forcing shutdown..."
    for pid in "${PIDS[@]}"; do
        if kill -0 "$pid" 2>/dev/null; then
            kill -9 "$pid" 2>/dev/null
        fi
    done
    exit 1
}
trap cleanup SIGTERM SIGINT

# Track PIDs for wait
PIDS=()

# Start API if enabled
if [ "$ENABLE_API" = "true" ]; then
    cd /app/api
    # Health probe must follow HINDSIGHT_API_PORT — hardcoding 8888 made the
    # readiness check hit a dead port whenever the API was bound elsewhere,
    # timing out after API_STARTUP_WAIT_SECONDS and exiting 1 → crash loop.
    API_HEALTH_URL="${HINDSIGHT_API_HEALTH_URL:-http://localhost:${HINDSIGHT_API_PORT:-8888}/health}"
    API_STARTUP_WAIT_SECONDS="${HINDSIGHT_API_STARTUP_WAIT_SECONDS:-300}"

    # Run API directly - Python's PYTHONUNBUFFERED=1 handles output buffering
    hindsight-api &
    API_PID=$!
    PIDS+=($API_PID)

    # Wait for API to be ready
    api_ready=false
    for ((i=1; i<=API_STARTUP_WAIT_SECONDS; i++)); do
        if ! kill -0 "$API_PID" 2>/dev/null; then
            wait "$API_PID"
            exit $?
        fi
        if curl -sf "$API_HEALTH_URL" &>/dev/null; then
            api_ready=true
            break
        fi
        sleep 1
    done

    if [ "$api_ready" != "true" ]; then
        echo "❌ API did not become healthy within ${API_STARTUP_WAIT_SECONDS}s"
        exit 1
    fi
else
    echo "API disabled (HINDSIGHT_ENABLE_API=false)"
fi

# Start Control Plane if enabled
if [ "$ENABLE_CP" = "true" ]; then
    echo "🎛️  Starting Control Plane..."
    cd /app/control-plane
    export HOSTNAME="${HINDSIGHT_CP_HOSTNAME:-0.0.0.0}"
    PORT="${HINDSIGHT_CP_PORT:-9999}" node server.js &
    CP_PID=$!
    PIDS+=($CP_PID)
else
    echo "Control Plane disabled (HINDSIGHT_ENABLE_CP=false)"
fi

# Print status
echo ""
echo "✅ Hindsight is running!"
echo ""
echo "📍 Access:"
if [ "$ENABLE_CP" = "true" ]; then
    echo "   Control Plane: http://localhost:${HINDSIGHT_CP_PORT:-9999}"
fi
if [ "$ENABLE_API" = "true" ]; then
    echo "   API:           http://localhost:${HINDSIGHT_API_PORT:-8888}"
fi
echo ""

# Check if any services are running
if [ ${#PIDS[@]} -eq 0 ]; then
    echo "❌ No services enabled! Set HINDSIGHT_ENABLE_API=true or HINDSIGHT_ENABLE_CP=true"
    exit 1
fi

# Wait for any process to exit (use wait -n with trap-safe loop)
while true; do
    # wait -n returns when any child exits; it also returns on signal delivery
    # (the trap handler will run and exit, so this loop is just for robustness).
    # `&& true` prevents `set -e` from killing the script when wait -n returns
    # non-zero (child exited with error or no backgrounded children remain).
    wait -n && true
    # Check if any tracked PID has exited
    for pid in "${PIDS[@]}"; do
        if ! kill -0 "$pid" 2>/dev/null; then
            wait "$pid" 2>/dev/null
            exit_code=$?
            echo "⚠️  Service (PID $pid) exited with code $exit_code"
            # Trigger cleanup for remaining services
            cleanup
        fi
    done
done