From ae808766718b5d6deec705f0b9bb0e0ff79bc441 Mon Sep 17 00:00:00 2001 From: Chris Bartholomew Date: Wed, 17 Dec 2025 16:01:53 -0500 Subject: [PATCH] fix: add procps to Docker image and smoke test to release workflow (#45) * fix: add procps to Docker image and smoke test to release workflow The Docker image was failing to start because pg0 uses `kill -0 ` to check if PostgreSQL is running, but the python:3.11-slim base image doesn't include the `kill` command. Adding procps provides it. This has been broken since release 0.1.6 when the fallback URI code was removed to support dynamic ports. Without the kill command, pg0 couldn't detect process status and returned None for the database URI. Also adds smoke testing to the release workflow: - Build image locally (single platform) and test before pushing - Run container and wait for /health endpoint (up to 120s) - Only push multi-platform release images if smoke test passes - Each image (api-only, cp-only, standalone) tested independently This prevents releasing broken Docker images to GHCR. * refactor: extract smoke test into reusable script Add scripts/docker-smoke-test.sh that can be run locally or in CI: - Takes image name and optional target (cp-only vs api) - Handles LLM credentials for API/standalone images - Configurable timeout via SMOKE_TEST_TIMEOUT env var - Colored output and clear error messages - Proper cleanup on exit Update release workflow to use the script instead of inline bash. --- .github/workflows/release.yml | 28 +++++- docker/standalone/Dockerfile | 6 +- scripts/docker-smoke-test.sh | 167 ++++++++++++++++++++++++++++++++++ 3 files changed, 197 insertions(+), 4 deletions(-) create mode 100755 scripts/docker-smoke-test.sh diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index ebe2940a..3c919479 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -206,7 +206,7 @@ jobs: id: get_version run: echo "VERSION=${GITHUB_REF#refs/tags/v}" >> $GITHUB_OUTPUT - - name: Extract metadata + - name: Extract metadata for release tags id: meta uses: docker/metadata-action@v5 with: @@ -217,7 +217,29 @@ jobs: type=semver,pattern={{major}},value=${{ steps.get_version.outputs.VERSION }} type=raw,value=latest - - name: Build and push + # Step 1: Build for local testing (single platform, no push) + # This creates an identical image to what will be released, just for one platform + - name: Build image for testing + uses: docker/build-push-action@v6 + with: + context: . + file: docker/standalone/Dockerfile + target: ${{ matrix.target }} + push: false + load: true + tags: ${{ matrix.image_name }}:test + cache-from: type=gha + cache-to: type=gha,mode=max + + # Step 2: Test the image before pushing anything + - name: Smoke test - verify container starts + env: + GROQ_API_KEY: ${{ secrets.GROQ_API_KEY }} + run: ./scripts/docker-smoke-test.sh "${{ matrix.image_name }}:test" "${{ matrix.target }}" + + # Step 3: Only if smoke test passed, build multi-platform and push to release tags + # Build layers are cached, so this is fast - just builds the other platform + - name: Build and push release images uses: docker/build-push-action@v6 with: context: . @@ -227,6 +249,8 @@ jobs: platforms: linux/amd64,linux/arm64 tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} + cache-from: type=gha + cache-to: type=gha,mode=max release-helm-chart: runs-on: ubuntu-latest diff --git a/docker/standalone/Dockerfile b/docker/standalone/Dockerfile index 44a9e0a3..b2801ed5 100644 --- a/docker/standalone/Dockerfile +++ b/docker/standalone/Dockerfile @@ -104,9 +104,10 @@ FROM python:3.11-slim AS api-only WORKDIR /app -# Install pg0 dependencies +# Install pg0 dependencies (procps provides 'kill' command needed by pg0) RUN apt-get update && apt-get install -y \ curl \ + procps \ libxml2 \ libssl3 \ libgssapi-krb5-2 \ @@ -200,9 +201,10 @@ FROM python:3.11-slim AS standalone WORKDIR /app -# Install Node.js, curl, uv, and pg0 dependencies +# Install Node.js, curl, uv, and pg0 dependencies (procps provides 'kill' command needed by pg0) RUN apt-get update && apt-get install -y \ curl \ + procps \ libxml2 \ libssl3 \ libgssapi-krb5-2 \ diff --git a/scripts/docker-smoke-test.sh b/scripts/docker-smoke-test.sh new file mode 100755 index 00000000..8f9b4d25 --- /dev/null +++ b/scripts/docker-smoke-test.sh @@ -0,0 +1,167 @@ +#!/bin/bash +# +# Docker Smoke Test Script +# +# Tests that a Hindsight Docker image starts correctly and becomes healthy. +# Can be run locally or in CI pipelines. +# +# Usage: +# ./scripts/docker-smoke-test.sh [target] +# +# Arguments: +# image - Docker image to test (e.g., hindsight-api:test, ghcr.io/vectorize-io/hindsight:latest) +# target - Optional: 'cp-only' for control plane, otherwise assumes API image (default: api) +# +# Environment variables: +# GROQ_API_KEY - Required for API/standalone images (LLM verification) +# HINDSIGHT_API_LLM_PROVIDER - LLM provider (default: groq) +# HINDSIGHT_API_LLM_MODEL - LLM model (default: llama-3.3-70b-versatile) +# SMOKE_TEST_TIMEOUT - Timeout in seconds (default: 120) +# SMOKE_TEST_CONTAINER_NAME - Container name (default: hindsight-smoke-test) +# +# Examples: +# # Test a locally built image +# ./scripts/docker-smoke-test.sh hindsight-api:test +# +# # Test a released image +# ./scripts/docker-smoke-test.sh ghcr.io/vectorize-io/hindsight:latest +# +# # Test control plane image +# ./scripts/docker-smoke-test.sh hindsight-control-plane:test cp-only +# +# Exit codes: +# 0 - Success (container healthy) +# 1 - Failure (container not healthy within timeout) +# 2 - Invalid arguments +# + +set -euo pipefail + +# Colors for output +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[0;33m' +NC='\033[0m' # No Color + +# Configuration +IMAGE="${1:-}" +TARGET="${2:-api}" +TIMEOUT="${SMOKE_TEST_TIMEOUT:-120}" +CONTAINER_NAME="${SMOKE_TEST_CONTAINER_NAME:-hindsight-smoke-test}" +LLM_PROVIDER="${HINDSIGHT_API_LLM_PROVIDER:-groq}" +LLM_MODEL="${HINDSIGHT_API_LLM_MODEL:-llama-3.3-70b-versatile}" + +# Validate arguments +if [ -z "$IMAGE" ]; then + echo -e "${RED}Error: Image argument is required${NC}" + echo "" + echo "Usage: $0 [target]" + echo "" + echo "Examples:" + echo " $0 hindsight-api:test" + echo " $0 ghcr.io/vectorize-io/hindsight:latest" + echo " $0 hindsight-control-plane:test cp-only" + exit 2 +fi + +# Determine health endpoint based on target +if [ "$TARGET" = "cp-only" ]; then + HEALTH_PORT=9999 + NEEDS_LLM=false +else + HEALTH_PORT=8888 + NEEDS_LLM=true +fi + +# Check for required environment variables +if [ "$NEEDS_LLM" = true ] && [ -z "${GROQ_API_KEY:-}" ]; then + echo -e "${RED}Error: GROQ_API_KEY environment variable is required for API/standalone images${NC}" + echo "Set it with: export GROQ_API_KEY=your-api-key" + exit 2 +fi + +# Cleanup function +cleanup() { + echo "Cleaning up..." + docker stop "$CONTAINER_NAME" 2>/dev/null || true + docker rm "$CONTAINER_NAME" 2>/dev/null || true +} + +# Set trap to cleanup on exit +trap cleanup EXIT + +echo -e "${YELLOW}Starting smoke test for: ${IMAGE}${NC}" +echo " Target: $TARGET" +echo " Health port: $HEALTH_PORT" +echo " Timeout: ${TIMEOUT}s" +echo "" + +# Remove any existing container with the same name +docker rm -f "$CONTAINER_NAME" 2>/dev/null || true + +# Start container based on target type +echo "Starting container..." +if [ "$TARGET" = "cp-only" ]; then + docker run -d --name "$CONTAINER_NAME" \ + -p "${HEALTH_PORT}:${HEALTH_PORT}" \ + "$IMAGE" +else + docker run -d --name "$CONTAINER_NAME" \ + -e HINDSIGHT_API_LLM_PROVIDER="$LLM_PROVIDER" \ + -e HINDSIGHT_API_LLM_API_KEY="${GROQ_API_KEY}" \ + -e HINDSIGHT_API_LLM_MODEL="$LLM_MODEL" \ + -p "${HEALTH_PORT}:${HEALTH_PORT}" \ + "$IMAGE" +fi + +# Wait for health endpoint +echo "Waiting for health endpoint on port ${HEALTH_PORT}..." +start_time=$(date +%s) + +for i in $(seq 1 "$TIMEOUT"); do + if curl -sf "http://localhost:${HEALTH_PORT}/health" > /dev/null 2>&1; then + end_time=$(date +%s) + duration=$((end_time - start_time)) + echo "" + echo -e "${GREEN}Container is healthy after ${duration}s${NC}" + echo "" + echo "=== Health Response ===" + curl -s "http://localhost:${HEALTH_PORT}/health" | python3 -m json.tool 2>/dev/null || curl -s "http://localhost:${HEALTH_PORT}/health" + echo "" + echo "" + echo "=== Container Logs (last 50 lines) ===" + docker logs "$CONTAINER_NAME" 2>&1 | tail -50 + echo "" + echo -e "${GREEN}Smoke test PASSED${NC}" + exit 0 + fi + + # Show progress every 10 seconds + if [ $((i % 10)) -eq 0 ]; then + echo " Still waiting... (${i}s)" + fi + + # Check if container is still running + if ! docker ps -q -f "name=$CONTAINER_NAME" | grep -q .; then + echo "" + echo -e "${RED}Container exited unexpectedly!${NC}" + echo "" + echo "=== Container Logs ===" + docker logs "$CONTAINER_NAME" 2>&1 + echo "" + echo -e "${RED}Smoke test FAILED${NC}" + exit 1 + fi + + sleep 1 +done + +# Timeout reached +echo "" +echo -e "${RED}Container failed to become healthy after ${TIMEOUT}s${NC}" +echo "" +echo "=== Container Logs ===" +docker logs "$CONTAINER_NAME" 2>&1 +echo "" +echo -e "${RED}Smoke test FAILED${NC}" +exit 1