fix: add procps to Docker image and smoke test to release workflow (#45)
* fix: add procps to Docker image and smoke test to release workflow The Docker image was failing to start because pg0 uses `kill -0 <pid>` to check if PostgreSQL is running, but the python:3.11-slim base image doesn't include the `kill` command. Adding procps provides it. This has been broken since release 0.1.6 when the fallback URI code was removed to support dynamic ports. Without the kill command, pg0 couldn't detect process status and returned None for the database URI. Also adds smoke testing to the release workflow: - Build image locally (single platform) and test before pushing - Run container and wait for /health endpoint (up to 120s) - Only push multi-platform release images if smoke test passes - Each image (api-only, cp-only, standalone) tested independently This prevents releasing broken Docker images to GHCR. * refactor: extract smoke test into reusable script Add scripts/docker-smoke-test.sh that can be run locally or in CI: - Takes image name and optional target (cp-only vs api) - Handles LLM credentials for API/standalone images - Configurable timeout via SMOKE_TEST_TIMEOUT env var - Colored output and clear error messages - Proper cleanup on exit Update release workflow to use the script instead of inline bash.
This commit is contained in:
Chris Bartholomew
GitHub
parent
476a62da47
commit
ae80876671
3 changed files with 197 additions and 4 deletions
28
.github/workflows/release.yml
vendored
28
.github/workflows/release.yml
vendored
|
|
@ -206,7 +206,7 @@ jobs:
|
|||
id: get_version
|
||||
run: echo "VERSION=${GITHUB_REF#refs/tags/v}" >> $GITHUB_OUTPUT
|
||||
|
||||
- name: Extract metadata
|
||||
- name: Extract metadata for release tags
|
||||
id: meta
|
||||
uses: docker/metadata-action@v5
|
||||
with:
|
||||
|
|
@ -217,7 +217,29 @@ jobs:
|
|||
type=semver,pattern={{major}},value=${{ steps.get_version.outputs.VERSION }}
|
||||
type=raw,value=latest
|
||||
|
||||
- name: Build and push
|
||||
# Step 1: Build for local testing (single platform, no push)
|
||||
# This creates an identical image to what will be released, just for one platform
|
||||
- name: Build image for testing
|
||||
uses: docker/build-push-action@v6
|
||||
with:
|
||||
context: .
|
||||
file: docker/standalone/Dockerfile
|
||||
target: ${{ matrix.target }}
|
||||
push: false
|
||||
load: true
|
||||
tags: ${{ matrix.image_name }}:test
|
||||
cache-from: type=gha
|
||||
cache-to: type=gha,mode=max
|
||||
|
||||
# Step 2: Test the image before pushing anything
|
||||
- name: Smoke test - verify container starts
|
||||
env:
|
||||
GROQ_API_KEY: ${{ secrets.GROQ_API_KEY }}
|
||||
run: ./scripts/docker-smoke-test.sh "${{ matrix.image_name }}:test" "${{ matrix.target }}"
|
||||
|
||||
# Step 3: Only if smoke test passed, build multi-platform and push to release tags
|
||||
# Build layers are cached, so this is fast - just builds the other platform
|
||||
- name: Build and push release images
|
||||
uses: docker/build-push-action@v6
|
||||
with:
|
||||
context: .
|
||||
|
|
@ -227,6 +249,8 @@ jobs:
|
|||
platforms: linux/amd64,linux/arm64
|
||||
tags: ${{ steps.meta.outputs.tags }}
|
||||
labels: ${{ steps.meta.outputs.labels }}
|
||||
cache-from: type=gha
|
||||
cache-to: type=gha,mode=max
|
||||
|
||||
release-helm-chart:
|
||||
runs-on: ubuntu-latest
|
||||
|
|
|
|||
|
|
@ -104,9 +104,10 @@ FROM python:3.11-slim AS api-only
|
|||
|
||||
WORKDIR /app
|
||||
|
||||
# Install pg0 dependencies
|
||||
# Install pg0 dependencies (procps provides 'kill' command needed by pg0)
|
||||
RUN apt-get update && apt-get install -y \
|
||||
curl \
|
||||
procps \
|
||||
libxml2 \
|
||||
libssl3 \
|
||||
libgssapi-krb5-2 \
|
||||
|
|
@ -200,9 +201,10 @@ FROM python:3.11-slim AS standalone
|
|||
|
||||
WORKDIR /app
|
||||
|
||||
# Install Node.js, curl, uv, and pg0 dependencies
|
||||
# Install Node.js, curl, uv, and pg0 dependencies (procps provides 'kill' command needed by pg0)
|
||||
RUN apt-get update && apt-get install -y \
|
||||
curl \
|
||||
procps \
|
||||
libxml2 \
|
||||
libssl3 \
|
||||
libgssapi-krb5-2 \
|
||||
|
|
|
|||
167
scripts/docker-smoke-test.sh
Executable file
167
scripts/docker-smoke-test.sh
Executable file
|
|
@ -0,0 +1,167 @@
|
|||
#!/bin/bash
|
||||
#
|
||||
# Docker Smoke Test Script
|
||||
#
|
||||
# Tests that a Hindsight Docker image starts correctly and becomes healthy.
|
||||
# Can be run locally or in CI pipelines.
|
||||
#
|
||||
# Usage:
|
||||
# ./scripts/docker-smoke-test.sh <image> [target]
|
||||
#
|
||||
# Arguments:
|
||||
# image - Docker image to test (e.g., hindsight-api:test, ghcr.io/vectorize-io/hindsight:latest)
|
||||
# target - Optional: 'cp-only' for control plane, otherwise assumes API image (default: api)
|
||||
#
|
||||
# Environment variables:
|
||||
# GROQ_API_KEY - Required for API/standalone images (LLM verification)
|
||||
# HINDSIGHT_API_LLM_PROVIDER - LLM provider (default: groq)
|
||||
# HINDSIGHT_API_LLM_MODEL - LLM model (default: llama-3.3-70b-versatile)
|
||||
# SMOKE_TEST_TIMEOUT - Timeout in seconds (default: 120)
|
||||
# SMOKE_TEST_CONTAINER_NAME - Container name (default: hindsight-smoke-test)
|
||||
#
|
||||
# Examples:
|
||||
# # Test a locally built image
|
||||
# ./scripts/docker-smoke-test.sh hindsight-api:test
|
||||
#
|
||||
# # Test a released image
|
||||
# ./scripts/docker-smoke-test.sh ghcr.io/vectorize-io/hindsight:latest
|
||||
#
|
||||
# # Test control plane image
|
||||
# ./scripts/docker-smoke-test.sh hindsight-control-plane:test cp-only
|
||||
#
|
||||
# Exit codes:
|
||||
# 0 - Success (container healthy)
|
||||
# 1 - Failure (container not healthy within timeout)
|
||||
# 2 - Invalid arguments
|
||||
#
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
# Colors for output
|
||||
RED='\033[0;31m'
|
||||
GREEN='\033[0;32m'
|
||||
YELLOW='\033[0;33m'
|
||||
NC='\033[0m' # No Color
|
||||
|
||||
# Configuration
|
||||
IMAGE="${1:-}"
|
||||
TARGET="${2:-api}"
|
||||
TIMEOUT="${SMOKE_TEST_TIMEOUT:-120}"
|
||||
CONTAINER_NAME="${SMOKE_TEST_CONTAINER_NAME:-hindsight-smoke-test}"
|
||||
LLM_PROVIDER="${HINDSIGHT_API_LLM_PROVIDER:-groq}"
|
||||
LLM_MODEL="${HINDSIGHT_API_LLM_MODEL:-llama-3.3-70b-versatile}"
|
||||
|
||||
# Validate arguments
|
||||
if [ -z "$IMAGE" ]; then
|
||||
echo -e "${RED}Error: Image argument is required${NC}"
|
||||
echo ""
|
||||
echo "Usage: $0 <image> [target]"
|
||||
echo ""
|
||||
echo "Examples:"
|
||||
echo " $0 hindsight-api:test"
|
||||
echo " $0 ghcr.io/vectorize-io/hindsight:latest"
|
||||
echo " $0 hindsight-control-plane:test cp-only"
|
||||
exit 2
|
||||
fi
|
||||
|
||||
# Determine health endpoint based on target
|
||||
if [ "$TARGET" = "cp-only" ]; then
|
||||
HEALTH_PORT=9999
|
||||
NEEDS_LLM=false
|
||||
else
|
||||
HEALTH_PORT=8888
|
||||
NEEDS_LLM=true
|
||||
fi
|
||||
|
||||
# Check for required environment variables
|
||||
if [ "$NEEDS_LLM" = true ] && [ -z "${GROQ_API_KEY:-}" ]; then
|
||||
echo -e "${RED}Error: GROQ_API_KEY environment variable is required for API/standalone images${NC}"
|
||||
echo "Set it with: export GROQ_API_KEY=your-api-key"
|
||||
exit 2
|
||||
fi
|
||||
|
||||
# Cleanup function
|
||||
cleanup() {
|
||||
echo "Cleaning up..."
|
||||
docker stop "$CONTAINER_NAME" 2>/dev/null || true
|
||||
docker rm "$CONTAINER_NAME" 2>/dev/null || true
|
||||
}
|
||||
|
||||
# Set trap to cleanup on exit
|
||||
trap cleanup EXIT
|
||||
|
||||
echo -e "${YELLOW}Starting smoke test for: ${IMAGE}${NC}"
|
||||
echo " Target: $TARGET"
|
||||
echo " Health port: $HEALTH_PORT"
|
||||
echo " Timeout: ${TIMEOUT}s"
|
||||
echo ""
|
||||
|
||||
# Remove any existing container with the same name
|
||||
docker rm -f "$CONTAINER_NAME" 2>/dev/null || true
|
||||
|
||||
# Start container based on target type
|
||||
echo "Starting container..."
|
||||
if [ "$TARGET" = "cp-only" ]; then
|
||||
docker run -d --name "$CONTAINER_NAME" \
|
||||
-p "${HEALTH_PORT}:${HEALTH_PORT}" \
|
||||
"$IMAGE"
|
||||
else
|
||||
docker run -d --name "$CONTAINER_NAME" \
|
||||
-e HINDSIGHT_API_LLM_PROVIDER="$LLM_PROVIDER" \
|
||||
-e HINDSIGHT_API_LLM_API_KEY="${GROQ_API_KEY}" \
|
||||
-e HINDSIGHT_API_LLM_MODEL="$LLM_MODEL" \
|
||||
-p "${HEALTH_PORT}:${HEALTH_PORT}" \
|
||||
"$IMAGE"
|
||||
fi
|
||||
|
||||
# Wait for health endpoint
|
||||
echo "Waiting for health endpoint on port ${HEALTH_PORT}..."
|
||||
start_time=$(date +%s)
|
||||
|
||||
for i in $(seq 1 "$TIMEOUT"); do
|
||||
if curl -sf "http://localhost:${HEALTH_PORT}/health" > /dev/null 2>&1; then
|
||||
end_time=$(date +%s)
|
||||
duration=$((end_time - start_time))
|
||||
echo ""
|
||||
echo -e "${GREEN}Container is healthy after ${duration}s${NC}"
|
||||
echo ""
|
||||
echo "=== Health Response ==="
|
||||
curl -s "http://localhost:${HEALTH_PORT}/health" | python3 -m json.tool 2>/dev/null || curl -s "http://localhost:${HEALTH_PORT}/health"
|
||||
echo ""
|
||||
echo ""
|
||||
echo "=== Container Logs (last 50 lines) ==="
|
||||
docker logs "$CONTAINER_NAME" 2>&1 | tail -50
|
||||
echo ""
|
||||
echo -e "${GREEN}Smoke test PASSED${NC}"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# Show progress every 10 seconds
|
||||
if [ $((i % 10)) -eq 0 ]; then
|
||||
echo " Still waiting... (${i}s)"
|
||||
fi
|
||||
|
||||
# Check if container is still running
|
||||
if ! docker ps -q -f "name=$CONTAINER_NAME" | grep -q .; then
|
||||
echo ""
|
||||
echo -e "${RED}Container exited unexpectedly!${NC}"
|
||||
echo ""
|
||||
echo "=== Container Logs ==="
|
||||
docker logs "$CONTAINER_NAME" 2>&1
|
||||
echo ""
|
||||
echo -e "${RED}Smoke test FAILED${NC}"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
sleep 1
|
||||
done
|
||||
|
||||
# Timeout reached
|
||||
echo ""
|
||||
echo -e "${RED}Container failed to become healthy after ${TIMEOUT}s${NC}"
|
||||
echo ""
|
||||
echo "=== Container Logs ==="
|
||||
docker logs "$CONTAINER_NAME" 2>&1
|
||||
echo ""
|
||||
echo -e "${RED}Smoke test FAILED${NC}"
|
||||
exit 1
|
||||
Loading…
Reference in a new issue