fix: remove broken minimax test and enhance slim smoke test with retain/recall (#564)
- Delete test_minimax_provider.py which imports non-existent `create_llm` function (should be `create_llm_provider`), causing pytest collection errors - Add scripts/smoke-test-slim.sh: shared retain + recall validation script used by both Docker slim and pip slim CI jobs - Update docker/test-image.sh to run retain/recall after health check for all API targets - Update test-pip-slim CI job to run the shared smoke test script
This commit is contained in:
Nicolò Boschi
GitHub
parent
15ea23d5d6
commit
94598fbd25
4 changed files with 84 additions and 202 deletions
6
.github/workflows/test.yml
vendored
6
.github/workflows/test.yml
vendored
|
|
@ -1249,7 +1249,7 @@ jobs:
|
|||
working-directory: ./hindsight-api-slim
|
||||
run: uv sync --frozen --extra embedded-db --index-strategy unsafe-best-match
|
||||
|
||||
- name: Start API server and smoke test
|
||||
- name: Start API server
|
||||
working-directory: ./hindsight-api-slim
|
||||
run: |
|
||||
uv run hindsight-api --port 8888 > /tmp/slim-api-server.log 2>&1 &
|
||||
|
|
@ -1265,7 +1265,9 @@ jobs:
|
|||
fi
|
||||
sleep 1
|
||||
done
|
||||
echo "PASS: slim pip package smoke test (server started and healthy)"
|
||||
|
||||
- name: Smoke test - retain and recall
|
||||
run: ./scripts/smoke-test-slim.sh http://localhost:8888
|
||||
|
||||
- name: Show API server logs
|
||||
if: always()
|
||||
|
|
|
|||
|
|
@ -49,6 +49,9 @@
|
|||
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
REPO_ROOT="$(dirname "$SCRIPT_DIR")"
|
||||
|
||||
# Colors for output
|
||||
RED='\033[0;31m'
|
||||
GREEN='\033[0;32m'
|
||||
|
|
@ -178,6 +181,21 @@ for i in $(seq 1 "$TIMEOUT"); do
|
|||
echo "=== Health Response ==="
|
||||
curl -s "http://localhost:${HEALTH_PORT}${HEALTH_PATH}" | python3 -m json.tool 2>/dev/null || curl -s "http://localhost:${HEALTH_PORT}${HEALTH_PATH}"
|
||||
echo ""
|
||||
|
||||
# Run retain/recall smoke test for API targets
|
||||
if [ "$TARGET" != "cp-only" ]; then
|
||||
echo ""
|
||||
echo "=== Retain/Recall Smoke Test ==="
|
||||
if ! "$REPO_ROOT/scripts/smoke-test-slim.sh" "http://localhost:${HEALTH_PORT}"; then
|
||||
echo ""
|
||||
echo "=== Container Logs (last 50 lines) ==="
|
||||
docker logs "$CONTAINER_NAME" 2>&1 | tail -50
|
||||
echo ""
|
||||
echo -e "${RED}Smoke test FAILED${NC}"
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
|
||||
echo ""
|
||||
echo "=== Container Logs (last 50 lines) ==="
|
||||
docker logs "$CONTAINER_NAME" 2>&1 | tail -50
|
||||
|
|
|
|||
|
|
@ -1,200 +0,0 @@
|
|||
"""Tests for MiniMax provider integration.
|
||||
|
||||
Validates that MiniMax is correctly registered as an OpenAI-compatible provider
|
||||
with proper base URL, temperature clamping, and default model configuration.
|
||||
"""
|
||||
|
||||
import os
|
||||
|
||||
import pytest
|
||||
|
||||
from hindsight_api.engine.llm_wrapper import LLMProvider, create_llm
|
||||
|
||||
|
||||
def test_minimax_provider_creation():
|
||||
"""Test that MiniMax provider can be instantiated correctly."""
|
||||
llm = LLMProvider(
|
||||
provider="minimax",
|
||||
api_key="test-key",
|
||||
base_url="",
|
||||
model="MiniMax-M2.5",
|
||||
)
|
||||
assert llm.provider == "minimax"
|
||||
assert llm.model == "MiniMax-M2.5"
|
||||
assert llm.base_url == "https://api.minimax.io/v1"
|
||||
|
||||
|
||||
def test_minimax_default_base_url():
|
||||
"""Test that MiniMax uses the correct default base URL when none is provided."""
|
||||
llm = LLMProvider(
|
||||
provider="minimax",
|
||||
api_key="test-key",
|
||||
base_url="",
|
||||
model="MiniMax-M2.5",
|
||||
)
|
||||
assert llm.base_url == "https://api.minimax.io/v1"
|
||||
|
||||
|
||||
def test_minimax_custom_base_url():
|
||||
"""Test that a custom base URL overrides the default."""
|
||||
llm = LLMProvider(
|
||||
provider="minimax",
|
||||
api_key="test-key",
|
||||
base_url="https://custom.api.example.com/v1",
|
||||
model="MiniMax-M2.5",
|
||||
)
|
||||
assert llm.base_url == "https://custom.api.example.com/v1"
|
||||
|
||||
|
||||
def test_minimax_factory_function():
|
||||
"""Test that the create_llm factory function creates MiniMax provider correctly."""
|
||||
llm = create_llm(
|
||||
provider="minimax",
|
||||
api_key="test-key",
|
||||
base_url="",
|
||||
model="MiniMax-M2.5",
|
||||
)
|
||||
assert llm is not None
|
||||
|
||||
|
||||
def test_minimax_requires_api_key():
|
||||
"""Test that MiniMax provider requires an API key."""
|
||||
with pytest.raises(ValueError, match="API key"):
|
||||
LLMProvider(
|
||||
provider="minimax",
|
||||
api_key="",
|
||||
base_url="",
|
||||
model="MiniMax-M2.5",
|
||||
)
|
||||
|
||||
|
||||
def test_minimax_default_model_config():
|
||||
"""Test that MiniMax has a default model in PROVIDER_DEFAULT_MODELS."""
|
||||
from hindsight_api.config import PROVIDER_DEFAULT_MODELS
|
||||
|
||||
assert "minimax" in PROVIDER_DEFAULT_MODELS
|
||||
assert PROVIDER_DEFAULT_MODELS["minimax"] == "MiniMax-M2.5"
|
||||
|
||||
|
||||
def test_minimax_config_default_model():
|
||||
"""Test that MiniMax default model is used when model is not explicitly set."""
|
||||
from hindsight_api.config import HindsightConfig, clear_config_cache
|
||||
|
||||
original_provider = os.environ.get("HINDSIGHT_API_LLM_PROVIDER")
|
||||
original_model = os.environ.get("HINDSIGHT_API_LLM_MODEL")
|
||||
|
||||
try:
|
||||
clear_config_cache()
|
||||
os.environ["HINDSIGHT_API_LLM_PROVIDER"] = "minimax"
|
||||
if "HINDSIGHT_API_LLM_MODEL" in os.environ:
|
||||
del os.environ["HINDSIGHT_API_LLM_MODEL"]
|
||||
|
||||
config = HindsightConfig.from_env()
|
||||
assert config.llm_provider == "minimax"
|
||||
assert config.llm_model == "MiniMax-M2.5"
|
||||
finally:
|
||||
clear_config_cache()
|
||||
if original_provider:
|
||||
os.environ["HINDSIGHT_API_LLM_PROVIDER"] = original_provider
|
||||
elif "HINDSIGHT_API_LLM_PROVIDER" in os.environ:
|
||||
del os.environ["HINDSIGHT_API_LLM_PROVIDER"]
|
||||
if original_model:
|
||||
os.environ["HINDSIGHT_API_LLM_MODEL"] = original_model
|
||||
elif "HINDSIGHT_API_LLM_MODEL" in os.environ:
|
||||
del os.environ["HINDSIGHT_API_LLM_MODEL"]
|
||||
|
||||
|
||||
def test_minimax_temperature_clamping():
|
||||
"""Test that MiniMax temperature is clamped to (0.0, 1.0] range."""
|
||||
from hindsight_api.engine.providers.openai_compatible_llm import OpenAICompatibleLLM
|
||||
|
||||
llm = OpenAICompatibleLLM(
|
||||
provider="minimax",
|
||||
api_key="test-key",
|
||||
base_url="https://api.minimax.io/v1",
|
||||
model="MiniMax-M2.5",
|
||||
)
|
||||
|
||||
# Verify the provider is correctly set up for temperature clamping
|
||||
assert llm.provider == "minimax"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_minimax_integration():
|
||||
"""Integration test: verify MiniMax provider works with actual API.
|
||||
|
||||
Requires MINIMAX_API_KEY environment variable to be set.
|
||||
"""
|
||||
api_key = os.environ.get("MINIMAX_API_KEY")
|
||||
if not api_key:
|
||||
pytest.skip("MINIMAX_API_KEY not set")
|
||||
|
||||
llm = LLMProvider(
|
||||
provider="minimax",
|
||||
api_key=api_key,
|
||||
base_url="",
|
||||
model="MiniMax-M2.5",
|
||||
)
|
||||
|
||||
# Test verify_connection
|
||||
await llm.verify_connection()
|
||||
|
||||
# Test basic call
|
||||
response = await llm.call(
|
||||
messages=[
|
||||
{"role": "system", "content": "You are a helpful assistant."},
|
||||
{"role": "user", "content": "What is 2+2? Answer in one word."},
|
||||
],
|
||||
max_completion_tokens=50,
|
||||
)
|
||||
assert response is not None
|
||||
assert len(response) > 0
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_minimax_tool_calling():
|
||||
"""Integration test: verify MiniMax provider supports tool calling.
|
||||
|
||||
Requires MINIMAX_API_KEY environment variable to be set.
|
||||
"""
|
||||
api_key = os.environ.get("MINIMAX_API_KEY")
|
||||
if not api_key:
|
||||
pytest.skip("MINIMAX_API_KEY not set")
|
||||
|
||||
llm = LLMProvider(
|
||||
provider="minimax",
|
||||
api_key=api_key,
|
||||
base_url="",
|
||||
model="MiniMax-M2.5",
|
||||
)
|
||||
|
||||
tools = [
|
||||
{
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "get_weather",
|
||||
"description": "Get the weather for a location",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"location": {"type": "string", "description": "City name"},
|
||||
},
|
||||
"required": ["location"],
|
||||
},
|
||||
},
|
||||
}
|
||||
]
|
||||
|
||||
result = await llm.call_with_tools(
|
||||
messages=[
|
||||
{"role": "system", "content": "You are a helpful assistant with access to tools."},
|
||||
{"role": "user", "content": "What's the weather like in Paris?"},
|
||||
],
|
||||
tools=tools,
|
||||
max_completion_tokens=500,
|
||||
)
|
||||
|
||||
assert result is not None
|
||||
assert hasattr(result, "tool_calls")
|
||||
assert len(result.tool_calls) > 0
|
||||
assert result.tool_calls[0].name == "get_weather"
|
||||
62
scripts/smoke-test-slim.sh
Executable file
62
scripts/smoke-test-slim.sh
Executable file
|
|
@ -0,0 +1,62 @@
|
|||
#!/bin/bash
|
||||
#
|
||||
# Slim variant smoke test: retain + recall
|
||||
#
|
||||
# Verifies that a Hindsight API endpoint can store and retrieve memories.
|
||||
# Used by both Docker slim and pip slim CI jobs.
|
||||
#
|
||||
# Usage:
|
||||
# ./scripts/smoke-test-slim.sh [base_url]
|
||||
#
|
||||
# Arguments:
|
||||
# base_url - API base URL (default: http://localhost:8888)
|
||||
#
|
||||
# Exit codes:
|
||||
# 0 - Success
|
||||
# 1 - Failure
|
||||
#
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
RED='\033[0;31m'
|
||||
GREEN='\033[0;32m'
|
||||
NC='\033[0m'
|
||||
|
||||
BASE_URL="${1:-http://localhost:8888}"
|
||||
BANK_ID="smoke-test-$$"
|
||||
|
||||
echo "Running retain/recall smoke test against: $BASE_URL"
|
||||
echo "Bank: $BANK_ID"
|
||||
|
||||
# Retain
|
||||
echo ""
|
||||
echo "--- Retain ---"
|
||||
RETAIN_RESPONSE=$(curl -sf -X POST "$BASE_URL/v1/default/banks/$BANK_ID/memories" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{"items": [{"content": "Alice is a software engineer who loves Python and distributed systems."}]}')
|
||||
echo "$RETAIN_RESPONSE" | python3 -m json.tool 2>/dev/null || echo "$RETAIN_RESPONSE"
|
||||
|
||||
SUCCESS=$(echo "$RETAIN_RESPONSE" | python3 -c "import sys, json; d = json.load(sys.stdin); print(d.get('success', False))" 2>/dev/null || echo "False")
|
||||
if [ "$SUCCESS" != "True" ]; then
|
||||
echo -e "${RED}FAIL: retain did not return success=true${NC}"
|
||||
exit 1
|
||||
fi
|
||||
echo "Retain: OK"
|
||||
|
||||
# Recall
|
||||
echo ""
|
||||
echo "--- Recall ---"
|
||||
RECALL_RESPONSE=$(curl -sf -X POST "$BASE_URL/v1/default/banks/$BANK_ID/memories/recall" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{"query": "What does Alice do?"}')
|
||||
echo "$RECALL_RESPONSE" | python3 -m json.tool 2>/dev/null || echo "$RECALL_RESPONSE"
|
||||
|
||||
RESULTS_COUNT=$(echo "$RECALL_RESPONSE" | python3 -c "import sys, json; d = json.load(sys.stdin); print(len(d.get('results', [])))" 2>/dev/null || echo "0")
|
||||
if [ -z "$RESULTS_COUNT" ] || [ "$RESULTS_COUNT" -eq 0 ]; then
|
||||
echo -e "${RED}FAIL: recall returned no results${NC}"
|
||||
exit 1
|
||||
fi
|
||||
echo "Recall: OK ($RESULTS_COUNT results)"
|
||||
|
||||
echo ""
|
||||
echo -e "${GREEN}PASS: retain/recall smoke test${NC}"
|
||||
Loading…
Reference in a new issue