aefb3fcf4d
* fix: improve async batch retain with large payloads * fix: improve async batch retain with large payloads * api * api * api * api * api * Clean up perf benchmark: keep only Python files - Remove README.md and PERFORMANCE_FINDINGS.md - Remove results/ JSON files (gitignored) - Remove test_data/ directory - Keep only __init__.py and retain_perf.py * docs: explain automatic batch optimization for async retain - Add section explaining Hindsight automatically handles batch sizing - Users don't need to manually tune batch sizes with async mode - Hindsight splits large batches (>10k tokens) into optimized sub-batches - Include example showing best practices * docs: remove emojis and code example from performance page * fix: correct OperationDetails type to match API response - Change optional fields to use | null instead of ? - Fixes TypeScript compilation error in control plane build * fix: use discriminated union for OperationDetails type - Support both success and error states properly - Fixes TypeScript error when setting error state * fix: use unique document_ids in batch retain examples - Each item in a batch must have unique document_id - Update both Python and JavaScript examples - Fixes test-doc-examples CI failure * chore: trigger CI * fix: test mocking and duplicate document_ids in examples - Mock _get_pool() in test_async_retain_tags.py to avoid _initialized error - Set _initialized = True on mocked MemoryEngine instances - Fix duplicate document_ids in retain.py and retain.mjs examples * fix: properly mock async pool/connection and fix more duplicate document_ids - Use AsyncMock for pool.acquire() to fix 'can't be used in await' error - Fix duplicate document_ids in retain-async examples (retain.py and retain.mjs) - Remove batch-level document_id parameter that caused duplicates * ci: collect all doc example failures and show summary - Run all Python/Node.js/CLI examples regardless of individual failures - Collect failure list and display summary at the end - Show pass/fail count and list of failed files - Exit with failure only after running all examples * refactor: extract doc example testing to standalone script - Create scripts/test-doc-examples.sh to run all examples - Collects logs of failed examples separately - Shows full error logs only for failures at the end - Clean summary with pass/fail counts - Proper exit codes - Replaces inline bash in CI workflow * fix: doc examples - duplicate document_ids and error handling - retain.py: move document_id to item level to avoid duplicates - documents.mjs: add error handling for getDocument to show clear error message * fix: update tests for duplicate document_id validation - test_async_retain_tags: verify operation structure instead of exact UUID - test_delete_bank: use unique document_ids (team-doc-1, team-doc-2)
134 lines
3.1 KiB
Bash
Executable file
134 lines
3.1 KiB
Bash
Executable file
#!/bin/bash
|
|
# Run retain performance benchmark
|
|
|
|
set -e
|
|
|
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
PROJECT_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)"
|
|
|
|
cd "$PROJECT_ROOT"
|
|
|
|
# Default values
|
|
DOCUMENT="${DOCUMENT:-}"
|
|
BANK_ID="${BANK_ID:-perf-test}"
|
|
API_URL="${API_URL:-http://localhost:8000}"
|
|
TIMEOUT="${TIMEOUT:-300}"
|
|
OUTPUT="${OUTPUT:-}"
|
|
|
|
# Help message
|
|
show_help() {
|
|
cat << EOF
|
|
Run retain performance benchmark
|
|
|
|
Usage: $0 --document <path> [options]
|
|
|
|
Required:
|
|
--document <path> Path to document file to retain
|
|
|
|
Options:
|
|
--bank-id <id> Bank ID to use (default: perf-test)
|
|
--context <text> Optional context for the retain operation
|
|
--api-url <url> API base URL (default: http://localhost:8000)
|
|
--timeout <seconds> Request timeout (default: 300)
|
|
--output <path> Path to save results JSON (optional)
|
|
--in-memory Use in-memory MemoryEngine instead of HTTP
|
|
-h, --help Show this help message
|
|
|
|
Environment Variables:
|
|
DOCUMENT Document path (can be used instead of --document)
|
|
BANK_ID Bank ID (default: perf-test)
|
|
API_URL API URL (default: http://localhost:8000)
|
|
TIMEOUT Timeout in seconds (default: 300)
|
|
OUTPUT Output path for results JSON
|
|
|
|
Examples:
|
|
# Basic usage
|
|
$0 --document ./test_data/large_doc.txt
|
|
|
|
# With custom bank ID and save results
|
|
$0 --document ./test_data/large_doc.txt \\
|
|
--bank-id my-test-bank \\
|
|
--output results/retain_perf.json
|
|
|
|
# Using environment variables
|
|
DOCUMENT=./test_data/large_doc.txt \\
|
|
BANK_ID=my-test-bank \\
|
|
$0
|
|
EOF
|
|
}
|
|
|
|
# Parse arguments
|
|
CONTEXT=""
|
|
IN_MEMORY=""
|
|
while [[ $# -gt 0 ]]; do
|
|
case $1 in
|
|
--document)
|
|
DOCUMENT="$2"
|
|
shift 2
|
|
;;
|
|
--bank-id)
|
|
BANK_ID="$2"
|
|
shift 2
|
|
;;
|
|
--context)
|
|
CONTEXT="$2"
|
|
shift 2
|
|
;;
|
|
--api-url)
|
|
API_URL="$2"
|
|
shift 2
|
|
;;
|
|
--timeout)
|
|
TIMEOUT="$2"
|
|
shift 2
|
|
;;
|
|
--output)
|
|
OUTPUT="$2"
|
|
shift 2
|
|
;;
|
|
--in-memory)
|
|
IN_MEMORY="--in-memory"
|
|
shift 1
|
|
;;
|
|
-h|--help)
|
|
show_help
|
|
exit 0
|
|
;;
|
|
*)
|
|
echo "Unknown option: $1"
|
|
echo "Use --help for usage information"
|
|
exit 1
|
|
;;
|
|
esac
|
|
done
|
|
|
|
# Validate required arguments
|
|
if [ -z "$DOCUMENT" ]; then
|
|
echo "Error: --document is required"
|
|
echo "Use --help for usage information"
|
|
exit 1
|
|
fi
|
|
|
|
# Build command
|
|
CMD="uv run python hindsight-dev/benchmarks/perf/retain_perf.py --document \"$DOCUMENT\" --bank-id \"$BANK_ID\" --api-url \"$API_URL\" --timeout $TIMEOUT"
|
|
|
|
if [ -n "$CONTEXT" ]; then
|
|
CMD="$CMD --context \"$CONTEXT\""
|
|
fi
|
|
|
|
if [ -n "$OUTPUT" ]; then
|
|
CMD="$CMD --output \"$OUTPUT\""
|
|
fi
|
|
|
|
if [ -n "$IN_MEMORY" ]; then
|
|
CMD="$CMD --in-memory"
|
|
fi
|
|
|
|
# Run benchmark
|
|
echo "Running retain performance benchmark..."
|
|
echo "Document: $DOCUMENT"
|
|
echo "Bank ID: $BANK_ID"
|
|
echo "API URL: $API_URL"
|
|
echo ""
|
|
|
|
eval $CMD
|