28dac7c7f8
* fix: prevent silent memory loss on consolidation LLM failure
When all LLM retries are exhausted during consolidation, memories were
being marked consolidated_at unconditionally, permanently excluding them
from future consolidation runs without producing any observations.
Fix with two complementary mechanisms:
- Adaptive batch splitting: on LLM failure, the batch is halved and
retried recursively down to batch_size=1, recovering most transient
failures (rate limits, Pydantic validation on long prompts) without
operator intervention
- consolidation_failed_at column: only single-memory batches that still
fail after all retries are marked here instead of consolidated_at, so
they remain visible and retryable
- New API endpoint POST /v1/default/banks/{bank_id}/consolidation/retry-failed
resets these memories for the next consolidation run
* chore: regenerate OpenAPI spec
* fix: rename consolidation endpoint from /retry-failed to /recover
* fix: add consolidation_failed_at column, adaptive batch splitting, and recovery API
- Migration a3b4c5d6e7f8: add consolidation_failed_at TIMESTAMPTZ column to
memory_units with an index for efficient failure queries; properly chains off
g7h8i9j0k1l2 (backsweep_orphan_observations)
- Consolidator: filter pending memories with consolidation_failed_at IS NULL
so failed memories are not re-fetched in an infinite loop
- Consolidator: adaptive batch splitting — when a batch exhausts all 3 LLM
retries, halve it and retry sub-batches recursively; only single-memory
batches that also exhaust all retries get consolidation_failed_at set
- New tests (9 total) covering: adaptive splitting recovers all memories,
larger batch splitting, single-memory permanent failure, exclusion from
next run, partial batch failure, recover resets columns, recover returns
0 when none failed, recover-then-consolidate succeeds, HTTP endpoint
* chore: regenerate Go, Python, TypeScript clients with recover consolidation endpoint
* feat: add Recover Consolidation action to bank Actions dropdown
* style: apply ruff formatting to http.py and config.py
* fix: handle consolidation scope in large batch test mock LLM
The mock LLM was returning {"facts": ...} for ALL calls including consolidation.
Consolidation doesn't use skip_validation=True so it expects a _ConsolidationBatchResponse
instance, not a raw dict. Before this PR consolidation silently swallowed the AttributeError
(failed=False was returned); now failed=True triggers adaptive splitting and timeouts.
Fix: return _ConsolidationBatchResponse() when scope=="consolidation".
* fix: restrict claude-agent-sdk to macOS platform only (no Linux wheel available)
Also fix pre-existing type errors: use setattr for XLM-RoBERTa monkey-patch
and add missing reranker_local_fp16/bucket_batching/batch_size fields to main.py config constructor.
* fix: add UV_INDEX_STRATEGY=unsafe-best-match to fix markupsafe cp314 wheel conflict
PyTorch CPU index serves markupsafe==3.0.3 with only cp314 wheels.
uv's default first-index strategy stops at the first index with any version
even if no compatible wheel exists. unsafe-best-match searches all indices
for the best compatible wheel, falling back to PyPI for markupsafe.
* fix: use explicit pytorch index to prevent markupsafe wheel conflict
Configure the pytorch CPU index as explicit=true in pyproject.toml so it is
ONLY used for torch (via [tool.uv.sources]). All other packages (including
markupsafe) are resolved exclusively from PyPI, preventing the pytorch index
from serving incompatible cp314-only wheels for non-pytorch packages.
Remove UV_INDEX and UV_INDEX_STRATEGY from CI workflow (no longer needed
since the index is now configured in pyproject.toml).
* ci: trigger CI run
* ci: retry trigger
* ci: trigger after remote URL fix
* ci: add workflow_dispatch to unblock manual trigger
* fix: remove empty env blocks left after UV_INDEX removal
* fix: add type: ignore for optional claude_agent_sdk imports (macOS-only)
* fix: correct type: ignore rules for claude_agent_sdk and fix utcnow deprecation
205 lines
6 KiB
TOML
205 lines
6 KiB
TOML
[build-system]
|
|
requires = ["hatchling"]
|
|
build-backend = "hatchling.build"
|
|
|
|
[project]
|
|
name = "hindsight-api-slim"
|
|
version = "0.4.18"
|
|
description = "Hindsight: Agent Memory That Works Like Human Memory"
|
|
readme = "README.md"
|
|
requires-python = ">=3.11"
|
|
dependencies = [
|
|
"asyncpg>=0.29.0",
|
|
"python-dotenv>=1.0.0",
|
|
"openai>=1.0.0",
|
|
"pydantic>=2.0.0",
|
|
"rich>=13.0.0",
|
|
"langchain-text-splitters>=0.3.0",
|
|
"fastapi[standard]>=0.120.3",
|
|
"uvicorn>=0.38.0",
|
|
"wsproto>=1.0.0",
|
|
"sqlalchemy>=2.0.44",
|
|
"alembic>=1.17.1",
|
|
"pgvector>=0.4.1",
|
|
"greenlet>=3.2.4",
|
|
"psycopg2-binary>=2.9.11",
|
|
"tiktoken>=0.12.0",
|
|
"httpx>=0.27.0",
|
|
"PyJWT[crypto]>=2.8.0",
|
|
"fastmcp>=2.14.0", # CVE-2025-66416
|
|
"python-dateutil>=2.8.0",
|
|
"opentelemetry-api>=1.20.0",
|
|
"opentelemetry-sdk>=1.20.0",
|
|
"opentelemetry-instrumentation-fastapi>=0.41b0",
|
|
"opentelemetry-exporter-prometheus>=0.41b0",
|
|
"opentelemetry-exporter-otlp-proto-http>=1.20.0",
|
|
"opentelemetry-semantic-conventions>=0.41b0",
|
|
"dateparser>=1.2.2",
|
|
"google-genai>=1.0.0",
|
|
"google-auth>=2.0.0",
|
|
"anthropic>=0.40.0",
|
|
"typer>=0.9.0",
|
|
"cohere>=5.0.0",
|
|
"litellm>=1.0.0",
|
|
"markitdown[pdf,docx,pptx,xlsx,xls]>=0.1.4", # File to markdown conversion
|
|
"obstore>=0.4.0", # S3/GCS/Azure object storage client (Rust-backed)
|
|
"uvloop>=0.22.1",
|
|
# Transitive dependency security fixes
|
|
"pyasn1>=0.6.2", # DoS vulnerability fix
|
|
"urllib3>=2.6.3", # Decompression-bomb safeguards bypass fix
|
|
"langchain-core>=1.2.11", # Serialization injection + SSRF vulnerability fix
|
|
"langsmith>=0.6.3", # SSRF via tracing header injection fix
|
|
"protobuf>=6.33.5", # JSON recursion depth bypass fix
|
|
"pillow>=12.1.1", # Out-of-bounds write in PSD image loading fix
|
|
"cryptography>=46.0.5", # Subgroup attack vulnerability fix
|
|
"filelock>=3.20.1", # TOCTOU race condition fix
|
|
"authlib>=1.6.6", # Account takeover vulnerability fix
|
|
"aiohttp>=3.13.3", # Multiple DoS vulnerabilities
|
|
"claude-agent-sdk>=0.1.27; sys_platform == 'darwin'",
|
|
]
|
|
|
|
[project.optional-dependencies]
|
|
local-ml = [
|
|
# Local ML models for embeddings/reranking
|
|
"sentence-transformers>=3.3.0",
|
|
"transformers>=4.53.0", # Security fixes for ReDoS vulnerabilities
|
|
"torch>=2.6.0", # CVE fix for remote code execution
|
|
"einops>=0.8.2",
|
|
"flashrank>=0.2.0",
|
|
# Apple Silicon local inference
|
|
"mlx>=0.31.0",
|
|
"mlx-lm>=0.31.1",
|
|
"safetensors>=0.6.2",
|
|
]
|
|
embedded-db = [
|
|
"pg0-embedded>=0.11.0",
|
|
]
|
|
all = [
|
|
"hindsight-api-slim[local-ml,embedded-db]",
|
|
]
|
|
test = [
|
|
"pytest>=7.0.0",
|
|
"pytest-asyncio>=0.21.0",
|
|
"pytest-timeout>=2.4.0",
|
|
"pytest-xdist>=3.0.0",
|
|
"filelock>=3.20.1", # TOCTOU race condition fix
|
|
"testcontainers>=4.0.0",
|
|
]
|
|
|
|
[project.scripts]
|
|
hindsight-api = "hindsight_api.main:main"
|
|
hindsight-worker = "hindsight_api.worker.main:main"
|
|
hindsight-local-mcp = "hindsight_api.mcp_local:main"
|
|
hindsight-admin = "hindsight_api.admin.cli:main"
|
|
|
|
[tool.hatch.build.targets.wheel]
|
|
packages = ["hindsight_api"]
|
|
|
|
[tool.hatch.build.targets.wheel.sources]
|
|
"hindsight_api" = "hindsight_api"
|
|
|
|
[tool.hatch.build.targets.sdist]
|
|
include = [
|
|
"hindsight_api/**/*",
|
|
]
|
|
|
|
[tool.hatch.build]
|
|
include = [
|
|
"hindsight_api/**/*.py",
|
|
"hindsight_api/alembic/**/*",
|
|
]
|
|
|
|
[tool.pytest.ini_options]
|
|
log_cli = true
|
|
log_cli_level = "INFO"
|
|
log_cli_format = "%(asctime)s - %(levelname)s - %(name)s - %(message)s"
|
|
log_cli_date_format = "%Y-%m-%d %H:%M:%S"
|
|
addopts = "--timeout 300 -n 8 --dist loadgroup --durations=10 -v"
|
|
asyncio_mode = "auto"
|
|
asyncio_default_fixture_loop_scope = "function"
|
|
log_auto_indent = true
|
|
filterwarnings = [
|
|
"ignore:The @wait_container_is_ready decorator is deprecated:DeprecationWarning",
|
|
"ignore::RuntimeWarning:asyncio",
|
|
]
|
|
|
|
[dependency-groups]
|
|
dev = [
|
|
"pytest>=9.0.0",
|
|
"pytest-asyncio>=1.3.0",
|
|
"pytest-timeout>=2.4.0",
|
|
"pytest-xdist>=3.8.0",
|
|
"python-dotenv>=1.2.1",
|
|
"filelock>=3.20.1", # TOCTOU race condition fix
|
|
"ruff>=0.8.0",
|
|
"ty>=0.0.1",
|
|
"testcontainers>=4.0.0",
|
|
]
|
|
|
|
[tool.ruff]
|
|
line-length = 120
|
|
target-version = "py311"
|
|
exclude = [
|
|
"tests/",
|
|
"**/tests/",
|
|
]
|
|
|
|
[tool.ruff.lint]
|
|
select = [
|
|
"E", # pycodestyle errors
|
|
"W", # pycodestyle warnings
|
|
"F", # Pyflakes
|
|
"I", # isort
|
|
]
|
|
ignore = [
|
|
"E501", # line too long (handled by formatter)
|
|
"E402", # module import not at top of file
|
|
"F401", # unused import (too noisy during development)
|
|
"F841", # unused variable (too noisy during development)
|
|
"F811", # redefined while unused
|
|
"F821", # undefined name (forward references in type hints)
|
|
]
|
|
|
|
[tool.ruff.lint.isort]
|
|
known-third-party = ["alembic"]
|
|
|
|
[tool.ruff.format]
|
|
quote-style = "double"
|
|
indent-style = "space"
|
|
|
|
[tool.uv]
|
|
# Use explicit index for PyTorch to prevent the pytorch index from serving
|
|
# non-pytorch packages (e.g. markupsafe) with incompatible wheels
|
|
[[tool.uv.index]]
|
|
name = "pytorch-cpu"
|
|
url = "https://download.pytorch.org/whl/cpu"
|
|
explicit = true
|
|
|
|
[tool.uv.sources]
|
|
# Route torch to the CPU-only PyTorch index; everything else uses PyPI
|
|
torch = { index = "pytorch-cpu" }
|
|
|
|
[tool.ty]
|
|
# Type checking configuration
|
|
# ty is an extremely fast Python type checker from Astral (same team as ruff/uv)
|
|
|
|
[tool.ty.environment]
|
|
python-version = "3.11"
|
|
|
|
[tool.ty.src]
|
|
exclude = [
|
|
"tests/",
|
|
"hindsight_api/alembic/",
|
|
]
|
|
|
|
[tool.ty.rules]
|
|
# Disable noisy rules while keeping important ones
|
|
invalid-argument-type = "ignore" # False positives with **kwargs patterns
|
|
invalid-return-type = "ignore" # Often intentional in async code
|
|
invalid-parameter-default = "ignore" # Optional params with None default
|
|
possibly-missing-attribute = "ignore" # Common with Optional types
|
|
invalid-raise = "ignore" # False positives with exception tracking
|
|
call-non-callable = "ignore" # False positives with Optional types
|
|
invalid-key = "ignore" # Pydantic ConfigDict not understood
|
|
invalid-method-override = "ignore" # Intentional signature differences
|
|
unresolved-reference = "ignore" # Forward references not always resolved
|