feat: add HINDSIGHT_API_DEFAULT_BANK_TEMPLATE env var (#966)
* feat: add HINDSIGHT_API_DEFAULT_BANK_TEMPLATE env var Server-level default bank template applied automatically to every newly-created bank. Holds an inline JSON BankTemplateManifest with the same shape as the /import endpoint body. Fields set by the template become per-bank overrides so they take precedence over equivalent HINDSIGHT_API_* env defaults. The template is applied once on first creation and never reapplied, so user overrides via PATCH /config are never clobbered. Malformed manifests are logged and ignored so a broken server-level setting cannot wedge bank creation. * chore: regenerate docs skill * test: update async_retain test mock for renamed bank_profile helper
This commit is contained in:
Nicolò Boschi
GitHub
parent
576016f5dc
commit
fc941d5cae
9 changed files with 537 additions and 143 deletions
|
|
@ -1800,6 +1800,150 @@ class BankTemplateImportResponse(BaseModel):
|
|||
dry_run: bool = Field(default=False, description="True if this was a validation-only run")
|
||||
|
||||
|
||||
def validate_bank_template(manifest: "BankTemplateManifest") -> list[str]:
|
||||
"""Validate a parsed manifest beyond Pydantic's structural checks.
|
||||
|
||||
Returns a list of human-readable error strings (e.g. invalid
|
||||
extraction mode values, conflicting settings).
|
||||
"""
|
||||
errors: list[str] = []
|
||||
if manifest.bank:
|
||||
bank = manifest.bank
|
||||
if bank.retain_extraction_mode is not None:
|
||||
valid_modes = ("concise", "verbose", "custom", "chunks")
|
||||
if bank.retain_extraction_mode not in valid_modes:
|
||||
errors.append(
|
||||
f"bank.retain_extraction_mode: must be one of {valid_modes}, got '{bank.retain_extraction_mode}'"
|
||||
)
|
||||
if bank.retain_custom_instructions and bank.retain_extraction_mode != "custom":
|
||||
errors.append("bank.retain_custom_instructions: requires retain_extraction_mode='custom'")
|
||||
if manifest.mental_models:
|
||||
for i, mm in enumerate(manifest.mental_models):
|
||||
if not mm.name.strip():
|
||||
errors.append(f"mental_models[{i}].name: must not be empty")
|
||||
if not mm.source_query.strip():
|
||||
errors.append(f"mental_models[{i}].source_query: must not be empty")
|
||||
if manifest.directives:
|
||||
for i, d in enumerate(manifest.directives):
|
||||
if not d.name.strip():
|
||||
errors.append(f"directives[{i}].name: must not be empty")
|
||||
if not d.content.strip():
|
||||
errors.append(f"directives[{i}].content: must not be empty")
|
||||
return errors
|
||||
|
||||
|
||||
async def apply_bank_template_manifest(
|
||||
memory,
|
||||
bank_id: str,
|
||||
manifest: "BankTemplateManifest",
|
||||
request_context: "RequestContext",
|
||||
) -> "BankTemplateImportResponse":
|
||||
"""Apply a validated BankTemplateManifest to an existing bank.
|
||||
|
||||
Shared by the /import endpoint and the default-template-on-create hook
|
||||
driven by HINDSIGHT_API_DEFAULT_BANK_TEMPLATE. The bank MUST already
|
||||
exist; caller is responsible for validation (Pydantic + validate_bank_template).
|
||||
"""
|
||||
config_applied = False
|
||||
if manifest.bank:
|
||||
config_updates = manifest.bank.get_config_updates()
|
||||
if config_updates:
|
||||
await memory._config_resolver.update_bank_config(bank_id, config_updates, request_context)
|
||||
config_applied = True
|
||||
|
||||
created_ids: list[str] = []
|
||||
updated_ids: list[str] = []
|
||||
operation_ids: list[str] = []
|
||||
|
||||
if manifest.mental_models:
|
||||
# Fetch existing mental models to decide create vs update
|
||||
existing = await memory.list_mental_models(bank_id=bank_id, request_context=request_context)
|
||||
existing_by_id = {m["id"]: m for m in existing}
|
||||
|
||||
for mm in manifest.mental_models:
|
||||
if mm.id in existing_by_id:
|
||||
await memory.update_mental_model(
|
||||
bank_id=bank_id,
|
||||
mental_model_id=mm.id,
|
||||
name=mm.name,
|
||||
source_query=mm.source_query,
|
||||
max_tokens=mm.max_tokens,
|
||||
tags=mm.tags if mm.tags else None,
|
||||
trigger=mm.trigger.model_dump() if mm.trigger else None,
|
||||
request_context=request_context,
|
||||
)
|
||||
result = await memory.submit_async_refresh_mental_model(
|
||||
bank_id=bank_id,
|
||||
mental_model_id=mm.id,
|
||||
request_context=request_context,
|
||||
)
|
||||
operation_ids.append(result["operation_id"])
|
||||
updated_ids.append(mm.id)
|
||||
else:
|
||||
mental_model = await memory.create_mental_model(
|
||||
bank_id=bank_id,
|
||||
name=mm.name,
|
||||
source_query=mm.source_query,
|
||||
content="Generating content...",
|
||||
mental_model_id=mm.id,
|
||||
tags=mm.tags if mm.tags else None,
|
||||
max_tokens=mm.max_tokens,
|
||||
trigger=mm.trigger.model_dump() if mm.trigger else None,
|
||||
request_context=request_context,
|
||||
)
|
||||
result = await memory.submit_async_refresh_mental_model(
|
||||
bank_id=bank_id,
|
||||
mental_model_id=mental_model["id"],
|
||||
request_context=request_context,
|
||||
)
|
||||
operation_ids.append(result["operation_id"])
|
||||
created_ids.append(mm.id)
|
||||
|
||||
directives_created: list[str] = []
|
||||
directives_updated: list[str] = []
|
||||
|
||||
if manifest.directives:
|
||||
existing_directives = await memory.list_directives(
|
||||
bank_id=bank_id, active_only=False, request_context=request_context
|
||||
)
|
||||
existing_by_name = {d["name"]: d for d in existing_directives}
|
||||
|
||||
for directive in manifest.directives:
|
||||
if directive.name in existing_by_name:
|
||||
await memory.update_directive(
|
||||
bank_id=bank_id,
|
||||
directive_id=existing_by_name[directive.name]["id"],
|
||||
content=directive.content,
|
||||
priority=directive.priority,
|
||||
is_active=directive.is_active,
|
||||
tags=directive.tags if directive.tags else None,
|
||||
request_context=request_context,
|
||||
)
|
||||
directives_updated.append(directive.name)
|
||||
else:
|
||||
await memory.create_directive(
|
||||
bank_id=bank_id,
|
||||
name=directive.name,
|
||||
content=directive.content,
|
||||
priority=directive.priority,
|
||||
is_active=directive.is_active,
|
||||
tags=directive.tags if directive.tags else None,
|
||||
request_context=request_context,
|
||||
)
|
||||
directives_created.append(directive.name)
|
||||
|
||||
return BankTemplateImportResponse(
|
||||
bank_id=bank_id,
|
||||
config_applied=config_applied,
|
||||
mental_models_created=created_ids,
|
||||
mental_models_updated=updated_ids,
|
||||
directives_created=directives_created,
|
||||
directives_updated=directives_updated,
|
||||
operation_ids=operation_ids,
|
||||
dry_run=False,
|
||||
)
|
||||
|
||||
|
||||
class OperationResponse(BaseModel):
|
||||
"""Response model for a single async operation."""
|
||||
|
||||
|
|
@ -4377,38 +4521,6 @@ def _register_routes(app: FastAPI):
|
|||
# Bank Template Import / Export
|
||||
# =====================================================================
|
||||
|
||||
def _validate_template(manifest: BankTemplateManifest) -> list[str]:
|
||||
"""Validate a parsed manifest beyond Pydantic's structural checks.
|
||||
|
||||
Returns a list of human-readable error strings (e.g. invalid
|
||||
extraction mode values, conflicting settings).
|
||||
"""
|
||||
errors: list[str] = []
|
||||
if manifest.bank:
|
||||
bank = manifest.bank
|
||||
if bank.retain_extraction_mode is not None:
|
||||
valid_modes = ("concise", "verbose", "custom", "chunks")
|
||||
if bank.retain_extraction_mode not in valid_modes:
|
||||
errors.append(
|
||||
f"bank.retain_extraction_mode: must be one of {valid_modes}, "
|
||||
f"got '{bank.retain_extraction_mode}'"
|
||||
)
|
||||
if bank.retain_custom_instructions and bank.retain_extraction_mode != "custom":
|
||||
errors.append("bank.retain_custom_instructions: requires retain_extraction_mode='custom'")
|
||||
if manifest.mental_models:
|
||||
for i, mm in enumerate(manifest.mental_models):
|
||||
if not mm.name.strip():
|
||||
errors.append(f"mental_models[{i}].name: must not be empty")
|
||||
if not mm.source_query.strip():
|
||||
errors.append(f"mental_models[{i}].source_query: must not be empty")
|
||||
if manifest.directives:
|
||||
for i, d in enumerate(manifest.directives):
|
||||
if not d.name.strip():
|
||||
errors.append(f"directives[{i}].name: must not be empty")
|
||||
if not d.content.strip():
|
||||
errors.append(f"directives[{i}].content: must not be empty")
|
||||
return errors
|
||||
|
||||
@app.post(
|
||||
"/v1/default/banks/{bank_id}/import",
|
||||
response_model=BankTemplateImportResponse,
|
||||
|
|
@ -4444,7 +4556,7 @@ def _register_routes(app: FastAPI):
|
|||
)
|
||||
|
||||
# Semantic validation beyond Pydantic structural checks
|
||||
validation_errors = _validate_template(body)
|
||||
validation_errors = validate_bank_template(body)
|
||||
if validation_errors:
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
|
|
@ -4462,108 +4574,12 @@ def _register_routes(app: FastAPI):
|
|||
# Ensure bank exists (auto-creates with defaults if needed)
|
||||
await app.state.memory.get_bank_profile(bank_id, request_context=request_context)
|
||||
|
||||
config_applied = False
|
||||
if body.bank:
|
||||
config_updates = body.bank.get_config_updates()
|
||||
if config_updates:
|
||||
await app.state.memory._config_resolver.update_bank_config(bank_id, config_updates, request_context)
|
||||
config_applied = True
|
||||
|
||||
created_ids: list[str] = []
|
||||
updated_ids: list[str] = []
|
||||
operation_ids: list[str] = []
|
||||
|
||||
if body.mental_models:
|
||||
# Fetch existing mental models to decide create vs update
|
||||
existing = await app.state.memory.list_mental_models(bank_id=bank_id, request_context=request_context)
|
||||
existing_by_id = {m["id"]: m for m in existing}
|
||||
|
||||
for mm in body.mental_models:
|
||||
if mm.id in existing_by_id:
|
||||
# Update existing mental model metadata
|
||||
await app.state.memory.update_mental_model(
|
||||
return await apply_bank_template_manifest(
|
||||
memory=app.state.memory,
|
||||
bank_id=bank_id,
|
||||
mental_model_id=mm.id,
|
||||
name=mm.name,
|
||||
source_query=mm.source_query,
|
||||
max_tokens=mm.max_tokens,
|
||||
tags=mm.tags if mm.tags else None,
|
||||
trigger=mm.trigger.model_dump() if mm.trigger else None,
|
||||
manifest=body,
|
||||
request_context=request_context,
|
||||
)
|
||||
# Schedule a refresh to regenerate content with updated query
|
||||
result = await app.state.memory.submit_async_refresh_mental_model(
|
||||
bank_id=bank_id,
|
||||
mental_model_id=mm.id,
|
||||
request_context=request_context,
|
||||
)
|
||||
operation_ids.append(result["operation_id"])
|
||||
updated_ids.append(mm.id)
|
||||
else:
|
||||
# Create new mental model
|
||||
mental_model = await app.state.memory.create_mental_model(
|
||||
bank_id=bank_id,
|
||||
name=mm.name,
|
||||
source_query=mm.source_query,
|
||||
content="Generating content...",
|
||||
mental_model_id=mm.id,
|
||||
tags=mm.tags if mm.tags else None,
|
||||
max_tokens=mm.max_tokens,
|
||||
trigger=mm.trigger.model_dump() if mm.trigger else None,
|
||||
request_context=request_context,
|
||||
)
|
||||
result = await app.state.memory.submit_async_refresh_mental_model(
|
||||
bank_id=bank_id,
|
||||
mental_model_id=mental_model["id"],
|
||||
request_context=request_context,
|
||||
)
|
||||
operation_ids.append(result["operation_id"])
|
||||
created_ids.append(mm.id)
|
||||
|
||||
directives_created: list[str] = []
|
||||
directives_updated: list[str] = []
|
||||
|
||||
if body.directives:
|
||||
# Fetch existing directives to decide create vs update (matched by name)
|
||||
existing_directives = await app.state.memory.list_directives(
|
||||
bank_id=bank_id, active_only=False, request_context=request_context
|
||||
)
|
||||
existing_by_name = {d["name"]: d for d in existing_directives}
|
||||
|
||||
for directive in body.directives:
|
||||
if directive.name in existing_by_name:
|
||||
await app.state.memory.update_directive(
|
||||
bank_id=bank_id,
|
||||
directive_id=existing_by_name[directive.name]["id"],
|
||||
content=directive.content,
|
||||
priority=directive.priority,
|
||||
is_active=directive.is_active,
|
||||
tags=directive.tags if directive.tags else None,
|
||||
request_context=request_context,
|
||||
)
|
||||
directives_updated.append(directive.name)
|
||||
else:
|
||||
await app.state.memory.create_directive(
|
||||
bank_id=bank_id,
|
||||
name=directive.name,
|
||||
content=directive.content,
|
||||
priority=directive.priority,
|
||||
is_active=directive.is_active,
|
||||
tags=directive.tags if directive.tags else None,
|
||||
request_context=request_context,
|
||||
)
|
||||
directives_created.append(directive.name)
|
||||
|
||||
return BankTemplateImportResponse(
|
||||
bank_id=bank_id,
|
||||
config_applied=config_applied,
|
||||
mental_models_created=created_ids,
|
||||
mental_models_updated=updated_ids,
|
||||
directives_created=directives_created,
|
||||
directives_updated=directives_updated,
|
||||
operation_ids=operation_ids,
|
||||
dry_run=False,
|
||||
)
|
||||
except ValueError as e:
|
||||
raise HTTPException(status_code=400, detail=str(e))
|
||||
except OperationValidationError as e:
|
||||
|
|
@ -4964,7 +4980,9 @@ def _register_routes(app: FastAPI):
|
|||
from hindsight_api.engine.retain import bank_utils
|
||||
|
||||
# Ensure the bank row exists before inserting into webhooks (FK constraint).
|
||||
await bank_utils.get_bank_profile(pool, bank_id)
|
||||
_, created = await bank_utils.get_or_create_bank_profile(pool, bank_id)
|
||||
if created:
|
||||
await app.state.memory._apply_default_bank_template(bank_id, request_context)
|
||||
|
||||
webhook_id = uuid.uuid4()
|
||||
now = datetime.now(timezone.utc).isoformat()
|
||||
|
|
|
|||
|
|
@ -265,6 +265,7 @@ ENV_MCP_ENABLED = "HINDSIGHT_API_MCP_ENABLED"
|
|||
ENV_MCP_ENABLED_TOOLS = "HINDSIGHT_API_MCP_ENABLED_TOOLS"
|
||||
ENV_MCP_STATELESS = "HINDSIGHT_API_MCP_STATELESS"
|
||||
ENV_ENABLE_BANK_CONFIG_API = "HINDSIGHT_API_ENABLE_BANK_CONFIG_API"
|
||||
ENV_DEFAULT_BANK_TEMPLATE = "HINDSIGHT_API_DEFAULT_BANK_TEMPLATE"
|
||||
ENV_GRAPH_RETRIEVER = "HINDSIGHT_API_GRAPH_RETRIEVER"
|
||||
ENV_RECALL_MAX_CONCURRENT = "HINDSIGHT_API_RECALL_MAX_CONCURRENT"
|
||||
ENV_RECALL_CONNECTION_BUDGET = "HINDSIGHT_API_RECALL_CONNECTION_BUDGET"
|
||||
|
|
@ -502,6 +503,7 @@ DEFAULT_MCP_ENABLED = True
|
|||
DEFAULT_MCP_ENABLED_TOOLS: list[str] | None = None # None = all tools enabled
|
||||
DEFAULT_MCP_STATELESS = False # False = stateful (supports SSE/GET); True = stateless (POST-only)
|
||||
DEFAULT_ENABLE_BANK_CONFIG_API = True
|
||||
DEFAULT_DEFAULT_BANK_TEMPLATE: dict | None = None # BankTemplateManifest dict applied to newly-created banks
|
||||
DEFAULT_GRAPH_RETRIEVER = "link_expansion"
|
||||
DEFAULT_RECALL_MAX_CONCURRENT = 32 # Max concurrent recall operations per worker
|
||||
DEFAULT_RECALL_CONNECTION_BUDGET = 4 # Max concurrent DB connections per recall operation
|
||||
|
|
@ -676,6 +678,26 @@ def _get_default_model_for_provider(provider: str) -> str:
|
|||
return PROVIDER_DEFAULT_MODELS.get(provider.lower(), DEFAULT_LLM_MODEL)
|
||||
|
||||
|
||||
def _parse_default_bank_template(raw: str | None) -> dict | None:
|
||||
"""
|
||||
Parse HINDSIGHT_API_DEFAULT_BANK_TEMPLATE as JSON.
|
||||
|
||||
The env var holds a BankTemplateManifest (JSON object) applied verbatim to
|
||||
every newly-created bank. Full Pydantic validation is deferred to bank
|
||||
creation time (to avoid pulling API models into config.py), but we fail
|
||||
fast here if the value is not valid JSON or not a JSON object.
|
||||
"""
|
||||
if raw is None or raw.strip() == "":
|
||||
return DEFAULT_DEFAULT_BANK_TEMPLATE
|
||||
try:
|
||||
parsed = json.loads(raw)
|
||||
except json.JSONDecodeError as e:
|
||||
raise ValueError(f"Invalid {ENV_DEFAULT_BANK_TEMPLATE}: expected a JSON object, got invalid JSON: {e}") from e
|
||||
if not isinstance(parsed, dict):
|
||||
raise ValueError(f"Invalid {ENV_DEFAULT_BANK_TEMPLATE}: expected a JSON object, got {type(parsed).__name__}")
|
||||
return parsed
|
||||
|
||||
|
||||
@dataclass
|
||||
class HindsightConfig:
|
||||
"""Configuration container for Hindsight API."""
|
||||
|
|
@ -820,6 +842,9 @@ class HindsightConfig:
|
|||
mcp_enabled_tools: list[str] | None # None = all tools; explicit list = allowlist
|
||||
mcp_stateless: bool # True = stateless HTTP (POST-only); False = stateful (supports GET/SSE)
|
||||
enable_bank_config_api: bool
|
||||
# Default bank template (static, server-level only). When set, the manifest is applied
|
||||
# to every newly-created bank, overriding the env/config defaults for any fields it sets.
|
||||
default_bank_template: dict | None
|
||||
|
||||
# Recall
|
||||
graph_retriever: str
|
||||
|
|
@ -1346,6 +1371,7 @@ class HindsightConfig:
|
|||
mcp_stateless=os.getenv(ENV_MCP_STATELESS, str(DEFAULT_MCP_STATELESS)).lower() == "true",
|
||||
enable_bank_config_api=os.getenv(ENV_ENABLE_BANK_CONFIG_API, str(DEFAULT_ENABLE_BANK_CONFIG_API)).lower()
|
||||
== "true",
|
||||
default_bank_template=_parse_default_bank_template(os.getenv(ENV_DEFAULT_BANK_TEMPLATE)),
|
||||
# Recall
|
||||
graph_retriever=os.getenv(ENV_GRAPH_RETRIEVER, DEFAULT_GRAPH_RETRIEVER),
|
||||
recall_max_concurrent=int(os.getenv(ENV_RECALL_MAX_CONCURRENT, str(DEFAULT_RECALL_MAX_CONCURRENT))),
|
||||
|
|
|
|||
|
|
@ -5153,7 +5153,13 @@ class MemoryEngine(MemoryEngineInterface):
|
|||
ctx = BankReadContext(bank_id=bank_id, operation="get_bank_profile", request_context=request_context)
|
||||
await self._validate_operation(self._operation_validator.validate_bank_read(ctx))
|
||||
pool = await self._get_pool()
|
||||
profile = await bank_utils.get_bank_profile(pool, bank_id)
|
||||
profile, created = await bank_utils.get_or_create_bank_profile(pool, bank_id)
|
||||
|
||||
# Apply HINDSIGHT_API_DEFAULT_BANK_TEMPLATE to freshly-created banks. Done
|
||||
# before reading the resolved config below so the template's overrides
|
||||
# (e.g. reflect_mission, dispositions) are visible on this very call.
|
||||
if created:
|
||||
await self._apply_default_bank_template(bank_id, request_context)
|
||||
|
||||
# reflect_mission and disposition in config take precedence over the legacy DB columns
|
||||
config_dict = await self._config_resolver.get_bank_config(bank_id, request_context)
|
||||
|
|
@ -5178,6 +5184,62 @@ class MemoryEngine(MemoryEngineInterface):
|
|||
"mission": mission,
|
||||
}
|
||||
|
||||
async def _apply_default_bank_template(
|
||||
self,
|
||||
bank_id: str,
|
||||
request_context: "RequestContext",
|
||||
) -> None:
|
||||
"""Apply HINDSIGHT_API_DEFAULT_BANK_TEMPLATE to a freshly-created bank.
|
||||
|
||||
No-op if the env var is unset. A malformed default template is logged
|
||||
and swallowed here rather than raised, so a bad server-level setting
|
||||
cannot wedge bank creation across all callers. Misconfiguration is
|
||||
still surfaced loudly via `logger.error`.
|
||||
"""
|
||||
from ..config import get_config
|
||||
|
||||
template_dict = get_config().default_bank_template
|
||||
if not template_dict:
|
||||
return
|
||||
|
||||
# Lazy import to avoid a cycle (http.py imports memory_engine).
|
||||
from pydantic import ValidationError
|
||||
|
||||
from hindsight_api.api.http import (
|
||||
BankTemplateManifest,
|
||||
apply_bank_template_manifest,
|
||||
validate_bank_template,
|
||||
)
|
||||
|
||||
try:
|
||||
manifest = BankTemplateManifest.model_validate(template_dict)
|
||||
except ValidationError as e:
|
||||
errors = [f"{'.'.join(str(loc) for loc in err['loc'])}: {err['msg']}" for err in e.errors()]
|
||||
logger.error(
|
||||
"HINDSIGHT_API_DEFAULT_BANK_TEMPLATE failed schema validation "
|
||||
f"and will be ignored for bank '{bank_id}': {'; '.join(errors)}"
|
||||
)
|
||||
return
|
||||
|
||||
semantic_errors = validate_bank_template(manifest)
|
||||
if semantic_errors:
|
||||
logger.error(
|
||||
"HINDSIGHT_API_DEFAULT_BANK_TEMPLATE failed semantic validation "
|
||||
f"and will be ignored for bank '{bank_id}': {'; '.join(semantic_errors)}"
|
||||
)
|
||||
return
|
||||
|
||||
try:
|
||||
await apply_bank_template_manifest(
|
||||
memory=self,
|
||||
bank_id=bank_id,
|
||||
manifest=manifest,
|
||||
request_context=request_context,
|
||||
)
|
||||
logger.info(f"Applied HINDSIGHT_API_DEFAULT_BANK_TEMPLATE to newly-created bank '{bank_id}'")
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to apply HINDSIGHT_API_DEFAULT_BANK_TEMPLATE to bank '{bank_id}': {e}")
|
||||
|
||||
async def update_bank_disposition(
|
||||
self,
|
||||
bank_id: str,
|
||||
|
|
@ -7789,7 +7851,9 @@ class MemoryEngine(MemoryEngineInterface):
|
|||
|
||||
# Ensure the bank row exists before inserting async_operations (which now has a FK).
|
||||
# Banks are created lazily on first retain, but the FK requires the row to exist first.
|
||||
await bank_utils.get_bank_profile(pool, bank_id)
|
||||
_, created = await bank_utils.get_or_create_bank_profile(pool, bank_id)
|
||||
if created:
|
||||
await self._apply_default_bank_template(bank_id, request_context)
|
||||
|
||||
# Create typed metadata for parent operation
|
||||
parent_metadata = BatchRetainParentMetadata(
|
||||
|
|
|
|||
|
|
@ -113,6 +113,22 @@ async def get_bank_profile(pool, bank_id: str) -> BankProfile:
|
|||
Returns:
|
||||
BankProfile with name, typed DispositionTraits, and mission
|
||||
"""
|
||||
profile, _ = await get_or_create_bank_profile(pool, bank_id)
|
||||
return profile
|
||||
|
||||
|
||||
async def get_or_create_bank_profile(pool, bank_id: str) -> tuple[BankProfile, bool]:
|
||||
"""
|
||||
Get bank profile, auto-creating with defaults if it doesn't exist.
|
||||
|
||||
Same as get_bank_profile, but also returns a flag indicating whether the
|
||||
bank was freshly created on this call. Used by the memory engine to apply
|
||||
the HINDSIGHT_API_DEFAULT_BANK_TEMPLATE hook on first bank creation.
|
||||
|
||||
Returns:
|
||||
Tuple of (BankProfile, created) where created is True if the bank
|
||||
did not exist before this call.
|
||||
"""
|
||||
async with acquire_with_retry(pool) as conn:
|
||||
# Try to get existing bank
|
||||
row = await conn.fetchrow(
|
||||
|
|
@ -129,10 +145,13 @@ async def get_bank_profile(pool, bank_id: str) -> BankProfile:
|
|||
if isinstance(disposition_data, str):
|
||||
disposition_data = json.loads(disposition_data)
|
||||
|
||||
return BankProfile(
|
||||
return (
|
||||
BankProfile(
|
||||
name=row["name"],
|
||||
disposition=DispositionTraits(**disposition_data),
|
||||
mission=row["mission"] or "",
|
||||
),
|
||||
False,
|
||||
)
|
||||
|
||||
# Bank doesn't exist, create with defaults.
|
||||
|
|
@ -153,11 +172,15 @@ async def get_bank_profile(pool, bank_id: str) -> BankProfile:
|
|||
internal_id,
|
||||
)
|
||||
|
||||
if inserted:
|
||||
created = inserted is not None
|
||||
if created:
|
||||
# Fresh insert — create per-bank vector indexes (instant on empty bank)
|
||||
await create_bank_vector_indexes(conn, bank_id, str(internal_id))
|
||||
|
||||
return BankProfile(name=bank_id, disposition=DispositionTraits(**DEFAULT_DISPOSITION), mission="")
|
||||
return (
|
||||
BankProfile(name=bank_id, disposition=DispositionTraits(**DEFAULT_DISPOSITION), mission=""),
|
||||
created,
|
||||
)
|
||||
|
||||
|
||||
async def update_bank_disposition(pool, bank_id: str, disposition: dict[str, int]) -> None:
|
||||
|
|
|
|||
|
|
@ -34,7 +34,12 @@ async def test_submit_async_retain_includes_document_tags_in_task_payload():
|
|||
contents = [{"content": "Async retain payload test."}]
|
||||
document_tags = ["scope:tools", "user:alice"]
|
||||
|
||||
with patch("hindsight_api.engine.memory_engine.bank_utils.get_bank_profile", new_callable=AsyncMock):
|
||||
# Return (profile, created=False) so the default-template-on-create hook is skipped.
|
||||
with patch(
|
||||
"hindsight_api.engine.memory_engine.bank_utils.get_or_create_bank_profile",
|
||||
new_callable=AsyncMock,
|
||||
return_value=(MagicMock(), False),
|
||||
):
|
||||
result = await MemoryEngine.submit_async_retain(
|
||||
engine,
|
||||
bank_id="bank-1",
|
||||
|
|
|
|||
|
|
@ -598,3 +598,182 @@ class TestExport:
|
|||
assert resp.status_code == 200
|
||||
data = resp.json()
|
||||
assert data["version"] == "1"
|
||||
|
||||
|
||||
class TestDefaultBankTemplateEnvVar:
|
||||
"""Tests for HINDSIGHT_API_DEFAULT_BANK_TEMPLATE — a server-level env var
|
||||
whose manifest is applied automatically to every newly-created bank."""
|
||||
|
||||
@pytest.fixture
|
||||
def default_template(self):
|
||||
return {
|
||||
"version": "1",
|
||||
"bank": {
|
||||
"reflect_mission": "default-env-mission",
|
||||
"retain_extraction_mode": "verbose",
|
||||
"disposition_empathy": 5,
|
||||
"disposition_skepticism": 1,
|
||||
},
|
||||
"mental_models": [
|
||||
{
|
||||
"id": "default-env-model",
|
||||
"name": "Default Env Model",
|
||||
"source_query": "What is the default?",
|
||||
},
|
||||
],
|
||||
"directives": [
|
||||
{
|
||||
"name": "Default Env Directive",
|
||||
"content": "Follow the default behavior.",
|
||||
"priority": 7,
|
||||
},
|
||||
],
|
||||
}
|
||||
|
||||
@pytest.fixture
|
||||
def _patched_default_template(self, monkeypatch, default_template):
|
||||
"""Install the default template on the already-initialized global config.
|
||||
|
||||
We can't rely on env-var resolution here: MemoryEngine (and its
|
||||
ConfigResolver) snapshot the global config at fixture init time.
|
||||
Patching the field directly keeps the test deterministic while still
|
||||
exercising the same code path that reads `get_config().default_bank_template`.
|
||||
"""
|
||||
from hindsight_api.config import _get_raw_config
|
||||
|
||||
raw = _get_raw_config()
|
||||
monkeypatch.setattr(raw, "default_bank_template", default_template)
|
||||
yield default_template
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_default_template_applied_on_new_bank(
|
||||
self, api_client, bank_id, _patched_default_template
|
||||
):
|
||||
"""Creating a new bank applies the default template (config + mental models + directives)."""
|
||||
# Trigger bank auto-creation via GET profile
|
||||
resp = await api_client.put(f"/v1/default/banks/{bank_id}", json={})
|
||||
assert resp.status_code == 200
|
||||
|
||||
# Config from template should be present as bank overrides
|
||||
config_resp = await api_client.get(f"/v1/default/banks/{bank_id}/config")
|
||||
assert config_resp.status_code == 200
|
||||
overrides = config_resp.json()["overrides"]
|
||||
assert overrides["reflect_mission"] == "default-env-mission"
|
||||
assert overrides["retain_extraction_mode"] == "verbose"
|
||||
assert overrides["disposition_empathy"] == 5
|
||||
assert overrides["disposition_skepticism"] == 1
|
||||
|
||||
# Mental model from template should exist
|
||||
mm_resp = await api_client.get(f"/v1/default/banks/{bank_id}/mental-models/default-env-model")
|
||||
assert mm_resp.status_code == 200
|
||||
assert mm_resp.json()["name"] == "Default Env Model"
|
||||
|
||||
# Directive from template should exist
|
||||
dir_resp = await api_client.get(f"/v1/default/banks/{bank_id}/directives")
|
||||
assert dir_resp.status_code == 200
|
||||
names = [d["name"] for d in dir_resp.json()["items"]]
|
||||
assert "Default Env Directive" in names
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_default_template_overrides_env_config_defaults(
|
||||
self, api_client, bank_id, monkeypatch, default_template
|
||||
):
|
||||
"""Fields set by the default template override server-level env-var defaults.
|
||||
|
||||
We point both HINDSIGHT_API_RETAIN_EXTRACTION_MODE (env) and the
|
||||
default template at different values, then confirm the template wins
|
||||
via the per-bank config overrides layer (highest precedence).
|
||||
"""
|
||||
from hindsight_api.config import _get_raw_config
|
||||
|
||||
raw = _get_raw_config()
|
||||
# Simulate an env-level default of "concise", overridden by a template that sets "verbose".
|
||||
monkeypatch.setattr(raw, "retain_extraction_mode", "concise")
|
||||
monkeypatch.setattr(raw, "default_bank_template", default_template)
|
||||
|
||||
resp = await api_client.put(f"/v1/default/banks/{bank_id}", json={})
|
||||
assert resp.status_code == 200
|
||||
|
||||
config_resp = await api_client.get(f"/v1/default/banks/{bank_id}/config")
|
||||
overrides = config_resp.json()["overrides"]
|
||||
# Template value wins at the bank-override layer.
|
||||
assert overrides["retain_extraction_mode"] == "verbose"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_default_template_not_reapplied_on_existing_bank(
|
||||
self, api_client, bank_id, _patched_default_template
|
||||
):
|
||||
"""Template only applies on FIRST creation; subsequent puts are no-ops."""
|
||||
# First hit creates the bank and applies the template
|
||||
resp = await api_client.put(f"/v1/default/banks/{bank_id}", json={})
|
||||
assert resp.status_code == 200
|
||||
|
||||
# User explicitly overrides a template-set field
|
||||
patch_resp = await api_client.patch(
|
||||
f"/v1/default/banks/{bank_id}/config",
|
||||
json={"updates": {"reflect_mission": "user-override"}},
|
||||
)
|
||||
assert patch_resp.status_code == 200
|
||||
|
||||
# Second put — template must NOT be reapplied (would clobber the override)
|
||||
resp = await api_client.put(f"/v1/default/banks/{bank_id}", json={})
|
||||
assert resp.status_code == 200
|
||||
|
||||
config_resp = await api_client.get(f"/v1/default/banks/{bank_id}/config")
|
||||
assert config_resp.json()["overrides"]["reflect_mission"] == "user-override"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_default_template_unset_is_noop(self, api_client, bank_id):
|
||||
"""With the env var unset (fixture default), bank creation behaves as before."""
|
||||
resp = await api_client.put(f"/v1/default/banks/{bank_id}", json={})
|
||||
assert resp.status_code == 200
|
||||
|
||||
# No template = no overrides
|
||||
config_resp = await api_client.get(f"/v1/default/banks/{bank_id}/config")
|
||||
assert config_resp.json()["overrides"] == {}
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_default_template_malformed_is_swallowed(
|
||||
self, api_client, bank_id, monkeypatch
|
||||
):
|
||||
"""A malformed default template is logged and ignored — bank creation still succeeds."""
|
||||
from hindsight_api.config import _get_raw_config
|
||||
|
||||
raw = _get_raw_config()
|
||||
# Wrong version number fails Pydantic validation.
|
||||
monkeypatch.setattr(raw, "default_bank_template", {"version": "999"})
|
||||
|
||||
resp = await api_client.put(f"/v1/default/banks/{bank_id}", json={})
|
||||
# Bank creation must not fail even though the template is broken.
|
||||
assert resp.status_code == 200
|
||||
|
||||
def test_parse_default_bank_template_valid_json(self, monkeypatch):
|
||||
"""_parse_default_bank_template parses a valid JSON object env var."""
|
||||
from hindsight_api.config import _parse_default_bank_template
|
||||
|
||||
parsed = _parse_default_bank_template('{"version": "1", "bank": {"disposition_empathy": 4}}')
|
||||
assert parsed == {"version": "1", "bank": {"disposition_empathy": 4}}
|
||||
|
||||
def test_parse_default_bank_template_none_or_empty(self):
|
||||
"""Unset / empty env var resolves to None."""
|
||||
from hindsight_api.config import _parse_default_bank_template
|
||||
|
||||
assert _parse_default_bank_template(None) is None
|
||||
assert _parse_default_bank_template("") is None
|
||||
assert _parse_default_bank_template(" ") is None
|
||||
|
||||
def test_parse_default_bank_template_invalid_json_raises(self):
|
||||
"""Invalid JSON fails fast with a clear error."""
|
||||
from hindsight_api.config import _parse_default_bank_template
|
||||
|
||||
with pytest.raises(ValueError, match="HINDSIGHT_API_DEFAULT_BANK_TEMPLATE"):
|
||||
_parse_default_bank_template("not-json")
|
||||
|
||||
def test_parse_default_bank_template_non_object_raises(self):
|
||||
"""Non-object JSON (e.g. array, string) fails fast."""
|
||||
from hindsight_api.config import _parse_default_bank_template
|
||||
|
||||
with pytest.raises(ValueError, match="expected a JSON object"):
|
||||
_parse_default_bank_template("[1, 2, 3]")
|
||||
with pytest.raises(ValueError, match="expected a JSON object"):
|
||||
_parse_default_bank_template('"just a string"')
|
||||
|
|
|
|||
|
|
@ -1289,6 +1289,33 @@ Configuration fields are categorized for security:
|
|||
| Variable | Description | Default |
|
||||
|----------|-------------|---------|
|
||||
| `HINDSIGHT_API_ENABLE_BANK_CONFIG_API` | Enable per-bank config API | `true` |
|
||||
| `HINDSIGHT_API_DEFAULT_BANK_TEMPLATE` | Bank template manifest (JSON) applied automatically to every newly-created bank. See below. | _(unset)_ |
|
||||
|
||||
##### `HINDSIGHT_API_DEFAULT_BANK_TEMPLATE`
|
||||
|
||||
Server-level default bank template. When set, the manifest is applied once
|
||||
to every bank the server creates — triggered the first time a bank is
|
||||
touched (via `PUT /v1/default/banks/{bank_id}`, `/import`, `/retain`, etc.).
|
||||
The value is a JSON-encoded `BankTemplateManifest` with the same shape
|
||||
accepted by `POST /v1/default/banks/{bank_id}/import` (see the `bank`,
|
||||
`mental_models`, and `directives` sections in the Bank Templates API).
|
||||
|
||||
Precedence: fields set by the template become per-bank overrides, so they
|
||||
take precedence over the equivalent `HINDSIGHT_API_*` env-var defaults
|
||||
(e.g. `HINDSIGHT_API_RETAIN_EXTRACTION_MODE`). Users can still override
|
||||
individual fields later via `PATCH /v1/default/banks/{bank_id}/config`;
|
||||
the template is **not** re-applied on subsequent accesses, so explicit
|
||||
overrides are never clobbered.
|
||||
|
||||
A malformed manifest (bad JSON, unknown version, schema errors) is logged
|
||||
and ignored — bank creation still succeeds with plain defaults, so a
|
||||
broken server-level setting cannot wedge all callers.
|
||||
|
||||
Example (compact, single-line JSON as required by env vars):
|
||||
|
||||
```bash
|
||||
export HINDSIGHT_API_DEFAULT_BANK_TEMPLATE='{"version":"1","bank":{"reflect_mission":"Help support agents remember customer interactions.","retain_extraction_mode":"verbose","disposition_empathy":5},"directives":[{"name":"Be concise","content":"Always respond concisely.","priority":10}]}'
|
||||
```
|
||||
|
||||
#### API Endpoints
|
||||
|
||||
|
|
|
|||
|
|
@ -185,6 +185,31 @@ When you provide a `document_id`, Hindsight upserts the document: if a document
|
|||
|
||||
If you omit `document_id`, Hindsight assigns a random UUID per request, so re-ingesting the same content will create duplicate memories.
|
||||
|
||||
### update_mode
|
||||
|
||||
Controls how Hindsight handles an existing document when you retain with a `document_id` that already exists.
|
||||
|
||||
| Value | Behaviour |
|
||||
|-------|-----------|
|
||||
| `"replace"` *(default)* | Deletes the old document and all its memories, then processes the new content from scratch. This is the standard upsert described above. |
|
||||
| `"append"` | Concatenates the new content onto the existing document text and reprocesses the combined document. Delta retain automatically skips unchanged chunks, so only the new portion triggers LLM extraction. |
|
||||
|
||||
Append mode requires a `document_id` — without one there is no existing document to append to.
|
||||
|
||||
**When to use append**: Use `"append"` for content that grows incrementally — for example, a log file, a journal, or a chat transcript where you receive new messages one at a time. Instead of re-sending the entire history on each update, send only the new content with `update_mode: "append"` and Hindsight will efficiently merge it with what it already has.
|
||||
|
||||
```json
|
||||
{
|
||||
"items": [
|
||||
{
|
||||
"content": "New entry to add to the existing document.",
|
||||
"document_id": "my-growing-doc",
|
||||
"update_mode": "append"
|
||||
}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
### entities
|
||||
|
||||
A list of entities you want to guarantee are recognized, merged with any entities the LLM extracts automatically. Each entry has a `text` field (the entity name) and an optional `type` (e.g., `"PERSON"`, `"ORG"`, `"CONCEPT"` — defaults to `"CONCEPT"` if omitted).
|
||||
|
|
|
|||
|
|
@ -1289,6 +1289,33 @@ Configuration fields are categorized for security:
|
|||
| Variable | Description | Default |
|
||||
|----------|-------------|---------|
|
||||
| `HINDSIGHT_API_ENABLE_BANK_CONFIG_API` | Enable per-bank config API | `true` |
|
||||
| `HINDSIGHT_API_DEFAULT_BANK_TEMPLATE` | Bank template manifest (JSON) applied automatically to every newly-created bank. See below. | _(unset)_ |
|
||||
|
||||
##### `HINDSIGHT_API_DEFAULT_BANK_TEMPLATE`
|
||||
|
||||
Server-level default bank template. When set, the manifest is applied once
|
||||
to every bank the server creates — triggered the first time a bank is
|
||||
touched (via `PUT /v1/default/banks/{bank_id}`, `/import`, `/retain`, etc.).
|
||||
The value is a JSON-encoded `BankTemplateManifest` with the same shape
|
||||
accepted by `POST /v1/default/banks/{bank_id}/import` (see the `bank`,
|
||||
`mental_models`, and `directives` sections in the Bank Templates API).
|
||||
|
||||
Precedence: fields set by the template become per-bank overrides, so they
|
||||
take precedence over the equivalent `HINDSIGHT_API_*` env-var defaults
|
||||
(e.g. `HINDSIGHT_API_RETAIN_EXTRACTION_MODE`). Users can still override
|
||||
individual fields later via `PATCH /v1/default/banks/{bank_id}/config`;
|
||||
the template is **not** re-applied on subsequent accesses, so explicit
|
||||
overrides are never clobbered.
|
||||
|
||||
A malformed manifest (bad JSON, unknown version, schema errors) is logged
|
||||
and ignored — bank creation still succeeds with plain defaults, so a
|
||||
broken server-level setting cannot wedge all callers.
|
||||
|
||||
Example (compact, single-line JSON as required by env vars):
|
||||
|
||||
```bash
|
||||
export HINDSIGHT_API_DEFAULT_BANK_TEMPLATE='{"version":"1","bank":{"reflect_mission":"Help support agents remember customer interactions.","retain_extraction_mode":"verbose","disposition_empathy":5},"directives":[{"name":"Be concise","content":"Always respond concisely.","priority":10}]}'
|
||||
```
|
||||
|
||||
#### API Endpoints
|
||||
|
||||
|
|
|
|||
Loading…
Reference in a new issue