diff --git a/hindsight-api-slim/hindsight_api/api/http.py b/hindsight-api-slim/hindsight_api/api/http.py index 5645ebce..c6b55b79 100644 --- a/hindsight-api-slim/hindsight_api/api/http.py +++ b/hindsight-api-slim/hindsight_api/api/http.py @@ -1800,6 +1800,150 @@ class BankTemplateImportResponse(BaseModel): dry_run: bool = Field(default=False, description="True if this was a validation-only run") +def validate_bank_template(manifest: "BankTemplateManifest") -> list[str]: + """Validate a parsed manifest beyond Pydantic's structural checks. + + Returns a list of human-readable error strings (e.g. invalid + extraction mode values, conflicting settings). + """ + errors: list[str] = [] + if manifest.bank: + bank = manifest.bank + if bank.retain_extraction_mode is not None: + valid_modes = ("concise", "verbose", "custom", "chunks") + if bank.retain_extraction_mode not in valid_modes: + errors.append( + f"bank.retain_extraction_mode: must be one of {valid_modes}, got '{bank.retain_extraction_mode}'" + ) + if bank.retain_custom_instructions and bank.retain_extraction_mode != "custom": + errors.append("bank.retain_custom_instructions: requires retain_extraction_mode='custom'") + if manifest.mental_models: + for i, mm in enumerate(manifest.mental_models): + if not mm.name.strip(): + errors.append(f"mental_models[{i}].name: must not be empty") + if not mm.source_query.strip(): + errors.append(f"mental_models[{i}].source_query: must not be empty") + if manifest.directives: + for i, d in enumerate(manifest.directives): + if not d.name.strip(): + errors.append(f"directives[{i}].name: must not be empty") + if not d.content.strip(): + errors.append(f"directives[{i}].content: must not be empty") + return errors + + +async def apply_bank_template_manifest( + memory, + bank_id: str, + manifest: "BankTemplateManifest", + request_context: "RequestContext", +) -> "BankTemplateImportResponse": + """Apply a validated BankTemplateManifest to an existing bank. + + Shared by the /import endpoint and the default-template-on-create hook + driven by HINDSIGHT_API_DEFAULT_BANK_TEMPLATE. The bank MUST already + exist; caller is responsible for validation (Pydantic + validate_bank_template). + """ + config_applied = False + if manifest.bank: + config_updates = manifest.bank.get_config_updates() + if config_updates: + await memory._config_resolver.update_bank_config(bank_id, config_updates, request_context) + config_applied = True + + created_ids: list[str] = [] + updated_ids: list[str] = [] + operation_ids: list[str] = [] + + if manifest.mental_models: + # Fetch existing mental models to decide create vs update + existing = await memory.list_mental_models(bank_id=bank_id, request_context=request_context) + existing_by_id = {m["id"]: m for m in existing} + + for mm in manifest.mental_models: + if mm.id in existing_by_id: + await memory.update_mental_model( + bank_id=bank_id, + mental_model_id=mm.id, + name=mm.name, + source_query=mm.source_query, + max_tokens=mm.max_tokens, + tags=mm.tags if mm.tags else None, + trigger=mm.trigger.model_dump() if mm.trigger else None, + request_context=request_context, + ) + result = await memory.submit_async_refresh_mental_model( + bank_id=bank_id, + mental_model_id=mm.id, + request_context=request_context, + ) + operation_ids.append(result["operation_id"]) + updated_ids.append(mm.id) + else: + mental_model = await memory.create_mental_model( + bank_id=bank_id, + name=mm.name, + source_query=mm.source_query, + content="Generating content...", + mental_model_id=mm.id, + tags=mm.tags if mm.tags else None, + max_tokens=mm.max_tokens, + trigger=mm.trigger.model_dump() if mm.trigger else None, + request_context=request_context, + ) + result = await memory.submit_async_refresh_mental_model( + bank_id=bank_id, + mental_model_id=mental_model["id"], + request_context=request_context, + ) + operation_ids.append(result["operation_id"]) + created_ids.append(mm.id) + + directives_created: list[str] = [] + directives_updated: list[str] = [] + + if manifest.directives: + existing_directives = await memory.list_directives( + bank_id=bank_id, active_only=False, request_context=request_context + ) + existing_by_name = {d["name"]: d for d in existing_directives} + + for directive in manifest.directives: + if directive.name in existing_by_name: + await memory.update_directive( + bank_id=bank_id, + directive_id=existing_by_name[directive.name]["id"], + content=directive.content, + priority=directive.priority, + is_active=directive.is_active, + tags=directive.tags if directive.tags else None, + request_context=request_context, + ) + directives_updated.append(directive.name) + else: + await memory.create_directive( + bank_id=bank_id, + name=directive.name, + content=directive.content, + priority=directive.priority, + is_active=directive.is_active, + tags=directive.tags if directive.tags else None, + request_context=request_context, + ) + directives_created.append(directive.name) + + return BankTemplateImportResponse( + bank_id=bank_id, + config_applied=config_applied, + mental_models_created=created_ids, + mental_models_updated=updated_ids, + directives_created=directives_created, + directives_updated=directives_updated, + operation_ids=operation_ids, + dry_run=False, + ) + + class OperationResponse(BaseModel): """Response model for a single async operation.""" @@ -4377,38 +4521,6 @@ def _register_routes(app: FastAPI): # Bank Template Import / Export # ===================================================================== - def _validate_template(manifest: BankTemplateManifest) -> list[str]: - """Validate a parsed manifest beyond Pydantic's structural checks. - - Returns a list of human-readable error strings (e.g. invalid - extraction mode values, conflicting settings). - """ - errors: list[str] = [] - if manifest.bank: - bank = manifest.bank - if bank.retain_extraction_mode is not None: - valid_modes = ("concise", "verbose", "custom", "chunks") - if bank.retain_extraction_mode not in valid_modes: - errors.append( - f"bank.retain_extraction_mode: must be one of {valid_modes}, " - f"got '{bank.retain_extraction_mode}'" - ) - if bank.retain_custom_instructions and bank.retain_extraction_mode != "custom": - errors.append("bank.retain_custom_instructions: requires retain_extraction_mode='custom'") - if manifest.mental_models: - for i, mm in enumerate(manifest.mental_models): - if not mm.name.strip(): - errors.append(f"mental_models[{i}].name: must not be empty") - if not mm.source_query.strip(): - errors.append(f"mental_models[{i}].source_query: must not be empty") - if manifest.directives: - for i, d in enumerate(manifest.directives): - if not d.name.strip(): - errors.append(f"directives[{i}].name: must not be empty") - if not d.content.strip(): - errors.append(f"directives[{i}].content: must not be empty") - return errors - @app.post( "/v1/default/banks/{bank_id}/import", response_model=BankTemplateImportResponse, @@ -4444,7 +4556,7 @@ def _register_routes(app: FastAPI): ) # Semantic validation beyond Pydantic structural checks - validation_errors = _validate_template(body) + validation_errors = validate_bank_template(body) if validation_errors: raise HTTPException( status_code=400, @@ -4462,107 +4574,11 @@ def _register_routes(app: FastAPI): # Ensure bank exists (auto-creates with defaults if needed) await app.state.memory.get_bank_profile(bank_id, request_context=request_context) - config_applied = False - if body.bank: - config_updates = body.bank.get_config_updates() - if config_updates: - await app.state.memory._config_resolver.update_bank_config(bank_id, config_updates, request_context) - config_applied = True - - created_ids: list[str] = [] - updated_ids: list[str] = [] - operation_ids: list[str] = [] - - if body.mental_models: - # Fetch existing mental models to decide create vs update - existing = await app.state.memory.list_mental_models(bank_id=bank_id, request_context=request_context) - existing_by_id = {m["id"]: m for m in existing} - - for mm in body.mental_models: - if mm.id in existing_by_id: - # Update existing mental model metadata - await app.state.memory.update_mental_model( - bank_id=bank_id, - mental_model_id=mm.id, - name=mm.name, - source_query=mm.source_query, - max_tokens=mm.max_tokens, - tags=mm.tags if mm.tags else None, - trigger=mm.trigger.model_dump() if mm.trigger else None, - request_context=request_context, - ) - # Schedule a refresh to regenerate content with updated query - result = await app.state.memory.submit_async_refresh_mental_model( - bank_id=bank_id, - mental_model_id=mm.id, - request_context=request_context, - ) - operation_ids.append(result["operation_id"]) - updated_ids.append(mm.id) - else: - # Create new mental model - mental_model = await app.state.memory.create_mental_model( - bank_id=bank_id, - name=mm.name, - source_query=mm.source_query, - content="Generating content...", - mental_model_id=mm.id, - tags=mm.tags if mm.tags else None, - max_tokens=mm.max_tokens, - trigger=mm.trigger.model_dump() if mm.trigger else None, - request_context=request_context, - ) - result = await app.state.memory.submit_async_refresh_mental_model( - bank_id=bank_id, - mental_model_id=mental_model["id"], - request_context=request_context, - ) - operation_ids.append(result["operation_id"]) - created_ids.append(mm.id) - - directives_created: list[str] = [] - directives_updated: list[str] = [] - - if body.directives: - # Fetch existing directives to decide create vs update (matched by name) - existing_directives = await app.state.memory.list_directives( - bank_id=bank_id, active_only=False, request_context=request_context - ) - existing_by_name = {d["name"]: d for d in existing_directives} - - for directive in body.directives: - if directive.name in existing_by_name: - await app.state.memory.update_directive( - bank_id=bank_id, - directive_id=existing_by_name[directive.name]["id"], - content=directive.content, - priority=directive.priority, - is_active=directive.is_active, - tags=directive.tags if directive.tags else None, - request_context=request_context, - ) - directives_updated.append(directive.name) - else: - await app.state.memory.create_directive( - bank_id=bank_id, - name=directive.name, - content=directive.content, - priority=directive.priority, - is_active=directive.is_active, - tags=directive.tags if directive.tags else None, - request_context=request_context, - ) - directives_created.append(directive.name) - - return BankTemplateImportResponse( + return await apply_bank_template_manifest( + memory=app.state.memory, bank_id=bank_id, - config_applied=config_applied, - mental_models_created=created_ids, - mental_models_updated=updated_ids, - directives_created=directives_created, - directives_updated=directives_updated, - operation_ids=operation_ids, - dry_run=False, + manifest=body, + request_context=request_context, ) except ValueError as e: raise HTTPException(status_code=400, detail=str(e)) @@ -4964,7 +4980,9 @@ def _register_routes(app: FastAPI): from hindsight_api.engine.retain import bank_utils # Ensure the bank row exists before inserting into webhooks (FK constraint). - await bank_utils.get_bank_profile(pool, bank_id) + _, created = await bank_utils.get_or_create_bank_profile(pool, bank_id) + if created: + await app.state.memory._apply_default_bank_template(bank_id, request_context) webhook_id = uuid.uuid4() now = datetime.now(timezone.utc).isoformat() diff --git a/hindsight-api-slim/hindsight_api/config.py b/hindsight-api-slim/hindsight_api/config.py index 6f7859da..9b346671 100644 --- a/hindsight-api-slim/hindsight_api/config.py +++ b/hindsight-api-slim/hindsight_api/config.py @@ -265,6 +265,7 @@ ENV_MCP_ENABLED = "HINDSIGHT_API_MCP_ENABLED" ENV_MCP_ENABLED_TOOLS = "HINDSIGHT_API_MCP_ENABLED_TOOLS" ENV_MCP_STATELESS = "HINDSIGHT_API_MCP_STATELESS" ENV_ENABLE_BANK_CONFIG_API = "HINDSIGHT_API_ENABLE_BANK_CONFIG_API" +ENV_DEFAULT_BANK_TEMPLATE = "HINDSIGHT_API_DEFAULT_BANK_TEMPLATE" ENV_GRAPH_RETRIEVER = "HINDSIGHT_API_GRAPH_RETRIEVER" ENV_RECALL_MAX_CONCURRENT = "HINDSIGHT_API_RECALL_MAX_CONCURRENT" ENV_RECALL_CONNECTION_BUDGET = "HINDSIGHT_API_RECALL_CONNECTION_BUDGET" @@ -502,6 +503,7 @@ DEFAULT_MCP_ENABLED = True DEFAULT_MCP_ENABLED_TOOLS: list[str] | None = None # None = all tools enabled DEFAULT_MCP_STATELESS = False # False = stateful (supports SSE/GET); True = stateless (POST-only) DEFAULT_ENABLE_BANK_CONFIG_API = True +DEFAULT_DEFAULT_BANK_TEMPLATE: dict | None = None # BankTemplateManifest dict applied to newly-created banks DEFAULT_GRAPH_RETRIEVER = "link_expansion" DEFAULT_RECALL_MAX_CONCURRENT = 32 # Max concurrent recall operations per worker DEFAULT_RECALL_CONNECTION_BUDGET = 4 # Max concurrent DB connections per recall operation @@ -676,6 +678,26 @@ def _get_default_model_for_provider(provider: str) -> str: return PROVIDER_DEFAULT_MODELS.get(provider.lower(), DEFAULT_LLM_MODEL) +def _parse_default_bank_template(raw: str | None) -> dict | None: + """ + Parse HINDSIGHT_API_DEFAULT_BANK_TEMPLATE as JSON. + + The env var holds a BankTemplateManifest (JSON object) applied verbatim to + every newly-created bank. Full Pydantic validation is deferred to bank + creation time (to avoid pulling API models into config.py), but we fail + fast here if the value is not valid JSON or not a JSON object. + """ + if raw is None or raw.strip() == "": + return DEFAULT_DEFAULT_BANK_TEMPLATE + try: + parsed = json.loads(raw) + except json.JSONDecodeError as e: + raise ValueError(f"Invalid {ENV_DEFAULT_BANK_TEMPLATE}: expected a JSON object, got invalid JSON: {e}") from e + if not isinstance(parsed, dict): + raise ValueError(f"Invalid {ENV_DEFAULT_BANK_TEMPLATE}: expected a JSON object, got {type(parsed).__name__}") + return parsed + + @dataclass class HindsightConfig: """Configuration container for Hindsight API.""" @@ -820,6 +842,9 @@ class HindsightConfig: mcp_enabled_tools: list[str] | None # None = all tools; explicit list = allowlist mcp_stateless: bool # True = stateless HTTP (POST-only); False = stateful (supports GET/SSE) enable_bank_config_api: bool + # Default bank template (static, server-level only). When set, the manifest is applied + # to every newly-created bank, overriding the env/config defaults for any fields it sets. + default_bank_template: dict | None # Recall graph_retriever: str @@ -1346,6 +1371,7 @@ class HindsightConfig: mcp_stateless=os.getenv(ENV_MCP_STATELESS, str(DEFAULT_MCP_STATELESS)).lower() == "true", enable_bank_config_api=os.getenv(ENV_ENABLE_BANK_CONFIG_API, str(DEFAULT_ENABLE_BANK_CONFIG_API)).lower() == "true", + default_bank_template=_parse_default_bank_template(os.getenv(ENV_DEFAULT_BANK_TEMPLATE)), # Recall graph_retriever=os.getenv(ENV_GRAPH_RETRIEVER, DEFAULT_GRAPH_RETRIEVER), recall_max_concurrent=int(os.getenv(ENV_RECALL_MAX_CONCURRENT, str(DEFAULT_RECALL_MAX_CONCURRENT))), diff --git a/hindsight-api-slim/hindsight_api/engine/memory_engine.py b/hindsight-api-slim/hindsight_api/engine/memory_engine.py index 60bfc456..5bea103d 100644 --- a/hindsight-api-slim/hindsight_api/engine/memory_engine.py +++ b/hindsight-api-slim/hindsight_api/engine/memory_engine.py @@ -5153,7 +5153,13 @@ class MemoryEngine(MemoryEngineInterface): ctx = BankReadContext(bank_id=bank_id, operation="get_bank_profile", request_context=request_context) await self._validate_operation(self._operation_validator.validate_bank_read(ctx)) pool = await self._get_pool() - profile = await bank_utils.get_bank_profile(pool, bank_id) + profile, created = await bank_utils.get_or_create_bank_profile(pool, bank_id) + + # Apply HINDSIGHT_API_DEFAULT_BANK_TEMPLATE to freshly-created banks. Done + # before reading the resolved config below so the template's overrides + # (e.g. reflect_mission, dispositions) are visible on this very call. + if created: + await self._apply_default_bank_template(bank_id, request_context) # reflect_mission and disposition in config take precedence over the legacy DB columns config_dict = await self._config_resolver.get_bank_config(bank_id, request_context) @@ -5178,6 +5184,62 @@ class MemoryEngine(MemoryEngineInterface): "mission": mission, } + async def _apply_default_bank_template( + self, + bank_id: str, + request_context: "RequestContext", + ) -> None: + """Apply HINDSIGHT_API_DEFAULT_BANK_TEMPLATE to a freshly-created bank. + + No-op if the env var is unset. A malformed default template is logged + and swallowed here rather than raised, so a bad server-level setting + cannot wedge bank creation across all callers. Misconfiguration is + still surfaced loudly via `logger.error`. + """ + from ..config import get_config + + template_dict = get_config().default_bank_template + if not template_dict: + return + + # Lazy import to avoid a cycle (http.py imports memory_engine). + from pydantic import ValidationError + + from hindsight_api.api.http import ( + BankTemplateManifest, + apply_bank_template_manifest, + validate_bank_template, + ) + + try: + manifest = BankTemplateManifest.model_validate(template_dict) + except ValidationError as e: + errors = [f"{'.'.join(str(loc) for loc in err['loc'])}: {err['msg']}" for err in e.errors()] + logger.error( + "HINDSIGHT_API_DEFAULT_BANK_TEMPLATE failed schema validation " + f"and will be ignored for bank '{bank_id}': {'; '.join(errors)}" + ) + return + + semantic_errors = validate_bank_template(manifest) + if semantic_errors: + logger.error( + "HINDSIGHT_API_DEFAULT_BANK_TEMPLATE failed semantic validation " + f"and will be ignored for bank '{bank_id}': {'; '.join(semantic_errors)}" + ) + return + + try: + await apply_bank_template_manifest( + memory=self, + bank_id=bank_id, + manifest=manifest, + request_context=request_context, + ) + logger.info(f"Applied HINDSIGHT_API_DEFAULT_BANK_TEMPLATE to newly-created bank '{bank_id}'") + except Exception as e: + logger.error(f"Failed to apply HINDSIGHT_API_DEFAULT_BANK_TEMPLATE to bank '{bank_id}': {e}") + async def update_bank_disposition( self, bank_id: str, @@ -7789,7 +7851,9 @@ class MemoryEngine(MemoryEngineInterface): # Ensure the bank row exists before inserting async_operations (which now has a FK). # Banks are created lazily on first retain, but the FK requires the row to exist first. - await bank_utils.get_bank_profile(pool, bank_id) + _, created = await bank_utils.get_or_create_bank_profile(pool, bank_id) + if created: + await self._apply_default_bank_template(bank_id, request_context) # Create typed metadata for parent operation parent_metadata = BatchRetainParentMetadata( diff --git a/hindsight-api-slim/hindsight_api/engine/retain/bank_utils.py b/hindsight-api-slim/hindsight_api/engine/retain/bank_utils.py index e311629e..4f716803 100644 --- a/hindsight-api-slim/hindsight_api/engine/retain/bank_utils.py +++ b/hindsight-api-slim/hindsight_api/engine/retain/bank_utils.py @@ -113,6 +113,22 @@ async def get_bank_profile(pool, bank_id: str) -> BankProfile: Returns: BankProfile with name, typed DispositionTraits, and mission """ + profile, _ = await get_or_create_bank_profile(pool, bank_id) + return profile + + +async def get_or_create_bank_profile(pool, bank_id: str) -> tuple[BankProfile, bool]: + """ + Get bank profile, auto-creating with defaults if it doesn't exist. + + Same as get_bank_profile, but also returns a flag indicating whether the + bank was freshly created on this call. Used by the memory engine to apply + the HINDSIGHT_API_DEFAULT_BANK_TEMPLATE hook on first bank creation. + + Returns: + Tuple of (BankProfile, created) where created is True if the bank + did not exist before this call. + """ async with acquire_with_retry(pool) as conn: # Try to get existing bank row = await conn.fetchrow( @@ -129,10 +145,13 @@ async def get_bank_profile(pool, bank_id: str) -> BankProfile: if isinstance(disposition_data, str): disposition_data = json.loads(disposition_data) - return BankProfile( - name=row["name"], - disposition=DispositionTraits(**disposition_data), - mission=row["mission"] or "", + return ( + BankProfile( + name=row["name"], + disposition=DispositionTraits(**disposition_data), + mission=row["mission"] or "", + ), + False, ) # Bank doesn't exist, create with defaults. @@ -153,11 +172,15 @@ async def get_bank_profile(pool, bank_id: str) -> BankProfile: internal_id, ) - if inserted: + created = inserted is not None + if created: # Fresh insert — create per-bank vector indexes (instant on empty bank) await create_bank_vector_indexes(conn, bank_id, str(internal_id)) - return BankProfile(name=bank_id, disposition=DispositionTraits(**DEFAULT_DISPOSITION), mission="") + return ( + BankProfile(name=bank_id, disposition=DispositionTraits(**DEFAULT_DISPOSITION), mission=""), + created, + ) async def update_bank_disposition(pool, bank_id: str, disposition: dict[str, int]) -> None: diff --git a/hindsight-api-slim/tests/test_async_retain_tags.py b/hindsight-api-slim/tests/test_async_retain_tags.py index ed24a4ab..c8525bb5 100644 --- a/hindsight-api-slim/tests/test_async_retain_tags.py +++ b/hindsight-api-slim/tests/test_async_retain_tags.py @@ -34,7 +34,12 @@ async def test_submit_async_retain_includes_document_tags_in_task_payload(): contents = [{"content": "Async retain payload test."}] document_tags = ["scope:tools", "user:alice"] - with patch("hindsight_api.engine.memory_engine.bank_utils.get_bank_profile", new_callable=AsyncMock): + # Return (profile, created=False) so the default-template-on-create hook is skipped. + with patch( + "hindsight_api.engine.memory_engine.bank_utils.get_or_create_bank_profile", + new_callable=AsyncMock, + return_value=(MagicMock(), False), + ): result = await MemoryEngine.submit_async_retain( engine, bank_id="bank-1", diff --git a/hindsight-api-slim/tests/test_bank_templates.py b/hindsight-api-slim/tests/test_bank_templates.py index 4c13d899..cc323ab9 100644 --- a/hindsight-api-slim/tests/test_bank_templates.py +++ b/hindsight-api-slim/tests/test_bank_templates.py @@ -598,3 +598,182 @@ class TestExport: assert resp.status_code == 200 data = resp.json() assert data["version"] == "1" + + +class TestDefaultBankTemplateEnvVar: + """Tests for HINDSIGHT_API_DEFAULT_BANK_TEMPLATE — a server-level env var + whose manifest is applied automatically to every newly-created bank.""" + + @pytest.fixture + def default_template(self): + return { + "version": "1", + "bank": { + "reflect_mission": "default-env-mission", + "retain_extraction_mode": "verbose", + "disposition_empathy": 5, + "disposition_skepticism": 1, + }, + "mental_models": [ + { + "id": "default-env-model", + "name": "Default Env Model", + "source_query": "What is the default?", + }, + ], + "directives": [ + { + "name": "Default Env Directive", + "content": "Follow the default behavior.", + "priority": 7, + }, + ], + } + + @pytest.fixture + def _patched_default_template(self, monkeypatch, default_template): + """Install the default template on the already-initialized global config. + + We can't rely on env-var resolution here: MemoryEngine (and its + ConfigResolver) snapshot the global config at fixture init time. + Patching the field directly keeps the test deterministic while still + exercising the same code path that reads `get_config().default_bank_template`. + """ + from hindsight_api.config import _get_raw_config + + raw = _get_raw_config() + monkeypatch.setattr(raw, "default_bank_template", default_template) + yield default_template + + @pytest.mark.asyncio + async def test_default_template_applied_on_new_bank( + self, api_client, bank_id, _patched_default_template + ): + """Creating a new bank applies the default template (config + mental models + directives).""" + # Trigger bank auto-creation via GET profile + resp = await api_client.put(f"/v1/default/banks/{bank_id}", json={}) + assert resp.status_code == 200 + + # Config from template should be present as bank overrides + config_resp = await api_client.get(f"/v1/default/banks/{bank_id}/config") + assert config_resp.status_code == 200 + overrides = config_resp.json()["overrides"] + assert overrides["reflect_mission"] == "default-env-mission" + assert overrides["retain_extraction_mode"] == "verbose" + assert overrides["disposition_empathy"] == 5 + assert overrides["disposition_skepticism"] == 1 + + # Mental model from template should exist + mm_resp = await api_client.get(f"/v1/default/banks/{bank_id}/mental-models/default-env-model") + assert mm_resp.status_code == 200 + assert mm_resp.json()["name"] == "Default Env Model" + + # Directive from template should exist + dir_resp = await api_client.get(f"/v1/default/banks/{bank_id}/directives") + assert dir_resp.status_code == 200 + names = [d["name"] for d in dir_resp.json()["items"]] + assert "Default Env Directive" in names + + @pytest.mark.asyncio + async def test_default_template_overrides_env_config_defaults( + self, api_client, bank_id, monkeypatch, default_template + ): + """Fields set by the default template override server-level env-var defaults. + + We point both HINDSIGHT_API_RETAIN_EXTRACTION_MODE (env) and the + default template at different values, then confirm the template wins + via the per-bank config overrides layer (highest precedence). + """ + from hindsight_api.config import _get_raw_config + + raw = _get_raw_config() + # Simulate an env-level default of "concise", overridden by a template that sets "verbose". + monkeypatch.setattr(raw, "retain_extraction_mode", "concise") + monkeypatch.setattr(raw, "default_bank_template", default_template) + + resp = await api_client.put(f"/v1/default/banks/{bank_id}", json={}) + assert resp.status_code == 200 + + config_resp = await api_client.get(f"/v1/default/banks/{bank_id}/config") + overrides = config_resp.json()["overrides"] + # Template value wins at the bank-override layer. + assert overrides["retain_extraction_mode"] == "verbose" + + @pytest.mark.asyncio + async def test_default_template_not_reapplied_on_existing_bank( + self, api_client, bank_id, _patched_default_template + ): + """Template only applies on FIRST creation; subsequent puts are no-ops.""" + # First hit creates the bank and applies the template + resp = await api_client.put(f"/v1/default/banks/{bank_id}", json={}) + assert resp.status_code == 200 + + # User explicitly overrides a template-set field + patch_resp = await api_client.patch( + f"/v1/default/banks/{bank_id}/config", + json={"updates": {"reflect_mission": "user-override"}}, + ) + assert patch_resp.status_code == 200 + + # Second put — template must NOT be reapplied (would clobber the override) + resp = await api_client.put(f"/v1/default/banks/{bank_id}", json={}) + assert resp.status_code == 200 + + config_resp = await api_client.get(f"/v1/default/banks/{bank_id}/config") + assert config_resp.json()["overrides"]["reflect_mission"] == "user-override" + + @pytest.mark.asyncio + async def test_default_template_unset_is_noop(self, api_client, bank_id): + """With the env var unset (fixture default), bank creation behaves as before.""" + resp = await api_client.put(f"/v1/default/banks/{bank_id}", json={}) + assert resp.status_code == 200 + + # No template = no overrides + config_resp = await api_client.get(f"/v1/default/banks/{bank_id}/config") + assert config_resp.json()["overrides"] == {} + + @pytest.mark.asyncio + async def test_default_template_malformed_is_swallowed( + self, api_client, bank_id, monkeypatch + ): + """A malformed default template is logged and ignored — bank creation still succeeds.""" + from hindsight_api.config import _get_raw_config + + raw = _get_raw_config() + # Wrong version number fails Pydantic validation. + monkeypatch.setattr(raw, "default_bank_template", {"version": "999"}) + + resp = await api_client.put(f"/v1/default/banks/{bank_id}", json={}) + # Bank creation must not fail even though the template is broken. + assert resp.status_code == 200 + + def test_parse_default_bank_template_valid_json(self, monkeypatch): + """_parse_default_bank_template parses a valid JSON object env var.""" + from hindsight_api.config import _parse_default_bank_template + + parsed = _parse_default_bank_template('{"version": "1", "bank": {"disposition_empathy": 4}}') + assert parsed == {"version": "1", "bank": {"disposition_empathy": 4}} + + def test_parse_default_bank_template_none_or_empty(self): + """Unset / empty env var resolves to None.""" + from hindsight_api.config import _parse_default_bank_template + + assert _parse_default_bank_template(None) is None + assert _parse_default_bank_template("") is None + assert _parse_default_bank_template(" ") is None + + def test_parse_default_bank_template_invalid_json_raises(self): + """Invalid JSON fails fast with a clear error.""" + from hindsight_api.config import _parse_default_bank_template + + with pytest.raises(ValueError, match="HINDSIGHT_API_DEFAULT_BANK_TEMPLATE"): + _parse_default_bank_template("not-json") + + def test_parse_default_bank_template_non_object_raises(self): + """Non-object JSON (e.g. array, string) fails fast.""" + from hindsight_api.config import _parse_default_bank_template + + with pytest.raises(ValueError, match="expected a JSON object"): + _parse_default_bank_template("[1, 2, 3]") + with pytest.raises(ValueError, match="expected a JSON object"): + _parse_default_bank_template('"just a string"') diff --git a/hindsight-docs/docs/developer/configuration.md b/hindsight-docs/docs/developer/configuration.md index 1ae75b06..ace0d458 100644 --- a/hindsight-docs/docs/developer/configuration.md +++ b/hindsight-docs/docs/developer/configuration.md @@ -1289,6 +1289,33 @@ Configuration fields are categorized for security: | Variable | Description | Default | |----------|-------------|---------| | `HINDSIGHT_API_ENABLE_BANK_CONFIG_API` | Enable per-bank config API | `true` | +| `HINDSIGHT_API_DEFAULT_BANK_TEMPLATE` | Bank template manifest (JSON) applied automatically to every newly-created bank. See below. | _(unset)_ | + +##### `HINDSIGHT_API_DEFAULT_BANK_TEMPLATE` + +Server-level default bank template. When set, the manifest is applied once +to every bank the server creates — triggered the first time a bank is +touched (via `PUT /v1/default/banks/{bank_id}`, `/import`, `/retain`, etc.). +The value is a JSON-encoded `BankTemplateManifest` with the same shape +accepted by `POST /v1/default/banks/{bank_id}/import` (see the `bank`, +`mental_models`, and `directives` sections in the Bank Templates API). + +Precedence: fields set by the template become per-bank overrides, so they +take precedence over the equivalent `HINDSIGHT_API_*` env-var defaults +(e.g. `HINDSIGHT_API_RETAIN_EXTRACTION_MODE`). Users can still override +individual fields later via `PATCH /v1/default/banks/{bank_id}/config`; +the template is **not** re-applied on subsequent accesses, so explicit +overrides are never clobbered. + +A malformed manifest (bad JSON, unknown version, schema errors) is logged +and ignored — bank creation still succeeds with plain defaults, so a +broken server-level setting cannot wedge all callers. + +Example (compact, single-line JSON as required by env vars): + +```bash +export HINDSIGHT_API_DEFAULT_BANK_TEMPLATE='{"version":"1","bank":{"reflect_mission":"Help support agents remember customer interactions.","retain_extraction_mode":"verbose","disposition_empathy":5},"directives":[{"name":"Be concise","content":"Always respond concisely.","priority":10}]}' +``` #### API Endpoints diff --git a/skills/hindsight-docs/references/developer/api/retain.md b/skills/hindsight-docs/references/developer/api/retain.md index 084eee96..b013a36e 100644 --- a/skills/hindsight-docs/references/developer/api/retain.md +++ b/skills/hindsight-docs/references/developer/api/retain.md @@ -185,6 +185,31 @@ When you provide a `document_id`, Hindsight upserts the document: if a document If you omit `document_id`, Hindsight assigns a random UUID per request, so re-ingesting the same content will create duplicate memories. +### update_mode + +Controls how Hindsight handles an existing document when you retain with a `document_id` that already exists. + +| Value | Behaviour | +|-------|-----------| +| `"replace"` *(default)* | Deletes the old document and all its memories, then processes the new content from scratch. This is the standard upsert described above. | +| `"append"` | Concatenates the new content onto the existing document text and reprocesses the combined document. Delta retain automatically skips unchanged chunks, so only the new portion triggers LLM extraction. | + +Append mode requires a `document_id` — without one there is no existing document to append to. + +**When to use append**: Use `"append"` for content that grows incrementally — for example, a log file, a journal, or a chat transcript where you receive new messages one at a time. Instead of re-sending the entire history on each update, send only the new content with `update_mode: "append"` and Hindsight will efficiently merge it with what it already has. + +```json +{ + "items": [ + { + "content": "New entry to add to the existing document.", + "document_id": "my-growing-doc", + "update_mode": "append" + } + ] +} +``` + ### entities A list of entities you want to guarantee are recognized, merged with any entities the LLM extracts automatically. Each entry has a `text` field (the entity name) and an optional `type` (e.g., `"PERSON"`, `"ORG"`, `"CONCEPT"` — defaults to `"CONCEPT"` if omitted). diff --git a/skills/hindsight-docs/references/developer/configuration.md b/skills/hindsight-docs/references/developer/configuration.md index 1289ebfe..ab20c319 100644 --- a/skills/hindsight-docs/references/developer/configuration.md +++ b/skills/hindsight-docs/references/developer/configuration.md @@ -1289,6 +1289,33 @@ Configuration fields are categorized for security: | Variable | Description | Default | |----------|-------------|---------| | `HINDSIGHT_API_ENABLE_BANK_CONFIG_API` | Enable per-bank config API | `true` | +| `HINDSIGHT_API_DEFAULT_BANK_TEMPLATE` | Bank template manifest (JSON) applied automatically to every newly-created bank. See below. | _(unset)_ | + +##### `HINDSIGHT_API_DEFAULT_BANK_TEMPLATE` + +Server-level default bank template. When set, the manifest is applied once +to every bank the server creates — triggered the first time a bank is +touched (via `PUT /v1/default/banks/{bank_id}`, `/import`, `/retain`, etc.). +The value is a JSON-encoded `BankTemplateManifest` with the same shape +accepted by `POST /v1/default/banks/{bank_id}/import` (see the `bank`, +`mental_models`, and `directives` sections in the Bank Templates API). + +Precedence: fields set by the template become per-bank overrides, so they +take precedence over the equivalent `HINDSIGHT_API_*` env-var defaults +(e.g. `HINDSIGHT_API_RETAIN_EXTRACTION_MODE`). Users can still override +individual fields later via `PATCH /v1/default/banks/{bank_id}/config`; +the template is **not** re-applied on subsequent accesses, so explicit +overrides are never clobbered. + +A malformed manifest (bad JSON, unknown version, schema errors) is logged +and ignored — bank creation still succeeds with plain defaults, so a +broken server-level setting cannot wedge all callers. + +Example (compact, single-line JSON as required by env vars): + +```bash +export HINDSIGHT_API_DEFAULT_BANK_TEMPLATE='{"version":"1","bank":{"reflect_mission":"Help support agents remember customer interactions.","retain_extraction_mode":"verbose","disposition_empathy":5},"directives":[{"name":"Be concise","content":"Always respond concisely.","priority":10}]}' +``` #### API Endpoints