From 179e99e89d270d8ba5977a5f3f1999d36820eadf Mon Sep 17 00:00:00 2001 From: RCLL Date: Sun, 23 Aug 2026 23:50:01 +0300 Subject: [PATCH] feat(rooms): per-agent rooms, halls and durability layers (ADR-145) Recovered from the 2026-06-27 snapshot import by classifying the base..snapshot delta at line granularity. Upstream base: d054b884 (2026-04-10). --- .../aa1_add_room_hall_to_memory_units.py | 69 +++++++ hindsight-api-slim/hindsight_api/api/http.py | 49 +++++ .../hindsight_api/engine/memory_engine.py | 26 +++ .../hindsight_api/engine/response_models.py | 3 + .../engine/retain/fact_extraction.py | 9 + .../engine/retain/fact_storage.py | 21 ++- .../engine/retain/orchestrator.py | 13 ++ .../engine/retain/room_hall_classifier.py | 176 ++++++++++++++++++ .../hindsight_api/engine/retain/types.py | 20 ++ .../engine/search/link_expansion_retrieval.py | 26 +-- .../hindsight_api/engine/search/retrieval.py | 76 +++++++- .../hindsight_api/engine/search/types.py | 9 + hindsight-api-slim/hindsight_api/models.py | 3 + 13 files changed, 475 insertions(+), 25 deletions(-) create mode 100644 hindsight-api-slim/hindsight_api/alembic/versions/aa1_add_room_hall_to_memory_units.py create mode 100644 hindsight-api-slim/hindsight_api/engine/retain/room_hall_classifier.py diff --git a/hindsight-api-slim/hindsight_api/alembic/versions/aa1_add_room_hall_to_memory_units.py b/hindsight-api-slim/hindsight_api/alembic/versions/aa1_add_room_hall_to_memory_units.py new file mode 100644 index 00000000..ed945d66 --- /dev/null +++ b/hindsight-api-slim/hindsight_api/alembic/versions/aa1_add_room_hall_to_memory_units.py @@ -0,0 +1,69 @@ +"""Add room and hall columns to memory_units for hierarchical filtering (ADR-145) + +Revision ID: aa1_room_hall +Revises: z1u2v3w4x5y6 +Create Date: 2026-04-11 + +Adds room (topic) and hall (knowledge type) columns to memory_units. +Room/Hall taxonomy enables pre-semantic filtering: the candidate set is narrowed +by topic and knowledge type before the vector search runs. +""" + +from collections.abc import Sequence + +from alembic import context, op + +revision: str = "aa1_room_hall" +down_revision: str | Sequence[str] | None = "z1u2v3w4x5y6" +branch_labels: str | Sequence[str] | None = None +depends_on: str | Sequence[str] | None = None + + +def _get_schema_prefix() -> str: + """Get schema prefix for table names (required for multi-tenant support).""" + schema = context.config.get_main_option("target_schema") + return f'"{schema}".' if schema else "" + + +def upgrade() -> None: + schema = _get_schema_prefix() + + # Room: topic classification (what the memory is about) + op.execute(f"ALTER TABLE {schema}memory_units ADD COLUMN IF NOT EXISTS room TEXT") + + # Hall: knowledge type classification (what kind of knowledge) + op.execute(f"ALTER TABLE {schema}memory_units ADD COLUMN IF NOT EXISTS hall TEXT") + + # Layer: memory tier L0-L3 (ADR-145). Model declares server_default 'L2'; + # this column was missing from the original migration while retrieval/models + # reference it, causing `column "layer" does not exist` on recall. + op.execute(f"ALTER TABLE {schema}memory_units ADD COLUMN IF NOT EXISTS layer TEXT DEFAULT 'L2'") + + # Indexes for filtering BEFORE semantic search + op.execute( + f"CREATE INDEX IF NOT EXISTS idx_memory_units_room " + f"ON {schema}memory_units (bank_id, room) WHERE room IS NOT NULL" + ) + op.execute( + f"CREATE INDEX IF NOT EXISTS idx_memory_units_hall " + f"ON {schema}memory_units (bank_id, hall) WHERE hall IS NOT NULL" + ) + op.execute( + f"CREATE INDEX IF NOT EXISTS idx_memory_units_room_hall " + f"ON {schema}memory_units (bank_id, room, hall) WHERE room IS NOT NULL AND hall IS NOT NULL" + ) + op.execute( + f"CREATE INDEX IF NOT EXISTS idx_memory_units_layer " + f"ON {schema}memory_units (bank_id, layer) WHERE layer IS NOT NULL" + ) + + +def downgrade() -> None: + schema = _get_schema_prefix() + op.execute(f"DROP INDEX IF EXISTS {schema}idx_memory_units_layer") + op.execute(f"ALTER TABLE {schema}memory_units DROP COLUMN IF EXISTS layer") + op.execute(f"DROP INDEX IF EXISTS {schema}idx_memory_units_room_hall") + op.execute(f"DROP INDEX IF EXISTS {schema}idx_memory_units_hall") + op.execute(f"DROP INDEX IF EXISTS {schema}idx_memory_units_room") + op.execute(f"ALTER TABLE {schema}memory_units DROP COLUMN IF EXISTS hall") + op.execute(f"ALTER TABLE {schema}memory_units DROP COLUMN IF EXISTS room") diff --git a/hindsight-api-slim/hindsight_api/api/http.py b/hindsight-api-slim/hindsight_api/api/http.py index c6b55b79..321c0ffc 100644 --- a/hindsight-api-slim/hindsight_api/api/http.py +++ b/hindsight-api-slim/hindsight_api/api/http.py @@ -171,6 +171,22 @@ class RecallRequest(BaseModel): description="Compound tag filter using boolean groups. Groups in the list are AND-ed. " "Each group is a leaf {tags, match} or compound {and: [...]}, {or: [...]}, {not: ...}.", ) + room: list[str] | None = Field( + default=None, + description="Filter by room (topic). If specified, only memories in these rooms are searched. " + "Applied BEFORE semantic search, so the vector query runs over a narrowed candidate set.", + ) + hall: list[str] | None = Field( + default=None, + description="Filter by hall (knowledge type). If specified, only these knowledge types are searched. " + "Values: fact, event, decision, preference, discovery, procedure, warning.", + ) + max_layer: Literal["L0", "L1", "L2", "L3"] = Field( + default="L3", + description="Maximum memory layer to search (ADR-145). Recall searches L0 through max_layer. " + "L0=Identity only, L1=Identity+Critical, L2=+Session, L3=+Deep (all, default). " + "Results are ordered with L0 first (highest priority).", + ) @field_validator("query") @classmethod @@ -223,6 +239,9 @@ class RecallResult(BaseModel): metadata: dict[str, str] | None = None # User-defined metadata chunk_id: str | None = None # Chunk this fact was extracted from tags: list[str] | None = None # Visibility scope tags + room: str | None = None # Topic classification (ADR-145 RCLL) + hall: str | None = None # Knowledge type classification (ADR-145 RCLL) + layer: str | None = None # Memory layer (ADR-145: L0-L3) source_fact_ids: list[str] | None = ( None # IDs of source facts (observation type only, when source_facts is enabled) ) @@ -469,6 +488,24 @@ class MemoryItem(BaseModel): "'replace' (default) deletes old data and reprocesses from scratch. " "'append' concatenates new content to the existing document text and reprocesses.", ) + room: str | None = Field( + default=None, + description="Topic classification for hierarchical filtering (ADR-145 RCLL). " + "Examples: auth, pipeline, schema, tax, hr, legal, compliance, infrastructure, ui, api, deployment, monitoring. " + "Use 'custom:{name}' for custom topics. Auto-classified by LLM if not provided.", + ) + hall: str | None = Field( + default=None, + description="Knowledge type classification (ADR-145 RCLL). " + "One of: fact, event, decision, preference, discovery, procedure, warning. " + "Auto-classified by LLM if not provided.", + ) + layer: Literal["L0", "L1", "L2", "L3"] = Field( + default="L2", + description="Memory layer (ADR-145). L0=Identity (~50 tokens, always loaded), " + "L1=Critical Facts (~120 tokens, per-space), L2=Session Context (default), " + "L3=Deep Memory (full search).", + ) @field_validator("timestamp", mode="before") @classmethod @@ -2959,6 +2996,9 @@ def _register_routes(app: FastAPI): tags=request.tags, tags_match=request.tags_match, tag_groups=request.tag_groups, + room=request.room, + hall=request.hall, + max_layer=request.max_layer, ) # Convert core MemoryFact objects to API RecallResult objects (excluding internal metrics) @@ -2976,6 +3016,9 @@ def _register_routes(app: FastAPI): metadata=fact.metadata, chunk_id=fact.chunk_id, tags=fact.tags, + room=getattr(fact, 'room', None), + hall=getattr(fact, 'hall', None), + layer=getattr(fact, 'layer', None), source_fact_ids=fact.source_fact_ids, ) @@ -5333,6 +5376,12 @@ def _register_routes(app: FastAPI): content_dict["observation_scopes"] = item.observation_scopes if item.update_mode is not None: content_dict["update_mode"] = item.update_mode + if item.room: + content_dict["room"] = item.room + if item.hall: + content_dict["hall"] = item.hall + if item.layer and item.layer != "L2": + content_dict["layer"] = item.layer strategy_groups[effective].append(content_dict) if request.async_: diff --git a/hindsight-api-slim/hindsight_api/engine/memory_engine.py b/hindsight-api-slim/hindsight_api/engine/memory_engine.py index 5bea103d..fe37e27d 100644 --- a/hindsight-api-slim/hindsight_api/engine/memory_engine.py +++ b/hindsight-api-slim/hindsight_api/engine/memory_engine.py @@ -2427,6 +2427,9 @@ class MemoryEngine(MemoryEngineInterface): tags: list[str] | None = None, tags_match: TagsMatch = "any", tag_groups: list[TagGroup] | None = None, + room: list[str] | None = None, + hall: list[str] | None = None, + max_layer: str = "L3", _connection_budget: int | None = None, _quiet: bool = False, ) -> RecallResultModel: @@ -2571,6 +2574,9 @@ class MemoryEngine(MemoryEngineInterface): tags=tags, tags_match=tags_match, tag_groups=tag_groups, + room=room, + hall=hall, + max_layer=max_layer, connection_budget=_connection_budget, quiet=_quiet, include_source_facts=include_source_facts, @@ -2699,6 +2705,9 @@ class MemoryEngine(MemoryEngineInterface): tags: list[str] | None = None, tags_match: TagsMatch = "any", tag_groups: list[TagGroup] | None = None, + room: list[str] | None = None, + hall: list[str] | None = None, + max_layer: str = "L3", connection_budget: int | None = None, quiet: bool = False, include_source_facts: bool = False, @@ -2819,6 +2828,9 @@ class MemoryEngine(MemoryEngineInterface): tags=tags, tags_match=tags_match, tag_groups=tag_groups, + room=room, + hall=hall, + max_layer=max_layer, ) parallel_duration = time.time() - parallel_start finally: @@ -2866,6 +2878,17 @@ class MemoryEngine(MemoryEngineInterface): if not temporal_results: temporal_results = None + # ADR-145: Python-side room/hall filtering for graph results + # (semantic/BM25/temporal are already SQL-filtered) + if room or hall: + def _room_hall_match(r): + if room and getattr(r, 'room', None) not in room: + return False + if hall and getattr(r, 'hall', None) not in hall: + return False + return True + graph_results = [r for r in graph_results if _room_hall_match(r)] + # Sort combined results by score (descending) so higher-scored results # get better ranks in the trace, regardless of fact type semantic_results.sort(key=lambda r: r.similarity if hasattr(r, "similarity") else 0, reverse=True) @@ -3419,6 +3442,9 @@ class MemoryEngine(MemoryEngineInterface): metadata=result_dict.get("metadata"), chunk_id=result_dict.get("chunk_id"), tags=result_dict.get("tags"), + room=result_dict.get("room"), + hall=result_dict.get("hall"), + layer=result_dict.get("layer"), source_fact_ids=source_fact_ids_by_obs.get(result_id) if include_source_facts else None, ) ) diff --git a/hindsight-api-slim/hindsight_api/engine/response_models.py b/hindsight-api-slim/hindsight_api/engine/response_models.py index bfd7a69c..c22d3602 100644 --- a/hindsight-api-slim/hindsight_api/engine/response_models.py +++ b/hindsight-api-slim/hindsight_api/engine/response_models.py @@ -175,6 +175,9 @@ class MemoryFact(BaseModel): None, description="ID of the chunk this fact was extracted from (format: bank_id_document_id_chunk_index)" ) tags: list[str] | None = Field(None, description="Visibility scope tags associated with this fact") + room: str | None = Field(None, description="Topic classification (ADR-145 RCLL)") + hall: str | None = Field(None, description="Knowledge type classification (ADR-145 RCLL)") + layer: str | None = Field(None, description="Memory layer (ADR-145: L0, L1, L2, L3)") source_fact_ids: list[str] | None = Field( None, description="IDs of source facts this observation was derived from (observation type only, when source_facts is enabled)", diff --git a/hindsight-api-slim/hindsight_api/engine/retain/fact_extraction.py b/hindsight-api-slim/hindsight_api/engine/retain/fact_extraction.py index bba01ab7..f3b80884 100644 --- a/hindsight-api-slim/hindsight_api/engine/retain/fact_extraction.py +++ b/hindsight-api-slim/hindsight_api/engine/retain/fact_extraction.py @@ -1990,6 +1990,9 @@ async def extract_facts_from_contents_batch_api( metadata=content.metadata, tags=content.tags, observation_scopes=content.observation_scopes, + room=getattr(content, 'room', None), + hall=getattr(content, 'hall', None), + layer=getattr(content, 'layer', 'L2'), ) extracted_facts.append(extracted_fact) @@ -2044,6 +2047,9 @@ def _extract_facts_chunks( metadata=content.metadata, tags=content.tags, observation_scopes=content.observation_scopes, + room=getattr(content, 'room', None), + hall=getattr(content, 'hall', None), + layer=getattr(content, 'layer', 'L2'), ) ) global_chunk_idx += 1 @@ -2183,6 +2189,9 @@ async def extract_facts_from_contents( metadata=content.metadata, tags=content.tags, observation_scopes=content.observation_scopes, + room=getattr(content, 'room', None), + hall=getattr(content, 'hall', None), + layer=getattr(content, 'layer', 'L2'), ) extracted_facts.append(extracted_fact) diff --git a/hindsight-api-slim/hindsight_api/engine/retain/fact_storage.py b/hindsight-api-slim/hindsight_api/engine/retain/fact_storage.py index 9002175b..85f80c07 100644 --- a/hindsight-api-slim/hindsight_api/engine/retain/fact_storage.py +++ b/hindsight-api-slim/hindsight_api/engine/retain/fact_storage.py @@ -105,6 +105,9 @@ async def insert_facts_batch( except (ValueError, AttributeError): pass text_signals_list.append(" ".join(signal_parts) if signal_parts else None) + rooms_list.append(getattr(fact, 'room', None)) + halls_list.append(getattr(fact, 'hall', None)) + layers_list.append(getattr(fact, 'layer', 'L2')) # Batch insert all facts # Note: tags are passed as JSON strings and converted back to varchar[] via jsonb_array_elements_text + array_agg @@ -120,11 +123,11 @@ async def insert_facts_batch( $8::text[], $9::text[], $10::jsonb[], $11::text[], $12::text[], $13::jsonb[], $14::jsonb[], $15::text[] ) AS t(text, embedding, event_date, occurred_start, occurred_end, mentioned_at, context, fact_type, metadata, chunk_id, document_id, tags_json, - observation_scopes_json, text_signals) + observation_scopes_json, text_signals, room, hall, layer) ) INSERT INTO {fq_table("memory_units")} (bank_id, text, embedding, event_date, occurred_start, occurred_end, mentioned_at, context, fact_type, metadata, chunk_id, document_id, tags, - observation_scopes, text_signals, search_vector) + observation_scopes, text_signals, search_vector, room, hall, layer) SELECT $1, text, embedding, event_date, occurred_start, occurred_end, mentioned_at, @@ -138,7 +141,10 @@ async def insert_facts_batch( tokenize( COALESCE(text, '') || ' ' || COALESCE(context, '') || ' ' || COALESCE(text_signals, ''), 'llmlingua2' - )::bm25_catalog.bm25vector + )::bm25_catalog.bm25vector, + room, + hall, + COALESCE(layer, 'L2') FROM input_data RETURNING id """ @@ -152,11 +158,11 @@ async def insert_facts_batch( $8::text[], $9::text[], $10::jsonb[], $11::text[], $12::text[], $13::jsonb[], $14::jsonb[], $15::text[] ) AS t(text, embedding, event_date, occurred_start, occurred_end, mentioned_at, context, fact_type, metadata, chunk_id, document_id, tags_json, - observation_scopes_json, text_signals) + observation_scopes_json, text_signals, room, hall, layer) ) INSERT INTO {fq_table("memory_units")} (bank_id, text, embedding, event_date, occurred_start, occurred_end, mentioned_at, context, fact_type, metadata, chunk_id, document_id, tags, - observation_scopes, text_signals) + observation_scopes, text_signals, room, hall, layer) SELECT $1, text, embedding, event_date, occurred_start, occurred_end, mentioned_at, @@ -166,7 +172,10 @@ async def insert_facts_batch( '{{}}'::varchar[] ), observation_scopes_json, - text_signals + text_signals, + room, + hall, + COALESCE(layer, 'L2') FROM input_data RETURNING id """ diff --git a/hindsight-api-slim/hindsight_api/engine/retain/orchestrator.py b/hindsight-api-slim/hindsight_api/engine/retain/orchestrator.py index 46bf4ebd..96b69ffb 100644 --- a/hindsight-api-slim/hindsight_api/engine/retain/orchestrator.py +++ b/hindsight-api-slim/hindsight_api/engine/retain/orchestrator.py @@ -390,6 +390,10 @@ async def _extract_and_embed( processed_facts = [ProcessedFact.from_extracted_fact(ef, emb) for ef, emb in zip(extracted_facts, embeddings)] + # ADR-145: Auto-classify room/hall for facts that don't have them set + from .room_hall_classifier import classify_facts_batch + classify_facts_batch(processed_facts) + return extracted_facts, processed_facts, chunks, usage @@ -863,6 +867,9 @@ async def _streaming_retain_batch( entities=source.entities, tags=source.tags, observation_scopes=source.observation_scopes, + room=getattr(source, 'room', None), + hall=getattr(source, 'hall', None), + layer=getattr(source, 'layer', 'L2'), ) extracted, processed, chunk_meta, usage = await _extract_and_embed( [content], @@ -1480,6 +1487,9 @@ def _build_contents(contents_dicts: list[RetainContentDict], document_tags: list entities=item.get("entities", []), tags=merged_tags, observation_scopes=item.get("observation_scopes"), + room=item.get("room"), + hall=item.get("hall"), + layer=item.get("layer", "L2"), ) contents.append(content) return contents @@ -1536,6 +1546,9 @@ def _build_delta_contents( entities=template_content.entities, tags=template_content.tags, observation_scopes=template_content.observation_scopes, + room=getattr(template_content, 'room', None), + hall=getattr(template_content, 'hall', None), + layer=getattr(template_content, 'layer', 'L2'), ) delta_contents.append(delta_content) delta_chunk_map[len(delta_contents) - 1] = original_chunk_idx diff --git a/hindsight-api-slim/hindsight_api/engine/retain/room_hall_classifier.py b/hindsight-api-slim/hindsight_api/engine/retain/room_hall_classifier.py new file mode 100644 index 00000000..a824fef0 --- /dev/null +++ b/hindsight-api-slim/hindsight_api/engine/retain/room_hall_classifier.py @@ -0,0 +1,176 @@ +""" +Room/Hall auto-classification for ADR-145 RCLL. + +Classifies memories into room (topic) and hall (knowledge type) using +keyword heuristics. Falls back to "general" room and "fact" hall when +no clear match is found. + +This avoids an additional LLM call per fact (~50 tokens) while providing +reasonable classification accuracy. LLM-based classification can be added +as an enhancement later. +""" + +import logging +import re + +logger = logging.getLogger(__name__) + +# ── Hall definitions (knowledge type) ── +# Order matters: first match wins +HALL_PATTERNS: list[tuple[str, list[str]]] = [ + ("warning", [ + r"\bdon'?t\b", r"\bnever\b", r"\bavoid\b", r"\bdanger", r"\brisk", + r"\bcaution", r"\bwarning", r"\bcareful", r"\bdo not\b", r"\bforbid", + r"\bprohibit", r"\billegal", r"\bpenalt", r"\bfine\b", + ]), + ("decision", [ + r"\bdecid", r"\bchose\b", r"\bchosen\b", r"\bapproved?\b", r"\brejected?\b", + r"\bagreed\b", r"\bselected?\b", r"\bpicked\b", r"\bwent with\b", + r"\bswitched? to\b", r"\bmigrat", r"\badopted?\b", r"\bresolved?\b", + ]), + ("procedure", [ + r"\bstep\s*\d", r"\bfirst\b.*\bthen\b", r"\bprocess\b", r"\bworkflow\b", + r"\bhow to\b", r"\bprocedure\b", r"\binstructions?\b", r"\brecipe\b", + r"\brun\b.*\bcommand\b", r"\bexecute\b", r"\bsetup\b", r"\binstall\b", + ]), + ("event", [ + r"\bhappened\b", r"\boccurred\b", r"\bmeeting\b", r"\bcall\b", + r"\bincident\b", r"\boutage\b", r"\breleased?\b", r"\blaunched?\b", + r"\bdeployed?\b", r"\bstarted?\b", r"\bfinished?\b", r"\bcompleted?\b", + r"\bon \d{4}-\d{2}-\d{2}\b", r"\byesterday\b", r"\blast week\b", + ]), + ("preference", [ + r"\bprefer", r"\blike[sd]?\b", r"\bfavorite\b", r"\bwant[sed]?\b", + r"\brather\b", r"\bstyle\b", r"\btaste\b", r"\bchoice\b", + ]), + ("discovery", [ + r"\bfound\b", r"\bdiscover", r"\blearned\b", r"\brealized?\b", + r"\bturns out\b", r"\bnoticed\b", r"\binsight\b", r"\bresearch", + r"\banalysis\b", r"\bbenchmark", + ]), + # Default: "fact" — captured by no-match fallback +] + +# ── Room definitions (topic) ── +ROOM_PATTERNS: list[tuple[str, list[str]]] = [ + ("auth", [ + r"\bauth", r"\blogin", r"\bjwt\b", r"\btoken", r"\bsession", + r"\bpassword", r"\bcredential", r"\boauth", r"\bsso\b", r"\bsign[- ]?in\b", + ]), + ("pipeline", [ + r"\bpipeline", r"\bci/?cd\b", r"\bbuild\b", r"\bdeploy", r"\bgithub action", + r"\bjenkins", r"\bcircleci", r"\bworkflow\b.*\bautomat", + ]), + ("infrastructure", [ + r"\bserver", r"\bnginx\b", r"\bpm2\b", r"\bdocker", r"\bk8s\b", + r"\bkubernet", r"\binfra", r"\baws\b", r"\bgcp\b", r"\bazure\b", + r"\bvps\b", r"\bload balanc", r"\bdns\b", r"\bssl\b", r"\bcert\b", + ]), + ("deployment", [ + r"\bdeploy", r"\brelease\b", r"\brollback", r"\bhotfix", + r"\bstaging\b", r"\bproduction\b", r"\bblue[- ]?green", + ]), + ("schema", [ + r"\bschema", r"\bmigrat", r"\bcolumn\b", r"\btable\b", r"\bindex\b", + r"\bpostgres", r"\bdatabase\b", r"\bsql\b", r"\bquery\b", + r"\balembic", r"\bforeign key", + ]), + ("api", [ + r"\bapi\b", r"\bendpoint", r"\broute\b", r"\brest\b", r"\bgraphql", + r"\bhttp\b", r"\brequest\b", r"\bresponse\b", r"\bwebhook", + ]), + ("ui", [ + r"\bui\b", r"\bfrontend", r"\breact\b", r"\bcomponent", r"\bcss\b", + r"\bbutton\b", r"\bmodal\b", r"\bpanel\b", r"\blayout\b", + r"\bdesign\b", r"\bux\b", r"\bstyle\b", + ]), + ("tax", [ + r"\btax", r"\bfiling\b", r"\bird\b", r"\bmpf\b", r"\bprofit", + r"\bdeduction", r"\bexempt", r"\btaxable\b", r"\breturn\b.*\btax", + ]), + ("hr", [ + r"\bhr\b", r"\bhuman resource", r"\bsalary", r"\bleave\b", + r"\bemployee", r"\bhiring", r"\brecruitment", r"\bonboarding", + ]), + ("legal", [ + r"\blegal", r"\bcontract", r"\bclause\b", r"\blawyer", + r"\blitigation", r"\bcourt\b", r"\bregulat", r"\blicense\b", + ]), + ("compliance", [ + r"\bcompliance", r"\baudit\b", r"\bregulat", r"\bkyc\b", + r"\baml\b", r"\bgdpr\b", r"\bprivacy\b", r"\bdata protect", + ]), + ("monitoring", [ + r"\bmonitor", r"\balert", r"\bmetric", r"\blog[sg]?\b", + r"\bgrafana\b", r"\bprometheus\b", r"\bobservab", r"\btrac[ei]", + ]), + ("agent", [ + r"\bagent\b", r"\bllm\b", r"\bai\b", r"\bmemory\b.*\bsystem\b", + r"\bhindsight\b", r"\bprompt\b", r"\btool\b.*\bcall\b", + r"\bpes\b", r"\bspell\b", r"\borchestrat", + ]), + # Default: "general" — captured by no-match fallback +] + + +def _match_patterns(text: str, patterns: list[tuple[str, list[str]]]) -> str | None: + """Match text against pattern list, return first matching category or None.""" + text_lower = text.lower() + for category, regexes in patterns: + for pattern in regexes: + if re.search(pattern, text_lower): + return category + return None + + +def classify_room_hall( + fact_text: str, + context: str | None = None, + existing_room: str | None = None, + existing_hall: str | None = None, +) -> tuple[str, str]: + """ + Classify a fact into room (topic) and hall (knowledge type). + + If existing values are provided, they are kept as-is. + Otherwise, keyword heuristics determine the classification. + + Args: + fact_text: The fact text to classify + context: Optional context string for additional signals + existing_room: Pre-assigned room (kept if not None) + existing_hall: Pre-assigned hall (kept if not None) + + Returns: + Tuple of (room, hall) + """ + combined = f"{fact_text} {context or ''}" + + room = existing_room + if room is None: + room = _match_patterns(combined, ROOM_PATTERNS) or "general" + + hall = existing_hall + if hall is None: + hall = _match_patterns(combined, HALL_PATTERNS) or "fact" + + return room, hall + + +def classify_facts_batch(facts: list) -> None: + """ + Classify room/hall for a batch of ProcessedFact or ExtractedFact objects. + + Modifies facts in-place. Only sets room/hall if not already set. + """ + for fact in facts: + fact_text = getattr(fact, 'fact_text', '') or getattr(fact, 'text', '') or '' + context = getattr(fact, 'context', None) + room, hall = classify_room_hall( + fact_text, + context, + existing_room=getattr(fact, 'room', None), + existing_hall=getattr(fact, 'hall', None), + ) + fact.room = room + fact.hall = hall diff --git a/hindsight-api-slim/hindsight_api/engine/retain/types.py b/hindsight-api-slim/hindsight_api/engine/retain/types.py index f7a4a428..4888c90d 100644 --- a/hindsight-api-slim/hindsight_api/engine/retain/types.py +++ b/hindsight-api-slim/hindsight_api/engine/retain/types.py @@ -28,6 +28,9 @@ class RetainContentDict(TypedDict, total=False): update_mode: How to handle existing documents with the same document_id (optional). "replace" (default) deletes old data and reprocesses. "append" concatenates new content to the existing document and reprocesses. + room: Topic classification for hierarchical filtering (ADR-145) + hall: Knowledge type classification (ADR-145) + layer: Memory layer L0-L3 (ADR-145) """ content: str # Required @@ -41,6 +44,9 @@ class RetainContentDict(TypedDict, total=False): Literal["per_tag", "combined", "all_combinations"] | list[list[str]] ) # Observation scopes for consolidation update_mode: Literal["replace", "append"] + room: str # Topic classification (ADR-145 RCLL) + hall: str # Knowledge type classification (ADR-145 RCLL) + layer: str # Memory layer: L0, L1, L2 (default), L3 (ADR-145) @dataclass @@ -60,6 +66,9 @@ class RetainContent: observation_scopes: Literal["per_tag", "combined", "all_combinations"] | list[list[str]] | None = ( None # Observation scopes ) + room: str | None = None # Topic classification (ADR-145 RCLL) + hall: str | None = None # Knowledge type classification (ADR-145 RCLL) + layer: str = "L2" # Memory layer: L0, L1, L2 (default), L3 (ADR-145) @dataclass @@ -128,6 +137,9 @@ class ExtractedFact: observation_scopes: Literal["per_tag", "combined", "all_combinations"] | list[list[str]] | None = ( None # Observation scopes ) + room: str | None = None # Topic classification (ADR-145 RCLL) + hall: str | None = None # Knowledge type classification (ADR-145 RCLL) + layer: str = "L2" # Memory layer: L0, L1, L2 (default), L3 (ADR-145) @dataclass @@ -179,6 +191,11 @@ class ProcessedFact: # Observation scopes for consolidation observation_scopes: Literal["per_tag", "combined", "all_combinations"] | list[list[str]] | None = None + # ADR-145 RCLL: hierarchical classification + room: str | None = None # Topic classification + hall: str | None = None # Knowledge type classification + layer: str = "L2" # Memory layer: L0, L1, L2 (default), L3 + @property def is_duplicate(self) -> bool: """Check if this fact was marked as a duplicate.""" @@ -222,6 +239,9 @@ class ProcessedFact: content_index=extracted_fact.content_index, tags=extracted_fact.tags, observation_scopes=extracted_fact.observation_scopes, + room=extracted_fact.room, + hall=extracted_fact.hall, + layer=extracted_fact.layer, ) diff --git a/hindsight-api-slim/hindsight_api/engine/search/link_expansion_retrieval.py b/hindsight-api-slim/hindsight_api/engine/search/link_expansion_retrieval.py index 6a5f99f5..39f82689 100644 --- a/hindsight-api-slim/hindsight_api/engine/search/link_expansion_retrieval.py +++ b/hindsight-api-slim/hindsight_api/engine/search/link_expansion_retrieval.py @@ -64,7 +64,7 @@ async def _find_semantic_seeds( rows = await conn.fetch( f""" SELECT id, text, context, event_date, occurred_start, occurred_end, - mentioned_at, fact_type, document_id, chunk_id, tags, proof_count, + mentioned_at, fact_type, document_id, chunk_id, tags, proof_count, room, hall, layer, 1 - (embedding <=> $1::vector) AS similarity FROM {fq_table("memory_units")} WHERE bank_id = $2 @@ -289,7 +289,7 @@ class LinkExpansionRetriever(GraphRetriever): entity_expanded AS ( SELECT mu.id, mu.text, mu.context, mu.event_date, mu.occurred_start, mu.occurred_end, mu.mentioned_at, - mu.fact_type, mu.document_id, mu.chunk_id, mu.tags, mu.proof_count, + mu.fact_type, mu.document_id, mu.chunk_id, mu.tags, mu.proof_count, mu.room, mu.hall, mu.layer, COUNT(DISTINCT se.entity_id)::float AS score, 'entity'::text AS source FROM seed_entities se @@ -316,14 +316,14 @@ class LinkExpansionRetriever(GraphRetriever): SELECT id, text, context, event_date, occurred_start, occurred_end, mentioned_at, - fact_type, document_id, chunk_id, tags, proof_count, + fact_type, document_id, chunk_id, tags, proof_count, room, hall, layer, MAX(weight) AS score, 'semantic'::text AS source FROM ( SELECT mu.id, mu.text, mu.context, mu.event_date, mu.occurred_start, mu.occurred_end, mu.mentioned_at, - mu.fact_type, mu.document_id, mu.chunk_id, mu.tags, mu.proof_count, + mu.fact_type, mu.document_id, mu.chunk_id, mu.tags, mu.proof_count, mu.room, mu.hall, mu.layer, ml.weight FROM {ml} ml JOIN {mu} mu ON mu.id = ml.to_unit_id @@ -335,7 +335,7 @@ class LinkExpansionRetriever(GraphRetriever): SELECT mu.id, mu.text, mu.context, mu.event_date, mu.occurred_start, mu.occurred_end, mu.mentioned_at, - mu.fact_type, mu.document_id, mu.chunk_id, mu.tags, mu.proof_count, + mu.fact_type, mu.document_id, mu.chunk_id, mu.tags, mu.proof_count, mu.room, mu.hall, mu.layer, ml.weight FROM {ml} ml JOIN {mu} mu ON mu.id = ml.from_unit_id @@ -346,7 +346,7 @@ class LinkExpansionRetriever(GraphRetriever): ) sem_raw GROUP BY id, text, context, event_date, occurred_start, occurred_end, mentioned_at, - fact_type, document_id, chunk_id, tags, proof_count + fact_type, document_id, chunk_id, tags, proof_count, room, hall, layer ORDER BY score DESC LIMIT $3 ), @@ -357,7 +357,7 @@ class LinkExpansionRetriever(GraphRetriever): SELECT DISTINCT ON (mu.id) mu.id, mu.text, mu.context, mu.event_date, mu.occurred_start, mu.occurred_end, mu.mentioned_at, - mu.fact_type, mu.document_id, mu.chunk_id, mu.tags, mu.proof_count, + mu.fact_type, mu.document_id, mu.chunk_id, mu.tags, mu.proof_count, mu.room, mu.hall, mu.layer, ml.weight AS score, 'causal'::text AS source FROM {ml} ml @@ -480,7 +480,7 @@ class LinkExpansionRetriever(GraphRetriever): SELECT mu.id, mu.text, mu.context, mu.event_date, mu.occurred_start, mu.occurred_end, mu.mentioned_at, - mu.fact_type, mu.document_id, mu.chunk_id, mu.tags, mu.proof_count, + mu.fact_type, mu.document_id, mu.chunk_id, mu.tags, mu.proof_count, mu.room, mu.hall, mu.layer, (SELECT COUNT(DISTINCT s) FROM unnest(mu.source_memory_ids) s WHERE s = ANY(ca.source_ids))::float AS score FROM {fq_table("memory_units")} mu, connected_array ca WHERE mu.fact_type = 'observation' @@ -504,13 +504,13 @@ class LinkExpansionRetriever(GraphRetriever): SELECT id, text, context, event_date, occurred_start, occurred_end, mentioned_at, - fact_type, document_id, chunk_id, tags, proof_count, + fact_type, document_id, chunk_id, tags, proof_count, room, hall, layer, MAX(weight) AS score, 'semantic'::text AS source FROM ( SELECT mu.id, mu.text, mu.context, mu.event_date, mu.occurred_start, mu.occurred_end, mu.mentioned_at, mu.fact_type, mu.document_id, - mu.chunk_id, mu.tags, mu.proof_count, ml.weight + mu.chunk_id, mu.tags, mu.proof_count, mu.room, mu.hall, mu.layer, ml.weight FROM {ml} ml JOIN {mu} mu ON mu.id = ml.to_unit_id WHERE ml.from_unit_id = ANY($1::uuid[]) AND ml.link_type = 'semantic' AND mu.fact_type = 'observation' @@ -518,21 +518,21 @@ class LinkExpansionRetriever(GraphRetriever): UNION ALL SELECT mu.id, mu.text, mu.context, mu.event_date, mu.occurred_start, mu.occurred_end, mu.mentioned_at, mu.fact_type, mu.document_id, - mu.chunk_id, mu.tags, mu.proof_count, ml.weight + mu.chunk_id, mu.tags, mu.proof_count, mu.room, mu.hall, mu.layer, ml.weight FROM {ml} ml JOIN {mu} mu ON mu.id = ml.from_unit_id WHERE ml.to_unit_id = ANY($1::uuid[]) AND ml.link_type = 'semantic' AND mu.fact_type = 'observation' AND mu.id != ALL($1::uuid[]) ) sem_raw GROUP BY id, text, context, event_date, occurred_start, occurred_end, - mentioned_at, fact_type, document_id, chunk_id, tags, proof_count + mentioned_at, fact_type, document_id, chunk_id, tags, proof_count, room, hall, layer ORDER BY score DESC LIMIT $2 ), causal_expanded AS ( SELECT DISTINCT ON (mu.id) mu.id, mu.text, mu.context, mu.event_date, mu.occurred_start, mu.occurred_end, mu.mentioned_at, mu.fact_type, mu.document_id, - mu.chunk_id, mu.tags, mu.proof_count, ml.weight AS score, 'causal'::text AS source + mu.chunk_id, mu.tags, mu.proof_count, mu.room, mu.hall, mu.layer, ml.weight AS score, 'causal'::text AS source FROM {ml} ml JOIN {mu} mu ON ml.to_unit_id = mu.id WHERE ml.from_unit_id = ANY($1::uuid[]) AND ml.link_type IN ('causes', 'caused_by', 'enables', 'prevents') diff --git a/hindsight-api-slim/hindsight_api/engine/search/retrieval.py b/hindsight-api-slim/hindsight_api/engine/search/retrieval.py index f047bc4d..7983273d 100644 --- a/hindsight-api-slim/hindsight_api/engine/search/retrieval.py +++ b/hindsight-api-slim/hindsight_api/engine/search/retrieval.py @@ -98,6 +98,9 @@ async def retrieve_semantic_bm25_combined( tags: list[str] | None = None, tags_match: TagsMatch = "any", tag_groups: list[TagGroup] | None = None, + room: list[str] | None = None, + hall: list[str] | None = None, + max_layer: str = "L3", ) -> dict[str, tuple[list[RetrievalResult], list[RetrievalResult]]]: """ Combined semantic + BM25 retrieval for multiple fact types in a single query. @@ -141,7 +144,7 @@ async def retrieve_semantic_bm25_combined( cols = ( "id, text, context, event_date, occurred_start, occurred_end, mentioned_at, " - "fact_type, document_id, chunk_id, tags, metadata, proof_count" + "fact_type, document_id, chunk_id, tags, metadata, proof_count, room, hall, layer" ) table = fq_table("memory_units") @@ -161,7 +164,31 @@ async def retrieve_semantic_bm25_combined( # tag_groups params start immediately after the tags param slot tag_groups_param_start = tags_param_idx + (1 if tags else 0) - groups_clause, groups_params, _ = build_tag_groups_where_clause(tag_groups, tag_groups_param_start) + groups_clause, groups_params, next_param = build_tag_groups_where_clause(tag_groups, tag_groups_param_start) + + # ADR-145: Room/Hall filtering — applied BEFORE semantic search for accuracy boost + room_clause = "" + room_params = [] + if room: + room_clause = f"AND room = ANY(${next_param}::text[])" + room_params = [room] + next_param += 1 + hall_clause = "" + hall_params = [] + if hall: + hall_clause = f"AND hall = ANY(${next_param}::text[])" + hall_params = [hall] + next_param += 1 + + # ADR-145: Layer filtering — cascade from L0 to max_layer + layer_order = {"L0": 0, "L1": 1, "L2": 2, "L3": 3} + layer_clause = "" + layer_params = [] + if max_layer and max_layer != "L3": + allowed_layers = [l for l, v in layer_order.items() if v <= layer_order.get(max_layer, 3)] + layer_clause = f"AND COALESCE(layer, 'L2') = ANY(${next_param}::text[])" + layer_params = [allowed_layers] + next_param += 1 # --- Semantic UNION ALL arms (one per fact_type) --- # Each arm has its own ORDER BY embedding <=> $1 LIMIT {hnsw_fetch}, which @@ -266,6 +293,9 @@ async def retrieve_temporal_combined( tags: list[str] | None = None, tags_match: TagsMatch = "any", tag_groups: list[TagGroup] | None = None, + room: list[str] | None = None, + hall: list[str] | None = None, + max_layer: str = "L3", ) -> dict[str, list[RetrievalResult]]: """ Temporal retrieval for multiple fact types in a single query. @@ -298,7 +328,32 @@ async def retrieve_temporal_combined( # Entry point query: fixed params are $1-$6, tags at $7 tags_clause = build_tags_where_clause_simple(tags, 7, match=tags_match) tag_groups_param_start = 7 + (1 if tags else 0) - groups_clause, groups_params, _ = build_tag_groups_where_clause(tag_groups, tag_groups_param_start) + groups_clause, groups_params, next_param = build_tag_groups_where_clause(tag_groups, tag_groups_param_start) + + # ADR-145: Room/Hall filtering + room_clause = "" + room_params_list = [] + if room: + room_clause = f"AND room = ANY(${next_param}::text[])" + room_params_list = [room] + next_param += 1 + hall_clause = "" + hall_params_list = [] + if hall: + hall_clause = f"AND hall = ANY(${next_param}::text[])" + hall_params_list = [hall] + next_param += 1 + + # ADR-145: Layer filtering — cascade from L0 to max_layer + layer_order = {"L0": 0, "L1": 1, "L2": 2, "L3": 3} + layer_clause = "" + layer_params_list = [] + if max_layer and max_layer != "L3": + allowed_layers = [l for l, v in layer_order.items() if v <= layer_order.get(max_layer, 3)] + layer_clause = f"AND COALESCE(layer, 'L2') = ANY(${next_param}::text[])" + layer_params_list = [allowed_layers] + next_param += 1 + params: list = [query_emb_str, bank_id, fact_types, start_date, end_date, semantic_threshold] if tags: params.append(tags) @@ -336,7 +391,7 @@ async def retrieve_temporal_combined( {groups_clause} ), sim_ranked AS ( - SELECT mu.id, mu.text, mu.context, mu.event_date, mu.occurred_start, mu.occurred_end, mu.mentioned_at, mu.fact_type, mu.proof_count, mu.document_id, mu.chunk_id, mu.tags, mu.metadata, + SELECT mu.id, mu.text, mu.context, mu.event_date, mu.occurred_start, mu.occurred_end, mu.mentioned_at, mu.fact_type, mu.proof_count, mu.document_id, mu.chunk_id, mu.tags, mu.metadata, mu.room, mu.hall, mu.layer, 1 - (mu.embedding <=> $1::vector) AS similarity, ROW_NUMBER() OVER (PARTITION BY mu.fact_type ORDER BY mu.embedding <=> $1::vector) AS sim_rn FROM date_ranked dr @@ -344,7 +399,7 @@ async def retrieve_temporal_combined( WHERE dr.rn <= 50 AND (1 - (mu.embedding <=> $1::vector)) >= $6 ) - SELECT id, text, context, event_date, occurred_start, occurred_end, mentioned_at, fact_type, proof_count, document_id, chunk_id, tags, metadata, similarity + SELECT id, text, context, event_date, occurred_start, occurred_end, mentioned_at, fact_type, proof_count, document_id, chunk_id, tags, metadata, room, hall, layer, similarity FROM sim_ranked WHERE sim_rn <= 10 """, @@ -442,7 +497,7 @@ async def retrieve_temporal_combined( # bank_id on memory_units lets the planner use idx_memory_units_bank_fact_type. neighbors = await conn.fetch( f""" - SELECT src.from_unit_id, mu.id, mu.text, mu.context, mu.event_date, mu.occurred_start, mu.occurred_end, mu.mentioned_at, mu.fact_type, mu.document_id, mu.chunk_id, mu.tags, mu.metadata, + SELECT src.from_unit_id, mu.id, mu.text, mu.context, mu.event_date, mu.occurred_start, mu.occurred_end, mu.mentioned_at, mu.fact_type, mu.document_id, mu.chunk_id, mu.tags, mu.metadata, mu.room, mu.hall, l.weight, l.link_type, 1 - (mu.embedding <=> $1::vector) AS similarity FROM unnest($2::uuid[]) AS src(from_unit_id) @@ -536,6 +591,9 @@ async def retrieve_all_fact_types_parallel( tags: list[str] | None = None, tags_match: TagsMatch = "any", tag_groups: list[TagGroup] | None = None, + room: list[str] | None = None, + hall: list[str] | None = None, + max_layer: str = "L3", ) -> MultiFactTypeRetrievalResult: """ Optimized retrieval for multiple fact types using batched queries. @@ -594,6 +652,9 @@ async def retrieve_all_fact_types_parallel( tags=tags, tags_match=tags_match, tag_groups=tag_groups, + room=room, + hall=hall, + max_layer=max_layer, ) semantic_bm25_time = time.time() - semantic_bm25_start @@ -613,6 +674,9 @@ async def retrieve_all_fact_types_parallel( tags=tags, tags_match=tags_match, tag_groups=tag_groups, + room=room, + hall=hall, + max_layer=max_layer, ) temporal_time = time.time() - temporal_start diff --git a/hindsight-api-slim/hindsight_api/engine/search/types.py b/hindsight-api-slim/hindsight_api/engine/search/types.py index 9b464fc9..eefec8f8 100644 --- a/hindsight-api-slim/hindsight_api/engine/search/types.py +++ b/hindsight-api-slim/hindsight_api/engine/search/types.py @@ -49,6 +49,9 @@ class RetrievalResult: tags: list[str] | None = None # Visibility scope tags metadata: dict[str, str] | None = None # User-provided metadata proof_count: int | None = None # Number of supporting memories (observations only) + room: str | None = None # Topic classification (ADR-145 RCLL) + hall: str | None = None # Knowledge type classification (ADR-145 RCLL) + layer: str | None = None # Memory layer (ADR-145: L0-L3) # Retrieval-specific scores (only one will be set depending on retrieval method) similarity: float | None = None # Semantic retrieval @@ -74,6 +77,9 @@ class RetrievalResult: tags=row.get("tags"), metadata=row.get("metadata"), proof_count=row.get("proof_count"), + room=row.get("room"), + hall=row.get("hall"), + layer=row.get("layer"), similarity=row.get("similarity"), bm25_score=row.get("bm25_score"), activation=row.get("activation"), @@ -158,6 +164,9 @@ class ScoredResult: "chunk_id": self.retrieval.chunk_id, "tags": self.retrieval.tags, "metadata": self.retrieval.metadata, + "room": self.retrieval.room, + "hall": self.retrieval.hall, + "layer": self.retrieval.layer, "semantic_similarity": self.retrieval.similarity, "bm25_score": self.retrieval.bm25_score, } diff --git a/hindsight-api-slim/hindsight_api/models.py b/hindsight-api-slim/hindsight_api/models.py index 8de106ba..bd854dcf 100644 --- a/hindsight-api-slim/hindsight_api/models.py +++ b/hindsight-api-slim/hindsight_api/models.py @@ -100,6 +100,9 @@ class MemoryUnit(Base): unit_metadata: Mapped[dict] = mapped_column( "metadata", JSONB, server_default=sql_text("'{}'::jsonb") ) # User-defined metadata (str->str) + room: Mapped[str | None] = mapped_column(Text) # Topic classification (ADR-145 RCLL) + hall: Mapped[str | None] = mapped_column(Text) # Knowledge type classification (ADR-145 RCLL) + layer: Mapped[str | None] = mapped_column(Text, server_default="L2") # Memory layer (ADR-145 L0-L3) created_at: Mapped[datetime] = mapped_column(TIMESTAMP(timezone=True), server_default=func.now()) updated_at: Mapped[datetime] = mapped_column(TIMESTAMP(timezone=True), server_default=func.now())