holetron-lab/fleet-memory
1
0
Fork 0
Code Releases Packages Activity Actions

feat(rooms): per-agent rooms, halls and durability layers (ADR-145)

Browse source 
Recovered from the 2026-06-27 snapshot import by classifying the base..snapshot delta at line granularity. Upstream base: d054b884 (2026-04-10).
This commit is contained in:
RCLL 2026-08-23 23:50:01 +03:00
parent e3782b5af4
commit 179e99e89d
13 changed files with 475 additions and 25 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

69
hindsight-api-slim/hindsight_api/alembic/versions/aa1_add_room_hall_to_memory_units.py Normal file
View file

@ -0,0 +1,69 @@
"""Add room and hall columns to memory_units for hierarchical filtering (ADR-145)
Revision ID: aa1_room_hall
Revises: z1u2v3w4x5y6
Create Date: 2026-04-11
Adds room (topic) and hall (knowledge type) columns to memory_units.
Room/Hall taxonomy enables pre-semantic filtering: the candidate set is narrowed
by topic and knowledge type before the vector search runs.
"""
from collections.abc import Sequence
from alembic import context, op
revision: str = "aa1_room_hall"
down_revision: str | Sequence[str] | None = "z1u2v3w4x5y6"
branch_labels: str | Sequence[str] | None = None
depends_on: str | Sequence[str] | None = None
def _get_schema_prefix() -> str:
"""Get schema prefix for table names (required for multi-tenant support)."""
schema = context.config.get_main_option("target_schema")
return f'"{schema}".' if schema else ""
def upgrade() -> None:
schema = _get_schema_prefix()
# Room: topic classification (what the memory is about)
op.execute(f"ALTER TABLE {schema}memory_units ADD COLUMN IF NOT EXISTS room TEXT")
# Hall: knowledge type classification (what kind of knowledge)
op.execute(f"ALTER TABLE {schema}memory_units ADD COLUMN IF NOT EXISTS hall TEXT")
# Layer: memory tier L0-L3 (ADR-145). Model declares server_default 'L2';
# this column was missing from the original migration while retrieval/models
# reference it, causing `column "layer" does not exist` on recall.
op.execute(f"ALTER TABLE {schema}memory_units ADD COLUMN IF NOT EXISTS layer TEXT DEFAULT 'L2'")
# Indexes for filtering BEFORE semantic search
op.execute(
f"CREATE INDEX IF NOT EXISTS idx_memory_units_room "
f"ON {schema}memory_units (bank_id, room) WHERE room IS NOT NULL"
)
op.execute(
f"CREATE INDEX IF NOT EXISTS idx_memory_units_hall "
f"ON {schema}memory_units (bank_id, hall) WHERE hall IS NOT NULL"
)
op.execute(
f"CREATE INDEX IF NOT EXISTS idx_memory_units_room_hall "
f"ON {schema}memory_units (bank_id, room, hall) WHERE room IS NOT NULL AND hall IS NOT NULL"
)
op.execute(
f"CREATE INDEX IF NOT EXISTS idx_memory_units_layer "
f"ON {schema}memory_units (bank_id, layer) WHERE layer IS NOT NULL"
)
def downgrade() -> None:
schema = _get_schema_prefix()
op.execute(f"DROP INDEX IF EXISTS {schema}idx_memory_units_layer")
op.execute(f"ALTER TABLE {schema}memory_units DROP COLUMN IF EXISTS layer")
op.execute(f"DROP INDEX IF EXISTS {schema}idx_memory_units_room_hall")
op.execute(f"DROP INDEX IF EXISTS {schema}idx_memory_units_hall")
op.execute(f"DROP INDEX IF EXISTS {schema}idx_memory_units_room")
op.execute(f"ALTER TABLE {schema}memory_units DROP COLUMN IF EXISTS hall")
op.execute(f"ALTER TABLE {schema}memory_units DROP COLUMN IF EXISTS room")

49
hindsight-api-slim/hindsight_api/api/http.py
View file

@ -171,6 +171,22 @@ class RecallRequest(BaseModel):
description="Compound tag filter using boolean groups. Groups in the list are AND-ed. "
"Each group is a leaf {tags, match} or compound {and: [...]}, {or: [...]}, {not: ...}.",
)
room: list[str] | None = Field(
default=None,
description="Filter by room (topic). If specified, only memories in these rooms are searched. "
"Applied BEFORE semantic search, so the vector query runs over a narrowed candidate set.",
)
hall: list[str] | None = Field(
default=None,
description="Filter by hall (knowledge type). If specified, only these knowledge types are searched. "
"Values: fact, event, decision, preference, discovery, procedure, warning.",
)
max_layer: Literal["L0", "L1", "L2", "L3"] = Field(
default="L3",
description="Maximum memory layer to search (ADR-145). Recall searches L0 through max_layer. "
"L0=Identity only, L1=Identity+Critical, L2=+Session, L3=+Deep (all, default). "
"Results are ordered with L0 first (highest priority).",
)
@field_validator("query")
@classmethod
@ -223,6 +239,9 @@ class RecallResult(BaseModel):
metadata: dict[str, str] | None = None # User-defined metadata
chunk_id: str | None = None # Chunk this fact was extracted from
tags: list[str] | None = None # Visibility scope tags
room: str | None = None # Topic classification (ADR-145 RCLL)
hall: str | None = None # Knowledge type classification (ADR-145 RCLL)
layer: str | None = None # Memory layer (ADR-145: L0-L3)
source_fact_ids: list[str] | None = (
None # IDs of source facts (observation type only, when source_facts is enabled)
)
@ -469,6 +488,24 @@ class MemoryItem(BaseModel):
"'replace' (default) deletes old data and reprocesses from scratch. "
"'append' concatenates new content to the existing document text and reprocesses.",
)
room: str | None = Field(
default=None,
description="Topic classification for hierarchical filtering (ADR-145 RCLL). "
"Examples: auth, pipeline, schema, tax, hr, legal, compliance, infrastructure, ui, api, deployment, monitoring. "
"Use 'custom:{name}' for custom topics. Auto-classified by LLM if not provided.",
)
hall: str | None = Field(
default=None,
description="Knowledge type classification (ADR-145 RCLL). "
"One of: fact, event, decision, preference, discovery, procedure, warning. "
"Auto-classified by LLM if not provided.",
)
layer: Literal["L0", "L1", "L2", "L3"] = Field(
default="L2",
description="Memory layer (ADR-145). L0=Identity (~50 tokens, always loaded), "
"L1=Critical Facts (~120 tokens, per-space), L2=Session Context (default), "
"L3=Deep Memory (full search).",
)
@field_validator("timestamp", mode="before")
@classmethod
@ -2959,6 +2996,9 @@ def _register_routes(app: FastAPI):
tags=request.tags,
tags_match=request.tags_match,
tag_groups=request.tag_groups,
room=request.room,
hall=request.hall,
max_layer=request.max_layer,
)
# Convert core MemoryFact objects to API RecallResult objects (excluding internal metrics)
@ -2976,6 +3016,9 @@ def _register_routes(app: FastAPI):
metadata=fact.metadata,
chunk_id=fact.chunk_id,
tags=fact.tags,
room=getattr(fact, 'room', None),
hall=getattr(fact, 'hall', None),
layer=getattr(fact, 'layer', None),
source_fact_ids=fact.source_fact_ids,
)
@ -5333,6 +5376,12 @@ def _register_routes(app: FastAPI):
content_dict["observation_scopes"] = item.observation_scopes
if item.update_mode is not None:
content_dict["update_mode"] = item.update_mode
if item.room:
content_dict["room"] = item.room
if item.hall:
content_dict["hall"] = item.hall
if item.layer and item.layer != "L2":
content_dict["layer"] = item.layer
strategy_groups[effective].append(content_dict)
if request.async_:

26
hindsight-api-slim/hindsight_api/engine/memory_engine.py
View file

@ -2427,6 +2427,9 @@ class MemoryEngine(MemoryEngineInterface):
tags: list[str] | None = None,
tags_match: TagsMatch = "any",
tag_groups: list[TagGroup] | None = None,
room: list[str] | None = None,
hall: list[str] | None = None,
max_layer: str = "L3",
_connection_budget: int | None = None,
_quiet: bool = False,
) -> RecallResultModel:
@ -2571,6 +2574,9 @@ class MemoryEngine(MemoryEngineInterface):
tags=tags,
tags_match=tags_match,
tag_groups=tag_groups,
room=room,
hall=hall,
max_layer=max_layer,
connection_budget=_connection_budget,
quiet=_quiet,
include_source_facts=include_source_facts,
@ -2699,6 +2705,9 @@ class MemoryEngine(MemoryEngineInterface):
tags: list[str] | None = None,
tags_match: TagsMatch = "any",
tag_groups: list[TagGroup] | None = None,
room: list[str] | None = None,
hall: list[str] | None = None,
max_layer: str = "L3",
connection_budget: int | None = None,
quiet: bool = False,
include_source_facts: bool = False,
@ -2819,6 +2828,9 @@ class MemoryEngine(MemoryEngineInterface):
tags=tags,
tags_match=tags_match,
tag_groups=tag_groups,
room=room,
hall=hall,
max_layer=max_layer,
)
parallel_duration = time.time() - parallel_start
finally:
@ -2866,6 +2878,17 @@ class MemoryEngine(MemoryEngineInterface):
if not temporal_results:
temporal_results = None
# ADR-145: Python-side room/hall filtering for graph results
# (semantic/BM25/temporal are already SQL-filtered)
if room or hall:
def _room_hall_match(r):
if room and getattr(r, 'room', None) not in room:
return False
if hall and getattr(r, 'hall', None) not in hall:
return False
return True
graph_results = [r for r in graph_results if _room_hall_match(r)]
# Sort combined results by score (descending) so higher-scored results
# get better ranks in the trace, regardless of fact type
semantic_results.sort(key=lambda r: r.similarity if hasattr(r, "similarity") else 0, reverse=True)
@ -3419,6 +3442,9 @@ class MemoryEngine(MemoryEngineInterface):
metadata=result_dict.get("metadata"),
chunk_id=result_dict.get("chunk_id"),
tags=result_dict.get("tags"),
room=result_dict.get("room"),
hall=result_dict.get("hall"),
layer=result_dict.get("layer"),
source_fact_ids=source_fact_ids_by_obs.get(result_id) if include_source_facts else None,
)
)

3
hindsight-api-slim/hindsight_api/engine/response_models.py
View file

@ -175,6 +175,9 @@ class MemoryFact(BaseModel):
None, description="ID of the chunk this fact was extracted from (format: bank_id_document_id_chunk_index)"
)
tags: list[str] | None = Field(None, description="Visibility scope tags associated with this fact")
room: str | None = Field(None, description="Topic classification (ADR-145 RCLL)")
hall: str | None = Field(None, description="Knowledge type classification (ADR-145 RCLL)")
layer: str | None = Field(None, description="Memory layer (ADR-145: L0, L1, L2, L3)")
source_fact_ids: list[str] | None = Field(
None,
description="IDs of source facts this observation was derived from (observation type only, when source_facts is enabled)",

9
hindsight-api-slim/hindsight_api/engine/retain/fact_extraction.py
View file

@ -1990,6 +1990,9 @@ async def extract_facts_from_contents_batch_api(
metadata=content.metadata,
tags=content.tags,
observation_scopes=content.observation_scopes,
room=getattr(content, 'room', None),
hall=getattr(content, 'hall', None),
layer=getattr(content, 'layer', 'L2'),
)
extracted_facts.append(extracted_fact)
@ -2044,6 +2047,9 @@ def _extract_facts_chunks(
metadata=content.metadata,
tags=content.tags,
observation_scopes=content.observation_scopes,
room=getattr(content, 'room', None),
hall=getattr(content, 'hall', None),
layer=getattr(content, 'layer', 'L2'),
)
)
global_chunk_idx += 1
@ -2183,6 +2189,9 @@ async def extract_facts_from_contents(
metadata=content.metadata,
tags=content.tags,
observation_scopes=content.observation_scopes,
room=getattr(content, 'room', None),
hall=getattr(content, 'hall', None),
layer=getattr(content, 'layer', 'L2'),
)
extracted_facts.append(extracted_fact)

21
hindsight-api-slim/hindsight_api/engine/retain/fact_storage.py
View file

@ -105,6 +105,9 @@ async def insert_facts_batch(
except (ValueError, AttributeError):
pass
text_signals_list.append(" ".join(signal_parts) if signal_parts else None)
rooms_list.append(getattr(fact, 'room', None))
halls_list.append(getattr(fact, 'hall', None))
layers_list.append(getattr(fact, 'layer', 'L2'))
# Batch insert all facts
# Note: tags are passed as JSON strings and converted back to varchar[] via jsonb_array_elements_text + array_agg
@ -120,11 +123,11 @@ async def insert_facts_batch(
$8::text[], $9::text[], $10::jsonb[], $11::text[], $12::text[], $13::jsonb[], $14::jsonb[], $15::text[]
) AS t(text, embedding, event_date, occurred_start, occurred_end, mentioned_at,
context, fact_type, metadata, chunk_id, document_id, tags_json,
observation_scopes_json, text_signals)
observation_scopes_json, text_signals, room, hall, layer)
)
INSERT INTO {fq_table("memory_units")} (bank_id, text, embedding, event_date, occurred_start, occurred_end, mentioned_at,
context, fact_type, metadata, chunk_id, document_id, tags,
observation_scopes, text_signals, search_vector)
observation_scopes, text_signals, search_vector, room, hall, layer)
SELECT
$1,
text, embedding, event_date, occurred_start, occurred_end, mentioned_at,
@ -138,7 +141,10 @@ async def insert_facts_batch(
tokenize(
COALESCE(text, '') || ' ' || COALESCE(context, '') || ' ' || COALESCE(text_signals, ''),
'llmlingua2'
)::bm25_catalog.bm25vector
)::bm25_catalog.bm25vector,
room,
hall,
COALESCE(layer, 'L2')
FROM input_data
RETURNING id
"""
@ -152,11 +158,11 @@ async def insert_facts_batch(
$8::text[], $9::text[], $10::jsonb[], $11::text[], $12::text[], $13::jsonb[], $14::jsonb[], $15::text[]
) AS t(text, embedding, event_date, occurred_start, occurred_end, mentioned_at,
context, fact_type, metadata, chunk_id, document_id, tags_json,
observation_scopes_json, text_signals)
observation_scopes_json, text_signals, room, hall, layer)
)
INSERT INTO {fq_table("memory_units")} (bank_id, text, embedding, event_date, occurred_start, occurred_end, mentioned_at,
context, fact_type, metadata, chunk_id, document_id, tags,
observation_scopes, text_signals)
observation_scopes, text_signals, room, hall, layer)
SELECT
$1,
text, embedding, event_date, occurred_start, occurred_end, mentioned_at,
@ -166,7 +172,10 @@ async def insert_facts_batch(
'{{}}'::varchar[]
),
observation_scopes_json,
text_signals
text_signals,
room,
hall,
COALESCE(layer, 'L2')
FROM input_data
RETURNING id
"""

13
hindsight-api-slim/hindsight_api/engine/retain/orchestrator.py
View file

@ -390,6 +390,10 @@ async def _extract_and_embed(
processed_facts = [ProcessedFact.from_extracted_fact(ef, emb) for ef, emb in zip(extracted_facts, embeddings)]
# ADR-145: Auto-classify room/hall for facts that don't have them set
from .room_hall_classifier import classify_facts_batch
classify_facts_batch(processed_facts)
return extracted_facts, processed_facts, chunks, usage
@ -863,6 +867,9 @@ async def _streaming_retain_batch(
entities=source.entities,
tags=source.tags,
observation_scopes=source.observation_scopes,
room=getattr(source, 'room', None),
hall=getattr(source, 'hall', None),
layer=getattr(source, 'layer', 'L2'),
)
extracted, processed, chunk_meta, usage = await _extract_and_embed(
[content],
@ -1480,6 +1487,9 @@ def _build_contents(contents_dicts: list[RetainContentDict], document_tags: list
entities=item.get("entities", []),
tags=merged_tags,
observation_scopes=item.get("observation_scopes"),
room=item.get("room"),
hall=item.get("hall"),
layer=item.get("layer", "L2"),
)
contents.append(content)
return contents
@ -1536,6 +1546,9 @@ def _build_delta_contents(
entities=template_content.entities,
tags=template_content.tags,
observation_scopes=template_content.observation_scopes,
room=getattr(template_content, 'room', None),
hall=getattr(template_content, 'hall', None),
layer=getattr(template_content, 'layer', 'L2'),
)
delta_contents.append(delta_content)
delta_chunk_map[len(delta_contents) - 1] = original_chunk_idx

176
hindsight-api-slim/hindsight_api/engine/retain/room_hall_classifier.py Normal file
View file

@ -0,0 +1,176 @@
"""
Room/Hall auto-classification for ADR-145 RCLL.
Classifies memories into room (topic) and hall (knowledge type) using
keyword heuristics. Falls back to "general" room and "fact" hall when
no clear match is found.
This avoids an additional LLM call per fact (~50 tokens) while providing
reasonable classification accuracy. LLM-based classification can be added
as an enhancement later.
"""
import logging
import re
logger = logging.getLogger(__name__)
# ── Hall definitions (knowledge type) ──
# Order matters: first match wins
HALL_PATTERNS: list[tuple[str, list[str]]] = [
("warning", [
r"\bdon'?t\b", r"\bnever\b", r"\bavoid\b", r"\bdanger", r"\brisk",
r"\bcaution", r"\bwarning", r"\bcareful", r"\bdo not\b", r"\bforbid",
r"\bprohibit", r"\billegal", r"\bpenalt", r"\bfine\b",
]),
("decision", [
r"\bdecid", r"\bchose\b", r"\bchosen\b", r"\bapproved?\b", r"\brejected?\b",
r"\bagreed\b", r"\bselected?\b", r"\bpicked\b", r"\bwent with\b",
r"\bswitched? to\b", r"\bmigrat", r"\badopted?\b", r"\bresolved?\b",
]),
("procedure", [
r"\bstep\s*\d", r"\bfirst\b.*\bthen\b", r"\bprocess\b", r"\bworkflow\b",
r"\bhow to\b", r"\bprocedure\b", r"\binstructions?\b", r"\brecipe\b",
r"\brun\b.*\bcommand\b", r"\bexecute\b", r"\bsetup\b", r"\binstall\b",
]),
("event", [
r"\bhappened\b", r"\boccurred\b", r"\bmeeting\b", r"\bcall\b",
r"\bincident\b", r"\boutage\b", r"\breleased?\b", r"\blaunched?\b",
r"\bdeployed?\b", r"\bstarted?\b", r"\bfinished?\b", r"\bcompleted?\b",
r"\bon \d{4}-\d{2}-\d{2}\b", r"\byesterday\b", r"\blast week\b",
]),
("preference", [
r"\bprefer", r"\blike[sd]?\b", r"\bfavorite\b", r"\bwant[sed]?\b",
r"\brather\b", r"\bstyle\b", r"\btaste\b", r"\bchoice\b",
]),
("discovery", [
r"\bfound\b", r"\bdiscover", r"\blearned\b", r"\brealized?\b",
r"\bturns out\b", r"\bnoticed\b", r"\binsight\b", r"\bresearch",
r"\banalysis\b", r"\bbenchmark",
]),
# Default: "fact" — captured by no-match fallback
]
# ── Room definitions (topic) ──
ROOM_PATTERNS: list[tuple[str, list[str]]] = [
("auth", [
r"\bauth", r"\blogin", r"\bjwt\b", r"\btoken", r"\bsession",
r"\bpassword", r"\bcredential", r"\boauth", r"\bsso\b", r"\bsign[- ]?in\b",
]),
("pipeline", [
r"\bpipeline", r"\bci/?cd\b", r"\bbuild\b", r"\bdeploy", r"\bgithub action",
r"\bjenkins", r"\bcircleci", r"\bworkflow\b.*\bautomat",
]),
("infrastructure", [
r"\bserver", r"\bnginx\b", r"\bpm2\b", r"\bdocker", r"\bk8s\b",
r"\bkubernet", r"\binfra", r"\baws\b", r"\bgcp\b", r"\bazure\b",
r"\bvps\b", r"\bload balanc", r"\bdns\b", r"\bssl\b", r"\bcert\b",
]),
("deployment", [
r"\bdeploy", r"\brelease\b", r"\brollback", r"\bhotfix",
r"\bstaging\b", r"\bproduction\b", r"\bblue[- ]?green",
]),
("schema", [
r"\bschema", r"\bmigrat", r"\bcolumn\b", r"\btable\b", r"\bindex\b",
r"\bpostgres", r"\bdatabase\b", r"\bsql\b", r"\bquery\b",
r"\balembic", r"\bforeign key",
]),
("api", [
r"\bapi\b", r"\bendpoint", r"\broute\b", r"\brest\b", r"\bgraphql",
r"\bhttp\b", r"\brequest\b", r"\bresponse\b", r"\bwebhook",
]),
("ui", [
r"\bui\b", r"\bfrontend", r"\breact\b", r"\bcomponent", r"\bcss\b",
r"\bbutton\b", r"\bmodal\b", r"\bpanel\b", r"\blayout\b",
r"\bdesign\b", r"\bux\b", r"\bstyle\b",
]),
("tax", [
r"\btax", r"\bfiling\b", r"\bird\b", r"\bmpf\b", r"\bprofit",
r"\bdeduction", r"\bexempt", r"\btaxable\b", r"\breturn\b.*\btax",
]),
("hr", [
r"\bhr\b", r"\bhuman resource", r"\bsalary", r"\bleave\b",
r"\bemployee", r"\bhiring", r"\brecruitment", r"\bonboarding",
]),
("legal", [
r"\blegal", r"\bcontract", r"\bclause\b", r"\blawyer",
r"\blitigation", r"\bcourt\b", r"\bregulat", r"\blicense\b",
]),
("compliance", [
r"\bcompliance", r"\baudit\b", r"\bregulat", r"\bkyc\b",
r"\baml\b", r"\bgdpr\b", r"\bprivacy\b", r"\bdata protect",
]),
("monitoring", [
r"\bmonitor", r"\balert", r"\bmetric", r"\blog[sg]?\b",
r"\bgrafana\b", r"\bprometheus\b", r"\bobservab", r"\btrac[ei]",
]),
("agent", [
r"\bagent\b", r"\bllm\b", r"\bai\b", r"\bmemory\b.*\bsystem\b",
r"\bhindsight\b", r"\bprompt\b", r"\btool\b.*\bcall\b",
r"\bpes\b", r"\bspell\b", r"\borchestrat",
]),
# Default: "general" — captured by no-match fallback
]
def _match_patterns(text: str, patterns: list[tuple[str, list[str]]]) -> str | None:
"""Match text against pattern list, return first matching category or None."""
text_lower = text.lower()
for category, regexes in patterns:
for pattern in regexes:
if re.search(pattern, text_lower):
return category
return None
def classify_room_hall(
fact_text: str,
context: str | None = None,
existing_room: str | None = None,
existing_hall: str | None = None,
) -> tuple[str, str]:
"""
Classify a fact into room (topic) and hall (knowledge type).
If existing values are provided, they are kept as-is.
Otherwise, keyword heuristics determine the classification.
Args:
fact_text: The fact text to classify
context: Optional context string for additional signals
existing_room: Pre-assigned room (kept if not None)
existing_hall: Pre-assigned hall (kept if not None)
Returns:
Tuple of (room, hall)
"""
combined = f"{fact_text} {context or ''}"
room = existing_room
if room is None:
room = _match_patterns(combined, ROOM_PATTERNS) or "general"
hall = existing_hall
if hall is None:
hall = _match_patterns(combined, HALL_PATTERNS) or "fact"
return room, hall
def classify_facts_batch(facts: list) -> None:
"""
Classify room/hall for a batch of ProcessedFact or ExtractedFact objects.
Modifies facts in-place. Only sets room/hall if not already set.
"""
for fact in facts:
fact_text = getattr(fact, 'fact_text', '') or getattr(fact, 'text', '') or ''
context = getattr(fact, 'context', None)
room, hall = classify_room_hall(
fact_text,
context,
existing_room=getattr(fact, 'room', None),
existing_hall=getattr(fact, 'hall', None),
)
fact.room = room
fact.hall = hall

20
hindsight-api-slim/hindsight_api/engine/retain/types.py
View file

@ -28,6 +28,9 @@ class RetainContentDict(TypedDict, total=False):
update_mode: How to handle existing documents with the same document_id (optional).
"replace" (default) deletes old data and reprocesses. "append" concatenates
new content to the existing document and reprocesses.
room: Topic classification for hierarchical filtering (ADR-145)
hall: Knowledge type classification (ADR-145)
layer: Memory layer L0-L3 (ADR-145)
"""
content: str # Required
@ -41,6 +44,9 @@ class RetainContentDict(TypedDict, total=False):
Literal["per_tag", "combined", "all_combinations"] | list[list[str]]
) # Observation scopes for consolidation
update_mode: Literal["replace", "append"]
room: str # Topic classification (ADR-145 RCLL)
hall: str # Knowledge type classification (ADR-145 RCLL)
layer: str # Memory layer: L0, L1, L2 (default), L3 (ADR-145)
@dataclass
@ -60,6 +66,9 @@ class RetainContent:
observation_scopes: Literal["per_tag", "combined", "all_combinations"] | list[list[str]] | None = (
None # Observation scopes
)
room: str | None = None # Topic classification (ADR-145 RCLL)
hall: str | None = None # Knowledge type classification (ADR-145 RCLL)
layer: str = "L2" # Memory layer: L0, L1, L2 (default), L3 (ADR-145)
@dataclass
@ -128,6 +137,9 @@ class ExtractedFact:
observation_scopes: Literal["per_tag", "combined", "all_combinations"] | list[list[str]] | None = (
None # Observation scopes
)
room: str | None = None # Topic classification (ADR-145 RCLL)
hall: str | None = None # Knowledge type classification (ADR-145 RCLL)
layer: str = "L2" # Memory layer: L0, L1, L2 (default), L3 (ADR-145)
@dataclass
@ -179,6 +191,11 @@ class ProcessedFact:
# Observation scopes for consolidation
observation_scopes: Literal["per_tag", "combined", "all_combinations"] | list[list[str]] | None = None
# ADR-145 RCLL: hierarchical classification
room: str | None = None # Topic classification
hall: str | None = None # Knowledge type classification
layer: str = "L2" # Memory layer: L0, L1, L2 (default), L3
@property
def is_duplicate(self) -> bool:
"""Check if this fact was marked as a duplicate."""
@ -222,6 +239,9 @@ class ProcessedFact:
content_index=extracted_fact.content_index,
tags=extracted_fact.tags,
observation_scopes=extracted_fact.observation_scopes,
room=extracted_fact.room,
hall=extracted_fact.hall,
layer=extracted_fact.layer,
)

26
hindsight-api-slim/hindsight_api/engine/search/link_expansion_retrieval.py
View file

@ -64,7 +64,7 @@ async def _find_semantic_seeds(
rows = await conn.fetch(
f"""
SELECT id, text, context, event_date, occurred_start, occurred_end,
mentioned_at, fact_type, document_id, chunk_id, tags, proof_count,
mentioned_at, fact_type, document_id, chunk_id, tags, proof_count, room, hall, layer,
1 - (embedding <=> $1::vector) AS similarity
FROM {fq_table("memory_units")}
WHERE bank_id = $2
@ -289,7 +289,7 @@ class LinkExpansionRetriever(GraphRetriever):
entity_expanded AS (
SELECT mu.id, mu.text, mu.context, mu.event_date, mu.occurred_start,
mu.occurred_end, mu.mentioned_at,
mu.fact_type, mu.document_id, mu.chunk_id, mu.tags, mu.proof_count,
mu.fact_type, mu.document_id, mu.chunk_id, mu.tags, mu.proof_count, mu.room, mu.hall, mu.layer,
COUNT(DISTINCT se.entity_id)::float AS score,
'entity'::text AS source
FROM seed_entities se
@ -316,14 +316,14 @@ class LinkExpansionRetriever(GraphRetriever):
SELECT
id, text, context, event_date, occurred_start,
occurred_end, mentioned_at,
fact_type, document_id, chunk_id, tags, proof_count,
fact_type, document_id, chunk_id, tags, proof_count, room, hall, layer,
MAX(weight) AS score,
'semantic'::text AS source
FROM (
SELECT
mu.id, mu.text, mu.context, mu.event_date, mu.occurred_start,
mu.occurred_end, mu.mentioned_at,
mu.fact_type, mu.document_id, mu.chunk_id, mu.tags, mu.proof_count,
mu.fact_type, mu.document_id, mu.chunk_id, mu.tags, mu.proof_count, mu.room, mu.hall, mu.layer,
ml.weight
FROM {ml} ml
JOIN {mu} mu ON mu.id = ml.to_unit_id
@ -335,7 +335,7 @@ class LinkExpansionRetriever(GraphRetriever):
SELECT
mu.id, mu.text, mu.context, mu.event_date, mu.occurred_start,
mu.occurred_end, mu.mentioned_at,
mu.fact_type, mu.document_id, mu.chunk_id, mu.tags, mu.proof_count,
mu.fact_type, mu.document_id, mu.chunk_id, mu.tags, mu.proof_count, mu.room, mu.hall, mu.layer,
ml.weight
FROM {ml} ml
JOIN {mu} mu ON mu.id = ml.from_unit_id
@ -346,7 +346,7 @@ class LinkExpansionRetriever(GraphRetriever):
) sem_raw
GROUP BY id, text, context, event_date, occurred_start,
occurred_end, mentioned_at,
fact_type, document_id, chunk_id, tags, proof_count
fact_type, document_id, chunk_id, tags, proof_count, room, hall, layer
ORDER BY score DESC
LIMIT $3
),
@ -357,7 +357,7 @@ class LinkExpansionRetriever(GraphRetriever):
SELECT DISTINCT ON (mu.id)
mu.id, mu.text, mu.context, mu.event_date, mu.occurred_start,
mu.occurred_end, mu.mentioned_at,
mu.fact_type, mu.document_id, mu.chunk_id, mu.tags, mu.proof_count,
mu.fact_type, mu.document_id, mu.chunk_id, mu.tags, mu.proof_count, mu.room, mu.hall, mu.layer,
ml.weight AS score,
'causal'::text AS source
FROM {ml} ml
@ -480,7 +480,7 @@ class LinkExpansionRetriever(GraphRetriever):
SELECT
mu.id, mu.text, mu.context, mu.event_date, mu.occurred_start,
mu.occurred_end, mu.mentioned_at,
mu.fact_type, mu.document_id, mu.chunk_id, mu.tags, mu.proof_count,
mu.fact_type, mu.document_id, mu.chunk_id, mu.tags, mu.proof_count, mu.room, mu.hall, mu.layer,
(SELECT COUNT(DISTINCT s) FROM unnest(mu.source_memory_ids) s WHERE s = ANY(ca.source_ids))::float AS score
FROM {fq_table("memory_units")} mu, connected_array ca
WHERE mu.fact_type = 'observation'
@ -504,13 +504,13 @@ class LinkExpansionRetriever(GraphRetriever):
SELECT
id, text, context, event_date, occurred_start,
occurred_end, mentioned_at,
fact_type, document_id, chunk_id, tags, proof_count,
fact_type, document_id, chunk_id, tags, proof_count, room, hall, layer,
MAX(weight) AS score,
'semantic'::text AS source
FROM (
SELECT mu.id, mu.text, mu.context, mu.event_date, mu.occurred_start,
mu.occurred_end, mu.mentioned_at, mu.fact_type, mu.document_id,
mu.chunk_id, mu.tags, mu.proof_count, ml.weight
mu.chunk_id, mu.tags, mu.proof_count, mu.room, mu.hall, mu.layer, ml.weight
FROM {ml} ml JOIN {mu} mu ON mu.id = ml.to_unit_id
WHERE ml.from_unit_id = ANY($1::uuid[])
AND ml.link_type = 'semantic' AND mu.fact_type = 'observation'
@ -518,21 +518,21 @@ class LinkExpansionRetriever(GraphRetriever):
UNION ALL
SELECT mu.id, mu.text, mu.context, mu.event_date, mu.occurred_start,
mu.occurred_end, mu.mentioned_at, mu.fact_type, mu.document_id,
mu.chunk_id, mu.tags, mu.proof_count, ml.weight
mu.chunk_id, mu.tags, mu.proof_count, mu.room, mu.hall, mu.layer, ml.weight
FROM {ml} ml JOIN {mu} mu ON mu.id = ml.from_unit_id
WHERE ml.to_unit_id = ANY($1::uuid[])
AND ml.link_type = 'semantic' AND mu.fact_type = 'observation'
AND mu.id != ALL($1::uuid[])
) sem_raw
GROUP BY id, text, context, event_date, occurred_start, occurred_end,
mentioned_at, fact_type, document_id, chunk_id, tags, proof_count
mentioned_at, fact_type, document_id, chunk_id, tags, proof_count, room, hall, layer
ORDER BY score DESC LIMIT $2
),
causal_expanded AS (
SELECT DISTINCT ON (mu.id)
mu.id, mu.text, mu.context, mu.event_date, mu.occurred_start,
mu.occurred_end, mu.mentioned_at, mu.fact_type, mu.document_id,
mu.chunk_id, mu.tags, mu.proof_count, ml.weight AS score, 'causal'::text AS source
mu.chunk_id, mu.tags, mu.proof_count, mu.room, mu.hall, mu.layer, ml.weight AS score, 'causal'::text AS source
FROM {ml} ml JOIN {mu} mu ON ml.to_unit_id = mu.id
WHERE ml.from_unit_id = ANY($1::uuid[])
AND ml.link_type IN ('causes', 'caused_by', 'enables', 'prevents')

76
hindsight-api-slim/hindsight_api/engine/search/retrieval.py
View file

@ -98,6 +98,9 @@ async def retrieve_semantic_bm25_combined(
tags: list[str] | None = None,
tags_match: TagsMatch = "any",
tag_groups: list[TagGroup] | None = None,
room: list[str] | None = None,
hall: list[str] | None = None,
max_layer: str = "L3",
) -> dict[str, tuple[list[RetrievalResult], list[RetrievalResult]]]:
"""
Combined semantic + BM25 retrieval for multiple fact types in a single query.
@ -141,7 +144,7 @@ async def retrieve_semantic_bm25_combined(
cols = (
"id, text, context, event_date, occurred_start, occurred_end, mentioned_at, "
"fact_type, document_id, chunk_id, tags, metadata, proof_count"
"fact_type, document_id, chunk_id, tags, metadata, proof_count, room, hall, layer"
)
table = fq_table("memory_units")
@ -161,7 +164,31 @@ async def retrieve_semantic_bm25_combined(
# tag_groups params start immediately after the tags param slot
tag_groups_param_start = tags_param_idx + (1 if tags else 0)
groups_clause, groups_params, _ = build_tag_groups_where_clause(tag_groups, tag_groups_param_start)
groups_clause, groups_params, next_param = build_tag_groups_where_clause(tag_groups, tag_groups_param_start)
# ADR-145: Room/Hall filtering — applied BEFORE semantic search for accuracy boost
room_clause = ""
room_params = []
if room:
room_clause = f"AND room = ANY(${next_param}::text[])"
room_params = [room]
next_param += 1
hall_clause = ""
hall_params = []
if hall:
hall_clause = f"AND hall = ANY(${next_param}::text[])"
hall_params = [hall]
next_param += 1
# ADR-145: Layer filtering — cascade from L0 to max_layer
layer_order = {"L0": 0, "L1": 1, "L2": 2, "L3": 3}
layer_clause = ""
layer_params = []
if max_layer and max_layer != "L3":
allowed_layers = [l for l, v in layer_order.items() if v <= layer_order.get(max_layer, 3)]
layer_clause = f"AND COALESCE(layer, 'L2') = ANY(${next_param}::text[])"
layer_params = [allowed_layers]
next_param += 1
# --- Semantic UNION ALL arms (one per fact_type) ---
# Each arm has its own ORDER BY embedding <=> $1 LIMIT {hnsw_fetch}, which
@ -266,6 +293,9 @@ async def retrieve_temporal_combined(
tags: list[str] | None = None,
tags_match: TagsMatch = "any",
tag_groups: list[TagGroup] | None = None,
room: list[str] | None = None,
hall: list[str] | None = None,
max_layer: str = "L3",
) -> dict[str, list[RetrievalResult]]:
"""
Temporal retrieval for multiple fact types in a single query.
@ -298,7 +328,32 @@ async def retrieve_temporal_combined(
# Entry point query: fixed params are $1-$6, tags at $7
tags_clause = build_tags_where_clause_simple(tags, 7, match=tags_match)
tag_groups_param_start = 7 + (1 if tags else 0)
groups_clause, groups_params, _ = build_tag_groups_where_clause(tag_groups, tag_groups_param_start)
groups_clause, groups_params, next_param = build_tag_groups_where_clause(tag_groups, tag_groups_param_start)
# ADR-145: Room/Hall filtering
room_clause = ""
room_params_list = []
if room:
room_clause = f"AND room = ANY(${next_param}::text[])"
room_params_list = [room]
next_param += 1
hall_clause = ""
hall_params_list = []
if hall:
hall_clause = f"AND hall = ANY(${next_param}::text[])"
hall_params_list = [hall]
next_param += 1
# ADR-145: Layer filtering — cascade from L0 to max_layer
layer_order = {"L0": 0, "L1": 1, "L2": 2, "L3": 3}
layer_clause = ""
layer_params_list = []
if max_layer and max_layer != "L3":
allowed_layers = [l for l, v in layer_order.items() if v <= layer_order.get(max_layer, 3)]
layer_clause = f"AND COALESCE(layer, 'L2') = ANY(${next_param}::text[])"
layer_params_list = [allowed_layers]
next_param += 1
params: list = [query_emb_str, bank_id, fact_types, start_date, end_date, semantic_threshold]
if tags:
params.append(tags)
@ -336,7 +391,7 @@ async def retrieve_temporal_combined(
{groups_clause}
),
sim_ranked AS (
SELECT mu.id, mu.text, mu.context, mu.event_date, mu.occurred_start, mu.occurred_end, mu.mentioned_at, mu.fact_type, mu.proof_count, mu.document_id, mu.chunk_id, mu.tags, mu.metadata,
SELECT mu.id, mu.text, mu.context, mu.event_date, mu.occurred_start, mu.occurred_end, mu.mentioned_at, mu.fact_type, mu.proof_count, mu.document_id, mu.chunk_id, mu.tags, mu.metadata, mu.room, mu.hall, mu.layer,
1 - (mu.embedding <=> $1::vector) AS similarity,
ROW_NUMBER() OVER (PARTITION BY mu.fact_type ORDER BY mu.embedding <=> $1::vector) AS sim_rn
FROM date_ranked dr
@ -344,7 +399,7 @@ async def retrieve_temporal_combined(
WHERE dr.rn <= 50
AND (1 - (mu.embedding <=> $1::vector)) >= $6
)
SELECT id, text, context, event_date, occurred_start, occurred_end, mentioned_at, fact_type, proof_count, document_id, chunk_id, tags, metadata, similarity
SELECT id, text, context, event_date, occurred_start, occurred_end, mentioned_at, fact_type, proof_count, document_id, chunk_id, tags, metadata, room, hall, layer, similarity
FROM sim_ranked
WHERE sim_rn <= 10
""",
@ -442,7 +497,7 @@ async def retrieve_temporal_combined(
# bank_id on memory_units lets the planner use idx_memory_units_bank_fact_type.
neighbors = await conn.fetch(
f"""
SELECT src.from_unit_id, mu.id, mu.text, mu.context, mu.event_date, mu.occurred_start, mu.occurred_end, mu.mentioned_at, mu.fact_type, mu.document_id, mu.chunk_id, mu.tags, mu.metadata,
SELECT src.from_unit_id, mu.id, mu.text, mu.context, mu.event_date, mu.occurred_start, mu.occurred_end, mu.mentioned_at, mu.fact_type, mu.document_id, mu.chunk_id, mu.tags, mu.metadata, mu.room, mu.hall,
l.weight, l.link_type,
1 - (mu.embedding <=> $1::vector) AS similarity
FROM unnest($2::uuid[]) AS src(from_unit_id)
@ -536,6 +591,9 @@ async def retrieve_all_fact_types_parallel(
tags: list[str] | None = None,
tags_match: TagsMatch = "any",
tag_groups: list[TagGroup] | None = None,
room: list[str] | None = None,
hall: list[str] | None = None,
max_layer: str = "L3",
) -> MultiFactTypeRetrievalResult:
"""
Optimized retrieval for multiple fact types using batched queries.
@ -594,6 +652,9 @@ async def retrieve_all_fact_types_parallel(
tags=tags,
tags_match=tags_match,
tag_groups=tag_groups,
room=room,
hall=hall,
max_layer=max_layer,
)
semantic_bm25_time = time.time() - semantic_bm25_start
@ -613,6 +674,9 @@ async def retrieve_all_fact_types_parallel(
tags=tags,
tags_match=tags_match,
tag_groups=tag_groups,
room=room,
hall=hall,
max_layer=max_layer,
)
temporal_time = time.time() - temporal_start

9
hindsight-api-slim/hindsight_api/engine/search/types.py
View file

@ -49,6 +49,9 @@ class RetrievalResult:
tags: list[str] | None = None # Visibility scope tags
metadata: dict[str, str] | None = None # User-provided metadata
proof_count: int | None = None # Number of supporting memories (observations only)
room: str | None = None # Topic classification (ADR-145 RCLL)
hall: str | None = None # Knowledge type classification (ADR-145 RCLL)
layer: str | None = None # Memory layer (ADR-145: L0-L3)
# Retrieval-specific scores (only one will be set depending on retrieval method)
similarity: float | None = None # Semantic retrieval
@ -74,6 +77,9 @@ class RetrievalResult:
tags=row.get("tags"),
metadata=row.get("metadata"),
proof_count=row.get("proof_count"),
room=row.get("room"),
hall=row.get("hall"),
layer=row.get("layer"),
similarity=row.get("similarity"),
bm25_score=row.get("bm25_score"),
activation=row.get("activation"),
@ -158,6 +164,9 @@ class ScoredResult:
"chunk_id": self.retrieval.chunk_id,
"tags": self.retrieval.tags,
"metadata": self.retrieval.metadata,
"room": self.retrieval.room,
"hall": self.retrieval.hall,
"layer": self.retrieval.layer,
"semantic_similarity": self.retrieval.similarity,
"bm25_score": self.retrieval.bm25_score,
}

3
hindsight-api-slim/hindsight_api/models.py
View file

@ -100,6 +100,9 @@ class MemoryUnit(Base):
unit_metadata: Mapped[dict] = mapped_column(
"metadata", JSONB, server_default=sql_text("'{}'::jsonb")
) # User-defined metadata (str->str)
room: Mapped[str | None] = mapped_column(Text) # Topic classification (ADR-145 RCLL)
hall: Mapped[str | None] = mapped_column(Text) # Knowledge type classification (ADR-145 RCLL)
layer: Mapped[str | None] = mapped_column(Text, server_default="L2") # Memory layer (ADR-145 L0-L3)
created_at: Mapped[datetime] = mapped_column(TIMESTAMP(timezone=True), server_default=func.now())
updated_at: Mapped[datetime] = mapped_column(TIMESTAMP(timezone=True), server_default=func.now())