feat(rooms): per-agent rooms, halls and durability layers (ADR-145)
Recovered from the 2026-06-27 snapshot import by classifying the base..snapshot delta at line granularity. Upstream base: d054b884 (2026-04-10).
This commit is contained in:
parent
e3782b5af4
commit
179e99e89d
13 changed files with 475 additions and 25 deletions
|
|
@ -0,0 +1,69 @@
|
||||||
|
"""Add room and hall columns to memory_units for hierarchical filtering (ADR-145)
|
||||||
|
|
||||||
|
Revision ID: aa1_room_hall
|
||||||
|
Revises: z1u2v3w4x5y6
|
||||||
|
Create Date: 2026-04-11
|
||||||
|
|
||||||
|
Adds room (topic) and hall (knowledge type) columns to memory_units.
|
||||||
|
Room/Hall taxonomy enables pre-semantic filtering: the candidate set is narrowed
|
||||||
|
by topic and knowledge type before the vector search runs.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from collections.abc import Sequence
|
||||||
|
|
||||||
|
from alembic import context, op
|
||||||
|
|
||||||
|
revision: str = "aa1_room_hall"
|
||||||
|
down_revision: str | Sequence[str] | None = "z1u2v3w4x5y6"
|
||||||
|
branch_labels: str | Sequence[str] | None = None
|
||||||
|
depends_on: str | Sequence[str] | None = None
|
||||||
|
|
||||||
|
|
||||||
|
def _get_schema_prefix() -> str:
|
||||||
|
"""Get schema prefix for table names (required for multi-tenant support)."""
|
||||||
|
schema = context.config.get_main_option("target_schema")
|
||||||
|
return f'"{schema}".' if schema else ""
|
||||||
|
|
||||||
|
|
||||||
|
def upgrade() -> None:
|
||||||
|
schema = _get_schema_prefix()
|
||||||
|
|
||||||
|
# Room: topic classification (what the memory is about)
|
||||||
|
op.execute(f"ALTER TABLE {schema}memory_units ADD COLUMN IF NOT EXISTS room TEXT")
|
||||||
|
|
||||||
|
# Hall: knowledge type classification (what kind of knowledge)
|
||||||
|
op.execute(f"ALTER TABLE {schema}memory_units ADD COLUMN IF NOT EXISTS hall TEXT")
|
||||||
|
|
||||||
|
# Layer: memory tier L0-L3 (ADR-145). Model declares server_default 'L2';
|
||||||
|
# this column was missing from the original migration while retrieval/models
|
||||||
|
# reference it, causing `column "layer" does not exist` on recall.
|
||||||
|
op.execute(f"ALTER TABLE {schema}memory_units ADD COLUMN IF NOT EXISTS layer TEXT DEFAULT 'L2'")
|
||||||
|
|
||||||
|
# Indexes for filtering BEFORE semantic search
|
||||||
|
op.execute(
|
||||||
|
f"CREATE INDEX IF NOT EXISTS idx_memory_units_room "
|
||||||
|
f"ON {schema}memory_units (bank_id, room) WHERE room IS NOT NULL"
|
||||||
|
)
|
||||||
|
op.execute(
|
||||||
|
f"CREATE INDEX IF NOT EXISTS idx_memory_units_hall "
|
||||||
|
f"ON {schema}memory_units (bank_id, hall) WHERE hall IS NOT NULL"
|
||||||
|
)
|
||||||
|
op.execute(
|
||||||
|
f"CREATE INDEX IF NOT EXISTS idx_memory_units_room_hall "
|
||||||
|
f"ON {schema}memory_units (bank_id, room, hall) WHERE room IS NOT NULL AND hall IS NOT NULL"
|
||||||
|
)
|
||||||
|
op.execute(
|
||||||
|
f"CREATE INDEX IF NOT EXISTS idx_memory_units_layer "
|
||||||
|
f"ON {schema}memory_units (bank_id, layer) WHERE layer IS NOT NULL"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def downgrade() -> None:
|
||||||
|
schema = _get_schema_prefix()
|
||||||
|
op.execute(f"DROP INDEX IF EXISTS {schema}idx_memory_units_layer")
|
||||||
|
op.execute(f"ALTER TABLE {schema}memory_units DROP COLUMN IF EXISTS layer")
|
||||||
|
op.execute(f"DROP INDEX IF EXISTS {schema}idx_memory_units_room_hall")
|
||||||
|
op.execute(f"DROP INDEX IF EXISTS {schema}idx_memory_units_hall")
|
||||||
|
op.execute(f"DROP INDEX IF EXISTS {schema}idx_memory_units_room")
|
||||||
|
op.execute(f"ALTER TABLE {schema}memory_units DROP COLUMN IF EXISTS hall")
|
||||||
|
op.execute(f"ALTER TABLE {schema}memory_units DROP COLUMN IF EXISTS room")
|
||||||
|
|
@ -171,6 +171,22 @@ class RecallRequest(BaseModel):
|
||||||
description="Compound tag filter using boolean groups. Groups in the list are AND-ed. "
|
description="Compound tag filter using boolean groups. Groups in the list are AND-ed. "
|
||||||
"Each group is a leaf {tags, match} or compound {and: [...]}, {or: [...]}, {not: ...}.",
|
"Each group is a leaf {tags, match} or compound {and: [...]}, {or: [...]}, {not: ...}.",
|
||||||
)
|
)
|
||||||
|
room: list[str] | None = Field(
|
||||||
|
default=None,
|
||||||
|
description="Filter by room (topic). If specified, only memories in these rooms are searched. "
|
||||||
|
"Applied BEFORE semantic search, so the vector query runs over a narrowed candidate set.",
|
||||||
|
)
|
||||||
|
hall: list[str] | None = Field(
|
||||||
|
default=None,
|
||||||
|
description="Filter by hall (knowledge type). If specified, only these knowledge types are searched. "
|
||||||
|
"Values: fact, event, decision, preference, discovery, procedure, warning.",
|
||||||
|
)
|
||||||
|
max_layer: Literal["L0", "L1", "L2", "L3"] = Field(
|
||||||
|
default="L3",
|
||||||
|
description="Maximum memory layer to search (ADR-145). Recall searches L0 through max_layer. "
|
||||||
|
"L0=Identity only, L1=Identity+Critical, L2=+Session, L3=+Deep (all, default). "
|
||||||
|
"Results are ordered with L0 first (highest priority).",
|
||||||
|
)
|
||||||
|
|
||||||
@field_validator("query")
|
@field_validator("query")
|
||||||
@classmethod
|
@classmethod
|
||||||
|
|
@ -223,6 +239,9 @@ class RecallResult(BaseModel):
|
||||||
metadata: dict[str, str] | None = None # User-defined metadata
|
metadata: dict[str, str] | None = None # User-defined metadata
|
||||||
chunk_id: str | None = None # Chunk this fact was extracted from
|
chunk_id: str | None = None # Chunk this fact was extracted from
|
||||||
tags: list[str] | None = None # Visibility scope tags
|
tags: list[str] | None = None # Visibility scope tags
|
||||||
|
room: str | None = None # Topic classification (ADR-145 RCLL)
|
||||||
|
hall: str | None = None # Knowledge type classification (ADR-145 RCLL)
|
||||||
|
layer: str | None = None # Memory layer (ADR-145: L0-L3)
|
||||||
source_fact_ids: list[str] | None = (
|
source_fact_ids: list[str] | None = (
|
||||||
None # IDs of source facts (observation type only, when source_facts is enabled)
|
None # IDs of source facts (observation type only, when source_facts is enabled)
|
||||||
)
|
)
|
||||||
|
|
@ -469,6 +488,24 @@ class MemoryItem(BaseModel):
|
||||||
"'replace' (default) deletes old data and reprocesses from scratch. "
|
"'replace' (default) deletes old data and reprocesses from scratch. "
|
||||||
"'append' concatenates new content to the existing document text and reprocesses.",
|
"'append' concatenates new content to the existing document text and reprocesses.",
|
||||||
)
|
)
|
||||||
|
room: str | None = Field(
|
||||||
|
default=None,
|
||||||
|
description="Topic classification for hierarchical filtering (ADR-145 RCLL). "
|
||||||
|
"Examples: auth, pipeline, schema, tax, hr, legal, compliance, infrastructure, ui, api, deployment, monitoring. "
|
||||||
|
"Use 'custom:{name}' for custom topics. Auto-classified by LLM if not provided.",
|
||||||
|
)
|
||||||
|
hall: str | None = Field(
|
||||||
|
default=None,
|
||||||
|
description="Knowledge type classification (ADR-145 RCLL). "
|
||||||
|
"One of: fact, event, decision, preference, discovery, procedure, warning. "
|
||||||
|
"Auto-classified by LLM if not provided.",
|
||||||
|
)
|
||||||
|
layer: Literal["L0", "L1", "L2", "L3"] = Field(
|
||||||
|
default="L2",
|
||||||
|
description="Memory layer (ADR-145). L0=Identity (~50 tokens, always loaded), "
|
||||||
|
"L1=Critical Facts (~120 tokens, per-space), L2=Session Context (default), "
|
||||||
|
"L3=Deep Memory (full search).",
|
||||||
|
)
|
||||||
|
|
||||||
@field_validator("timestamp", mode="before")
|
@field_validator("timestamp", mode="before")
|
||||||
@classmethod
|
@classmethod
|
||||||
|
|
@ -2959,6 +2996,9 @@ def _register_routes(app: FastAPI):
|
||||||
tags=request.tags,
|
tags=request.tags,
|
||||||
tags_match=request.tags_match,
|
tags_match=request.tags_match,
|
||||||
tag_groups=request.tag_groups,
|
tag_groups=request.tag_groups,
|
||||||
|
room=request.room,
|
||||||
|
hall=request.hall,
|
||||||
|
max_layer=request.max_layer,
|
||||||
)
|
)
|
||||||
|
|
||||||
# Convert core MemoryFact objects to API RecallResult objects (excluding internal metrics)
|
# Convert core MemoryFact objects to API RecallResult objects (excluding internal metrics)
|
||||||
|
|
@ -2976,6 +3016,9 @@ def _register_routes(app: FastAPI):
|
||||||
metadata=fact.metadata,
|
metadata=fact.metadata,
|
||||||
chunk_id=fact.chunk_id,
|
chunk_id=fact.chunk_id,
|
||||||
tags=fact.tags,
|
tags=fact.tags,
|
||||||
|
room=getattr(fact, 'room', None),
|
||||||
|
hall=getattr(fact, 'hall', None),
|
||||||
|
layer=getattr(fact, 'layer', None),
|
||||||
source_fact_ids=fact.source_fact_ids,
|
source_fact_ids=fact.source_fact_ids,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
@ -5333,6 +5376,12 @@ def _register_routes(app: FastAPI):
|
||||||
content_dict["observation_scopes"] = item.observation_scopes
|
content_dict["observation_scopes"] = item.observation_scopes
|
||||||
if item.update_mode is not None:
|
if item.update_mode is not None:
|
||||||
content_dict["update_mode"] = item.update_mode
|
content_dict["update_mode"] = item.update_mode
|
||||||
|
if item.room:
|
||||||
|
content_dict["room"] = item.room
|
||||||
|
if item.hall:
|
||||||
|
content_dict["hall"] = item.hall
|
||||||
|
if item.layer and item.layer != "L2":
|
||||||
|
content_dict["layer"] = item.layer
|
||||||
strategy_groups[effective].append(content_dict)
|
strategy_groups[effective].append(content_dict)
|
||||||
|
|
||||||
if request.async_:
|
if request.async_:
|
||||||
|
|
|
||||||
|
|
@ -2427,6 +2427,9 @@ class MemoryEngine(MemoryEngineInterface):
|
||||||
tags: list[str] | None = None,
|
tags: list[str] | None = None,
|
||||||
tags_match: TagsMatch = "any",
|
tags_match: TagsMatch = "any",
|
||||||
tag_groups: list[TagGroup] | None = None,
|
tag_groups: list[TagGroup] | None = None,
|
||||||
|
room: list[str] | None = None,
|
||||||
|
hall: list[str] | None = None,
|
||||||
|
max_layer: str = "L3",
|
||||||
_connection_budget: int | None = None,
|
_connection_budget: int | None = None,
|
||||||
_quiet: bool = False,
|
_quiet: bool = False,
|
||||||
) -> RecallResultModel:
|
) -> RecallResultModel:
|
||||||
|
|
@ -2571,6 +2574,9 @@ class MemoryEngine(MemoryEngineInterface):
|
||||||
tags=tags,
|
tags=tags,
|
||||||
tags_match=tags_match,
|
tags_match=tags_match,
|
||||||
tag_groups=tag_groups,
|
tag_groups=tag_groups,
|
||||||
|
room=room,
|
||||||
|
hall=hall,
|
||||||
|
max_layer=max_layer,
|
||||||
connection_budget=_connection_budget,
|
connection_budget=_connection_budget,
|
||||||
quiet=_quiet,
|
quiet=_quiet,
|
||||||
include_source_facts=include_source_facts,
|
include_source_facts=include_source_facts,
|
||||||
|
|
@ -2699,6 +2705,9 @@ class MemoryEngine(MemoryEngineInterface):
|
||||||
tags: list[str] | None = None,
|
tags: list[str] | None = None,
|
||||||
tags_match: TagsMatch = "any",
|
tags_match: TagsMatch = "any",
|
||||||
tag_groups: list[TagGroup] | None = None,
|
tag_groups: list[TagGroup] | None = None,
|
||||||
|
room: list[str] | None = None,
|
||||||
|
hall: list[str] | None = None,
|
||||||
|
max_layer: str = "L3",
|
||||||
connection_budget: int | None = None,
|
connection_budget: int | None = None,
|
||||||
quiet: bool = False,
|
quiet: bool = False,
|
||||||
include_source_facts: bool = False,
|
include_source_facts: bool = False,
|
||||||
|
|
@ -2819,6 +2828,9 @@ class MemoryEngine(MemoryEngineInterface):
|
||||||
tags=tags,
|
tags=tags,
|
||||||
tags_match=tags_match,
|
tags_match=tags_match,
|
||||||
tag_groups=tag_groups,
|
tag_groups=tag_groups,
|
||||||
|
room=room,
|
||||||
|
hall=hall,
|
||||||
|
max_layer=max_layer,
|
||||||
)
|
)
|
||||||
parallel_duration = time.time() - parallel_start
|
parallel_duration = time.time() - parallel_start
|
||||||
finally:
|
finally:
|
||||||
|
|
@ -2866,6 +2878,17 @@ class MemoryEngine(MemoryEngineInterface):
|
||||||
if not temporal_results:
|
if not temporal_results:
|
||||||
temporal_results = None
|
temporal_results = None
|
||||||
|
|
||||||
|
# ADR-145: Python-side room/hall filtering for graph results
|
||||||
|
# (semantic/BM25/temporal are already SQL-filtered)
|
||||||
|
if room or hall:
|
||||||
|
def _room_hall_match(r):
|
||||||
|
if room and getattr(r, 'room', None) not in room:
|
||||||
|
return False
|
||||||
|
if hall and getattr(r, 'hall', None) not in hall:
|
||||||
|
return False
|
||||||
|
return True
|
||||||
|
graph_results = [r for r in graph_results if _room_hall_match(r)]
|
||||||
|
|
||||||
# Sort combined results by score (descending) so higher-scored results
|
# Sort combined results by score (descending) so higher-scored results
|
||||||
# get better ranks in the trace, regardless of fact type
|
# get better ranks in the trace, regardless of fact type
|
||||||
semantic_results.sort(key=lambda r: r.similarity if hasattr(r, "similarity") else 0, reverse=True)
|
semantic_results.sort(key=lambda r: r.similarity if hasattr(r, "similarity") else 0, reverse=True)
|
||||||
|
|
@ -3419,6 +3442,9 @@ class MemoryEngine(MemoryEngineInterface):
|
||||||
metadata=result_dict.get("metadata"),
|
metadata=result_dict.get("metadata"),
|
||||||
chunk_id=result_dict.get("chunk_id"),
|
chunk_id=result_dict.get("chunk_id"),
|
||||||
tags=result_dict.get("tags"),
|
tags=result_dict.get("tags"),
|
||||||
|
room=result_dict.get("room"),
|
||||||
|
hall=result_dict.get("hall"),
|
||||||
|
layer=result_dict.get("layer"),
|
||||||
source_fact_ids=source_fact_ids_by_obs.get(result_id) if include_source_facts else None,
|
source_fact_ids=source_fact_ids_by_obs.get(result_id) if include_source_facts else None,
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
|
|
|
||||||
|
|
@ -175,6 +175,9 @@ class MemoryFact(BaseModel):
|
||||||
None, description="ID of the chunk this fact was extracted from (format: bank_id_document_id_chunk_index)"
|
None, description="ID of the chunk this fact was extracted from (format: bank_id_document_id_chunk_index)"
|
||||||
)
|
)
|
||||||
tags: list[str] | None = Field(None, description="Visibility scope tags associated with this fact")
|
tags: list[str] | None = Field(None, description="Visibility scope tags associated with this fact")
|
||||||
|
room: str | None = Field(None, description="Topic classification (ADR-145 RCLL)")
|
||||||
|
hall: str | None = Field(None, description="Knowledge type classification (ADR-145 RCLL)")
|
||||||
|
layer: str | None = Field(None, description="Memory layer (ADR-145: L0, L1, L2, L3)")
|
||||||
source_fact_ids: list[str] | None = Field(
|
source_fact_ids: list[str] | None = Field(
|
||||||
None,
|
None,
|
||||||
description="IDs of source facts this observation was derived from (observation type only, when source_facts is enabled)",
|
description="IDs of source facts this observation was derived from (observation type only, when source_facts is enabled)",
|
||||||
|
|
|
||||||
|
|
@ -1990,6 +1990,9 @@ async def extract_facts_from_contents_batch_api(
|
||||||
metadata=content.metadata,
|
metadata=content.metadata,
|
||||||
tags=content.tags,
|
tags=content.tags,
|
||||||
observation_scopes=content.observation_scopes,
|
observation_scopes=content.observation_scopes,
|
||||||
|
room=getattr(content, 'room', None),
|
||||||
|
hall=getattr(content, 'hall', None),
|
||||||
|
layer=getattr(content, 'layer', 'L2'),
|
||||||
)
|
)
|
||||||
|
|
||||||
extracted_facts.append(extracted_fact)
|
extracted_facts.append(extracted_fact)
|
||||||
|
|
@ -2044,6 +2047,9 @@ def _extract_facts_chunks(
|
||||||
metadata=content.metadata,
|
metadata=content.metadata,
|
||||||
tags=content.tags,
|
tags=content.tags,
|
||||||
observation_scopes=content.observation_scopes,
|
observation_scopes=content.observation_scopes,
|
||||||
|
room=getattr(content, 'room', None),
|
||||||
|
hall=getattr(content, 'hall', None),
|
||||||
|
layer=getattr(content, 'layer', 'L2'),
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
global_chunk_idx += 1
|
global_chunk_idx += 1
|
||||||
|
|
@ -2183,6 +2189,9 @@ async def extract_facts_from_contents(
|
||||||
metadata=content.metadata,
|
metadata=content.metadata,
|
||||||
tags=content.tags,
|
tags=content.tags,
|
||||||
observation_scopes=content.observation_scopes,
|
observation_scopes=content.observation_scopes,
|
||||||
|
room=getattr(content, 'room', None),
|
||||||
|
hall=getattr(content, 'hall', None),
|
||||||
|
layer=getattr(content, 'layer', 'L2'),
|
||||||
)
|
)
|
||||||
|
|
||||||
extracted_facts.append(extracted_fact)
|
extracted_facts.append(extracted_fact)
|
||||||
|
|
|
||||||
|
|
@ -105,6 +105,9 @@ async def insert_facts_batch(
|
||||||
except (ValueError, AttributeError):
|
except (ValueError, AttributeError):
|
||||||
pass
|
pass
|
||||||
text_signals_list.append(" ".join(signal_parts) if signal_parts else None)
|
text_signals_list.append(" ".join(signal_parts) if signal_parts else None)
|
||||||
|
rooms_list.append(getattr(fact, 'room', None))
|
||||||
|
halls_list.append(getattr(fact, 'hall', None))
|
||||||
|
layers_list.append(getattr(fact, 'layer', 'L2'))
|
||||||
|
|
||||||
# Batch insert all facts
|
# Batch insert all facts
|
||||||
# Note: tags are passed as JSON strings and converted back to varchar[] via jsonb_array_elements_text + array_agg
|
# Note: tags are passed as JSON strings and converted back to varchar[] via jsonb_array_elements_text + array_agg
|
||||||
|
|
@ -120,11 +123,11 @@ async def insert_facts_batch(
|
||||||
$8::text[], $9::text[], $10::jsonb[], $11::text[], $12::text[], $13::jsonb[], $14::jsonb[], $15::text[]
|
$8::text[], $9::text[], $10::jsonb[], $11::text[], $12::text[], $13::jsonb[], $14::jsonb[], $15::text[]
|
||||||
) AS t(text, embedding, event_date, occurred_start, occurred_end, mentioned_at,
|
) AS t(text, embedding, event_date, occurred_start, occurred_end, mentioned_at,
|
||||||
context, fact_type, metadata, chunk_id, document_id, tags_json,
|
context, fact_type, metadata, chunk_id, document_id, tags_json,
|
||||||
observation_scopes_json, text_signals)
|
observation_scopes_json, text_signals, room, hall, layer)
|
||||||
)
|
)
|
||||||
INSERT INTO {fq_table("memory_units")} (bank_id, text, embedding, event_date, occurred_start, occurred_end, mentioned_at,
|
INSERT INTO {fq_table("memory_units")} (bank_id, text, embedding, event_date, occurred_start, occurred_end, mentioned_at,
|
||||||
context, fact_type, metadata, chunk_id, document_id, tags,
|
context, fact_type, metadata, chunk_id, document_id, tags,
|
||||||
observation_scopes, text_signals, search_vector)
|
observation_scopes, text_signals, search_vector, room, hall, layer)
|
||||||
SELECT
|
SELECT
|
||||||
$1,
|
$1,
|
||||||
text, embedding, event_date, occurred_start, occurred_end, mentioned_at,
|
text, embedding, event_date, occurred_start, occurred_end, mentioned_at,
|
||||||
|
|
@ -138,7 +141,10 @@ async def insert_facts_batch(
|
||||||
tokenize(
|
tokenize(
|
||||||
COALESCE(text, '') || ' ' || COALESCE(context, '') || ' ' || COALESCE(text_signals, ''),
|
COALESCE(text, '') || ' ' || COALESCE(context, '') || ' ' || COALESCE(text_signals, ''),
|
||||||
'llmlingua2'
|
'llmlingua2'
|
||||||
)::bm25_catalog.bm25vector
|
)::bm25_catalog.bm25vector,
|
||||||
|
room,
|
||||||
|
hall,
|
||||||
|
COALESCE(layer, 'L2')
|
||||||
FROM input_data
|
FROM input_data
|
||||||
RETURNING id
|
RETURNING id
|
||||||
"""
|
"""
|
||||||
|
|
@ -152,11 +158,11 @@ async def insert_facts_batch(
|
||||||
$8::text[], $9::text[], $10::jsonb[], $11::text[], $12::text[], $13::jsonb[], $14::jsonb[], $15::text[]
|
$8::text[], $9::text[], $10::jsonb[], $11::text[], $12::text[], $13::jsonb[], $14::jsonb[], $15::text[]
|
||||||
) AS t(text, embedding, event_date, occurred_start, occurred_end, mentioned_at,
|
) AS t(text, embedding, event_date, occurred_start, occurred_end, mentioned_at,
|
||||||
context, fact_type, metadata, chunk_id, document_id, tags_json,
|
context, fact_type, metadata, chunk_id, document_id, tags_json,
|
||||||
observation_scopes_json, text_signals)
|
observation_scopes_json, text_signals, room, hall, layer)
|
||||||
)
|
)
|
||||||
INSERT INTO {fq_table("memory_units")} (bank_id, text, embedding, event_date, occurred_start, occurred_end, mentioned_at,
|
INSERT INTO {fq_table("memory_units")} (bank_id, text, embedding, event_date, occurred_start, occurred_end, mentioned_at,
|
||||||
context, fact_type, metadata, chunk_id, document_id, tags,
|
context, fact_type, metadata, chunk_id, document_id, tags,
|
||||||
observation_scopes, text_signals)
|
observation_scopes, text_signals, room, hall, layer)
|
||||||
SELECT
|
SELECT
|
||||||
$1,
|
$1,
|
||||||
text, embedding, event_date, occurred_start, occurred_end, mentioned_at,
|
text, embedding, event_date, occurred_start, occurred_end, mentioned_at,
|
||||||
|
|
@ -166,7 +172,10 @@ async def insert_facts_batch(
|
||||||
'{{}}'::varchar[]
|
'{{}}'::varchar[]
|
||||||
),
|
),
|
||||||
observation_scopes_json,
|
observation_scopes_json,
|
||||||
text_signals
|
text_signals,
|
||||||
|
room,
|
||||||
|
hall,
|
||||||
|
COALESCE(layer, 'L2')
|
||||||
FROM input_data
|
FROM input_data
|
||||||
RETURNING id
|
RETURNING id
|
||||||
"""
|
"""
|
||||||
|
|
|
||||||
|
|
@ -390,6 +390,10 @@ async def _extract_and_embed(
|
||||||
|
|
||||||
processed_facts = [ProcessedFact.from_extracted_fact(ef, emb) for ef, emb in zip(extracted_facts, embeddings)]
|
processed_facts = [ProcessedFact.from_extracted_fact(ef, emb) for ef, emb in zip(extracted_facts, embeddings)]
|
||||||
|
|
||||||
|
# ADR-145: Auto-classify room/hall for facts that don't have them set
|
||||||
|
from .room_hall_classifier import classify_facts_batch
|
||||||
|
classify_facts_batch(processed_facts)
|
||||||
|
|
||||||
return extracted_facts, processed_facts, chunks, usage
|
return extracted_facts, processed_facts, chunks, usage
|
||||||
|
|
||||||
|
|
||||||
|
|
@ -863,6 +867,9 @@ async def _streaming_retain_batch(
|
||||||
entities=source.entities,
|
entities=source.entities,
|
||||||
tags=source.tags,
|
tags=source.tags,
|
||||||
observation_scopes=source.observation_scopes,
|
observation_scopes=source.observation_scopes,
|
||||||
|
room=getattr(source, 'room', None),
|
||||||
|
hall=getattr(source, 'hall', None),
|
||||||
|
layer=getattr(source, 'layer', 'L2'),
|
||||||
)
|
)
|
||||||
extracted, processed, chunk_meta, usage = await _extract_and_embed(
|
extracted, processed, chunk_meta, usage = await _extract_and_embed(
|
||||||
[content],
|
[content],
|
||||||
|
|
@ -1480,6 +1487,9 @@ def _build_contents(contents_dicts: list[RetainContentDict], document_tags: list
|
||||||
entities=item.get("entities", []),
|
entities=item.get("entities", []),
|
||||||
tags=merged_tags,
|
tags=merged_tags,
|
||||||
observation_scopes=item.get("observation_scopes"),
|
observation_scopes=item.get("observation_scopes"),
|
||||||
|
room=item.get("room"),
|
||||||
|
hall=item.get("hall"),
|
||||||
|
layer=item.get("layer", "L2"),
|
||||||
)
|
)
|
||||||
contents.append(content)
|
contents.append(content)
|
||||||
return contents
|
return contents
|
||||||
|
|
@ -1536,6 +1546,9 @@ def _build_delta_contents(
|
||||||
entities=template_content.entities,
|
entities=template_content.entities,
|
||||||
tags=template_content.tags,
|
tags=template_content.tags,
|
||||||
observation_scopes=template_content.observation_scopes,
|
observation_scopes=template_content.observation_scopes,
|
||||||
|
room=getattr(template_content, 'room', None),
|
||||||
|
hall=getattr(template_content, 'hall', None),
|
||||||
|
layer=getattr(template_content, 'layer', 'L2'),
|
||||||
)
|
)
|
||||||
delta_contents.append(delta_content)
|
delta_contents.append(delta_content)
|
||||||
delta_chunk_map[len(delta_contents) - 1] = original_chunk_idx
|
delta_chunk_map[len(delta_contents) - 1] = original_chunk_idx
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,176 @@
|
||||||
|
"""
|
||||||
|
Room/Hall auto-classification for ADR-145 RCLL.
|
||||||
|
|
||||||
|
Classifies memories into room (topic) and hall (knowledge type) using
|
||||||
|
keyword heuristics. Falls back to "general" room and "fact" hall when
|
||||||
|
no clear match is found.
|
||||||
|
|
||||||
|
This avoids an additional LLM call per fact (~50 tokens) while providing
|
||||||
|
reasonable classification accuracy. LLM-based classification can be added
|
||||||
|
as an enhancement later.
|
||||||
|
"""
|
||||||
|
|
||||||
|
import logging
|
||||||
|
import re
|
||||||
|
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
# ── Hall definitions (knowledge type) ──
|
||||||
|
# Order matters: first match wins
|
||||||
|
HALL_PATTERNS: list[tuple[str, list[str]]] = [
|
||||||
|
("warning", [
|
||||||
|
r"\bdon'?t\b", r"\bnever\b", r"\bavoid\b", r"\bdanger", r"\brisk",
|
||||||
|
r"\bcaution", r"\bwarning", r"\bcareful", r"\bdo not\b", r"\bforbid",
|
||||||
|
r"\bprohibit", r"\billegal", r"\bpenalt", r"\bfine\b",
|
||||||
|
]),
|
||||||
|
("decision", [
|
||||||
|
r"\bdecid", r"\bchose\b", r"\bchosen\b", r"\bapproved?\b", r"\brejected?\b",
|
||||||
|
r"\bagreed\b", r"\bselected?\b", r"\bpicked\b", r"\bwent with\b",
|
||||||
|
r"\bswitched? to\b", r"\bmigrat", r"\badopted?\b", r"\bresolved?\b",
|
||||||
|
]),
|
||||||
|
("procedure", [
|
||||||
|
r"\bstep\s*\d", r"\bfirst\b.*\bthen\b", r"\bprocess\b", r"\bworkflow\b",
|
||||||
|
r"\bhow to\b", r"\bprocedure\b", r"\binstructions?\b", r"\brecipe\b",
|
||||||
|
r"\brun\b.*\bcommand\b", r"\bexecute\b", r"\bsetup\b", r"\binstall\b",
|
||||||
|
]),
|
||||||
|
("event", [
|
||||||
|
r"\bhappened\b", r"\boccurred\b", r"\bmeeting\b", r"\bcall\b",
|
||||||
|
r"\bincident\b", r"\boutage\b", r"\breleased?\b", r"\blaunched?\b",
|
||||||
|
r"\bdeployed?\b", r"\bstarted?\b", r"\bfinished?\b", r"\bcompleted?\b",
|
||||||
|
r"\bon \d{4}-\d{2}-\d{2}\b", r"\byesterday\b", r"\blast week\b",
|
||||||
|
]),
|
||||||
|
("preference", [
|
||||||
|
r"\bprefer", r"\blike[sd]?\b", r"\bfavorite\b", r"\bwant[sed]?\b",
|
||||||
|
r"\brather\b", r"\bstyle\b", r"\btaste\b", r"\bchoice\b",
|
||||||
|
]),
|
||||||
|
("discovery", [
|
||||||
|
r"\bfound\b", r"\bdiscover", r"\blearned\b", r"\brealized?\b",
|
||||||
|
r"\bturns out\b", r"\bnoticed\b", r"\binsight\b", r"\bresearch",
|
||||||
|
r"\banalysis\b", r"\bbenchmark",
|
||||||
|
]),
|
||||||
|
# Default: "fact" — captured by no-match fallback
|
||||||
|
]
|
||||||
|
|
||||||
|
# ── Room definitions (topic) ──
|
||||||
|
ROOM_PATTERNS: list[tuple[str, list[str]]] = [
|
||||||
|
("auth", [
|
||||||
|
r"\bauth", r"\blogin", r"\bjwt\b", r"\btoken", r"\bsession",
|
||||||
|
r"\bpassword", r"\bcredential", r"\boauth", r"\bsso\b", r"\bsign[- ]?in\b",
|
||||||
|
]),
|
||||||
|
("pipeline", [
|
||||||
|
r"\bpipeline", r"\bci/?cd\b", r"\bbuild\b", r"\bdeploy", r"\bgithub action",
|
||||||
|
r"\bjenkins", r"\bcircleci", r"\bworkflow\b.*\bautomat",
|
||||||
|
]),
|
||||||
|
("infrastructure", [
|
||||||
|
r"\bserver", r"\bnginx\b", r"\bpm2\b", r"\bdocker", r"\bk8s\b",
|
||||||
|
r"\bkubernet", r"\binfra", r"\baws\b", r"\bgcp\b", r"\bazure\b",
|
||||||
|
r"\bvps\b", r"\bload balanc", r"\bdns\b", r"\bssl\b", r"\bcert\b",
|
||||||
|
]),
|
||||||
|
("deployment", [
|
||||||
|
r"\bdeploy", r"\brelease\b", r"\brollback", r"\bhotfix",
|
||||||
|
r"\bstaging\b", r"\bproduction\b", r"\bblue[- ]?green",
|
||||||
|
]),
|
||||||
|
("schema", [
|
||||||
|
r"\bschema", r"\bmigrat", r"\bcolumn\b", r"\btable\b", r"\bindex\b",
|
||||||
|
r"\bpostgres", r"\bdatabase\b", r"\bsql\b", r"\bquery\b",
|
||||||
|
r"\balembic", r"\bforeign key",
|
||||||
|
]),
|
||||||
|
("api", [
|
||||||
|
r"\bapi\b", r"\bendpoint", r"\broute\b", r"\brest\b", r"\bgraphql",
|
||||||
|
r"\bhttp\b", r"\brequest\b", r"\bresponse\b", r"\bwebhook",
|
||||||
|
]),
|
||||||
|
("ui", [
|
||||||
|
r"\bui\b", r"\bfrontend", r"\breact\b", r"\bcomponent", r"\bcss\b",
|
||||||
|
r"\bbutton\b", r"\bmodal\b", r"\bpanel\b", r"\blayout\b",
|
||||||
|
r"\bdesign\b", r"\bux\b", r"\bstyle\b",
|
||||||
|
]),
|
||||||
|
("tax", [
|
||||||
|
r"\btax", r"\bfiling\b", r"\bird\b", r"\bmpf\b", r"\bprofit",
|
||||||
|
r"\bdeduction", r"\bexempt", r"\btaxable\b", r"\breturn\b.*\btax",
|
||||||
|
]),
|
||||||
|
("hr", [
|
||||||
|
r"\bhr\b", r"\bhuman resource", r"\bsalary", r"\bleave\b",
|
||||||
|
r"\bemployee", r"\bhiring", r"\brecruitment", r"\bonboarding",
|
||||||
|
]),
|
||||||
|
("legal", [
|
||||||
|
r"\blegal", r"\bcontract", r"\bclause\b", r"\blawyer",
|
||||||
|
r"\blitigation", r"\bcourt\b", r"\bregulat", r"\blicense\b",
|
||||||
|
]),
|
||||||
|
("compliance", [
|
||||||
|
r"\bcompliance", r"\baudit\b", r"\bregulat", r"\bkyc\b",
|
||||||
|
r"\baml\b", r"\bgdpr\b", r"\bprivacy\b", r"\bdata protect",
|
||||||
|
]),
|
||||||
|
("monitoring", [
|
||||||
|
r"\bmonitor", r"\balert", r"\bmetric", r"\blog[sg]?\b",
|
||||||
|
r"\bgrafana\b", r"\bprometheus\b", r"\bobservab", r"\btrac[ei]",
|
||||||
|
]),
|
||||||
|
("agent", [
|
||||||
|
r"\bagent\b", r"\bllm\b", r"\bai\b", r"\bmemory\b.*\bsystem\b",
|
||||||
|
r"\bhindsight\b", r"\bprompt\b", r"\btool\b.*\bcall\b",
|
||||||
|
r"\bpes\b", r"\bspell\b", r"\borchestrat",
|
||||||
|
]),
|
||||||
|
# Default: "general" — captured by no-match fallback
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
|
def _match_patterns(text: str, patterns: list[tuple[str, list[str]]]) -> str | None:
|
||||||
|
"""Match text against pattern list, return first matching category or None."""
|
||||||
|
text_lower = text.lower()
|
||||||
|
for category, regexes in patterns:
|
||||||
|
for pattern in regexes:
|
||||||
|
if re.search(pattern, text_lower):
|
||||||
|
return category
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def classify_room_hall(
|
||||||
|
fact_text: str,
|
||||||
|
context: str | None = None,
|
||||||
|
existing_room: str | None = None,
|
||||||
|
existing_hall: str | None = None,
|
||||||
|
) -> tuple[str, str]:
|
||||||
|
"""
|
||||||
|
Classify a fact into room (topic) and hall (knowledge type).
|
||||||
|
|
||||||
|
If existing values are provided, they are kept as-is.
|
||||||
|
Otherwise, keyword heuristics determine the classification.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
fact_text: The fact text to classify
|
||||||
|
context: Optional context string for additional signals
|
||||||
|
existing_room: Pre-assigned room (kept if not None)
|
||||||
|
existing_hall: Pre-assigned hall (kept if not None)
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
Tuple of (room, hall)
|
||||||
|
"""
|
||||||
|
combined = f"{fact_text} {context or ''}"
|
||||||
|
|
||||||
|
room = existing_room
|
||||||
|
if room is None:
|
||||||
|
room = _match_patterns(combined, ROOM_PATTERNS) or "general"
|
||||||
|
|
||||||
|
hall = existing_hall
|
||||||
|
if hall is None:
|
||||||
|
hall = _match_patterns(combined, HALL_PATTERNS) or "fact"
|
||||||
|
|
||||||
|
return room, hall
|
||||||
|
|
||||||
|
|
||||||
|
def classify_facts_batch(facts: list) -> None:
|
||||||
|
"""
|
||||||
|
Classify room/hall for a batch of ProcessedFact or ExtractedFact objects.
|
||||||
|
|
||||||
|
Modifies facts in-place. Only sets room/hall if not already set.
|
||||||
|
"""
|
||||||
|
for fact in facts:
|
||||||
|
fact_text = getattr(fact, 'fact_text', '') or getattr(fact, 'text', '') or ''
|
||||||
|
context = getattr(fact, 'context', None)
|
||||||
|
room, hall = classify_room_hall(
|
||||||
|
fact_text,
|
||||||
|
context,
|
||||||
|
existing_room=getattr(fact, 'room', None),
|
||||||
|
existing_hall=getattr(fact, 'hall', None),
|
||||||
|
)
|
||||||
|
fact.room = room
|
||||||
|
fact.hall = hall
|
||||||
|
|
@ -28,6 +28,9 @@ class RetainContentDict(TypedDict, total=False):
|
||||||
update_mode: How to handle existing documents with the same document_id (optional).
|
update_mode: How to handle existing documents with the same document_id (optional).
|
||||||
"replace" (default) deletes old data and reprocesses. "append" concatenates
|
"replace" (default) deletes old data and reprocesses. "append" concatenates
|
||||||
new content to the existing document and reprocesses.
|
new content to the existing document and reprocesses.
|
||||||
|
room: Topic classification for hierarchical filtering (ADR-145)
|
||||||
|
hall: Knowledge type classification (ADR-145)
|
||||||
|
layer: Memory layer L0-L3 (ADR-145)
|
||||||
"""
|
"""
|
||||||
|
|
||||||
content: str # Required
|
content: str # Required
|
||||||
|
|
@ -41,6 +44,9 @@ class RetainContentDict(TypedDict, total=False):
|
||||||
Literal["per_tag", "combined", "all_combinations"] | list[list[str]]
|
Literal["per_tag", "combined", "all_combinations"] | list[list[str]]
|
||||||
) # Observation scopes for consolidation
|
) # Observation scopes for consolidation
|
||||||
update_mode: Literal["replace", "append"]
|
update_mode: Literal["replace", "append"]
|
||||||
|
room: str # Topic classification (ADR-145 RCLL)
|
||||||
|
hall: str # Knowledge type classification (ADR-145 RCLL)
|
||||||
|
layer: str # Memory layer: L0, L1, L2 (default), L3 (ADR-145)
|
||||||
|
|
||||||
|
|
||||||
@dataclass
|
@dataclass
|
||||||
|
|
@ -60,6 +66,9 @@ class RetainContent:
|
||||||
observation_scopes: Literal["per_tag", "combined", "all_combinations"] | list[list[str]] | None = (
|
observation_scopes: Literal["per_tag", "combined", "all_combinations"] | list[list[str]] | None = (
|
||||||
None # Observation scopes
|
None # Observation scopes
|
||||||
)
|
)
|
||||||
|
room: str | None = None # Topic classification (ADR-145 RCLL)
|
||||||
|
hall: str | None = None # Knowledge type classification (ADR-145 RCLL)
|
||||||
|
layer: str = "L2" # Memory layer: L0, L1, L2 (default), L3 (ADR-145)
|
||||||
|
|
||||||
|
|
||||||
@dataclass
|
@dataclass
|
||||||
|
|
@ -128,6 +137,9 @@ class ExtractedFact:
|
||||||
observation_scopes: Literal["per_tag", "combined", "all_combinations"] | list[list[str]] | None = (
|
observation_scopes: Literal["per_tag", "combined", "all_combinations"] | list[list[str]] | None = (
|
||||||
None # Observation scopes
|
None # Observation scopes
|
||||||
)
|
)
|
||||||
|
room: str | None = None # Topic classification (ADR-145 RCLL)
|
||||||
|
hall: str | None = None # Knowledge type classification (ADR-145 RCLL)
|
||||||
|
layer: str = "L2" # Memory layer: L0, L1, L2 (default), L3 (ADR-145)
|
||||||
|
|
||||||
|
|
||||||
@dataclass
|
@dataclass
|
||||||
|
|
@ -179,6 +191,11 @@ class ProcessedFact:
|
||||||
# Observation scopes for consolidation
|
# Observation scopes for consolidation
|
||||||
observation_scopes: Literal["per_tag", "combined", "all_combinations"] | list[list[str]] | None = None
|
observation_scopes: Literal["per_tag", "combined", "all_combinations"] | list[list[str]] | None = None
|
||||||
|
|
||||||
|
# ADR-145 RCLL: hierarchical classification
|
||||||
|
room: str | None = None # Topic classification
|
||||||
|
hall: str | None = None # Knowledge type classification
|
||||||
|
layer: str = "L2" # Memory layer: L0, L1, L2 (default), L3
|
||||||
|
|
||||||
@property
|
@property
|
||||||
def is_duplicate(self) -> bool:
|
def is_duplicate(self) -> bool:
|
||||||
"""Check if this fact was marked as a duplicate."""
|
"""Check if this fact was marked as a duplicate."""
|
||||||
|
|
@ -222,6 +239,9 @@ class ProcessedFact:
|
||||||
content_index=extracted_fact.content_index,
|
content_index=extracted_fact.content_index,
|
||||||
tags=extracted_fact.tags,
|
tags=extracted_fact.tags,
|
||||||
observation_scopes=extracted_fact.observation_scopes,
|
observation_scopes=extracted_fact.observation_scopes,
|
||||||
|
room=extracted_fact.room,
|
||||||
|
hall=extracted_fact.hall,
|
||||||
|
layer=extracted_fact.layer,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -64,7 +64,7 @@ async def _find_semantic_seeds(
|
||||||
rows = await conn.fetch(
|
rows = await conn.fetch(
|
||||||
f"""
|
f"""
|
||||||
SELECT id, text, context, event_date, occurred_start, occurred_end,
|
SELECT id, text, context, event_date, occurred_start, occurred_end,
|
||||||
mentioned_at, fact_type, document_id, chunk_id, tags, proof_count,
|
mentioned_at, fact_type, document_id, chunk_id, tags, proof_count, room, hall, layer,
|
||||||
1 - (embedding <=> $1::vector) AS similarity
|
1 - (embedding <=> $1::vector) AS similarity
|
||||||
FROM {fq_table("memory_units")}
|
FROM {fq_table("memory_units")}
|
||||||
WHERE bank_id = $2
|
WHERE bank_id = $2
|
||||||
|
|
@ -289,7 +289,7 @@ class LinkExpansionRetriever(GraphRetriever):
|
||||||
entity_expanded AS (
|
entity_expanded AS (
|
||||||
SELECT mu.id, mu.text, mu.context, mu.event_date, mu.occurred_start,
|
SELECT mu.id, mu.text, mu.context, mu.event_date, mu.occurred_start,
|
||||||
mu.occurred_end, mu.mentioned_at,
|
mu.occurred_end, mu.mentioned_at,
|
||||||
mu.fact_type, mu.document_id, mu.chunk_id, mu.tags, mu.proof_count,
|
mu.fact_type, mu.document_id, mu.chunk_id, mu.tags, mu.proof_count, mu.room, mu.hall, mu.layer,
|
||||||
COUNT(DISTINCT se.entity_id)::float AS score,
|
COUNT(DISTINCT se.entity_id)::float AS score,
|
||||||
'entity'::text AS source
|
'entity'::text AS source
|
||||||
FROM seed_entities se
|
FROM seed_entities se
|
||||||
|
|
@ -316,14 +316,14 @@ class LinkExpansionRetriever(GraphRetriever):
|
||||||
SELECT
|
SELECT
|
||||||
id, text, context, event_date, occurred_start,
|
id, text, context, event_date, occurred_start,
|
||||||
occurred_end, mentioned_at,
|
occurred_end, mentioned_at,
|
||||||
fact_type, document_id, chunk_id, tags, proof_count,
|
fact_type, document_id, chunk_id, tags, proof_count, room, hall, layer,
|
||||||
MAX(weight) AS score,
|
MAX(weight) AS score,
|
||||||
'semantic'::text AS source
|
'semantic'::text AS source
|
||||||
FROM (
|
FROM (
|
||||||
SELECT
|
SELECT
|
||||||
mu.id, mu.text, mu.context, mu.event_date, mu.occurred_start,
|
mu.id, mu.text, mu.context, mu.event_date, mu.occurred_start,
|
||||||
mu.occurred_end, mu.mentioned_at,
|
mu.occurred_end, mu.mentioned_at,
|
||||||
mu.fact_type, mu.document_id, mu.chunk_id, mu.tags, mu.proof_count,
|
mu.fact_type, mu.document_id, mu.chunk_id, mu.tags, mu.proof_count, mu.room, mu.hall, mu.layer,
|
||||||
ml.weight
|
ml.weight
|
||||||
FROM {ml} ml
|
FROM {ml} ml
|
||||||
JOIN {mu} mu ON mu.id = ml.to_unit_id
|
JOIN {mu} mu ON mu.id = ml.to_unit_id
|
||||||
|
|
@ -335,7 +335,7 @@ class LinkExpansionRetriever(GraphRetriever):
|
||||||
SELECT
|
SELECT
|
||||||
mu.id, mu.text, mu.context, mu.event_date, mu.occurred_start,
|
mu.id, mu.text, mu.context, mu.event_date, mu.occurred_start,
|
||||||
mu.occurred_end, mu.mentioned_at,
|
mu.occurred_end, mu.mentioned_at,
|
||||||
mu.fact_type, mu.document_id, mu.chunk_id, mu.tags, mu.proof_count,
|
mu.fact_type, mu.document_id, mu.chunk_id, mu.tags, mu.proof_count, mu.room, mu.hall, mu.layer,
|
||||||
ml.weight
|
ml.weight
|
||||||
FROM {ml} ml
|
FROM {ml} ml
|
||||||
JOIN {mu} mu ON mu.id = ml.from_unit_id
|
JOIN {mu} mu ON mu.id = ml.from_unit_id
|
||||||
|
|
@ -346,7 +346,7 @@ class LinkExpansionRetriever(GraphRetriever):
|
||||||
) sem_raw
|
) sem_raw
|
||||||
GROUP BY id, text, context, event_date, occurred_start,
|
GROUP BY id, text, context, event_date, occurred_start,
|
||||||
occurred_end, mentioned_at,
|
occurred_end, mentioned_at,
|
||||||
fact_type, document_id, chunk_id, tags, proof_count
|
fact_type, document_id, chunk_id, tags, proof_count, room, hall, layer
|
||||||
ORDER BY score DESC
|
ORDER BY score DESC
|
||||||
LIMIT $3
|
LIMIT $3
|
||||||
),
|
),
|
||||||
|
|
@ -357,7 +357,7 @@ class LinkExpansionRetriever(GraphRetriever):
|
||||||
SELECT DISTINCT ON (mu.id)
|
SELECT DISTINCT ON (mu.id)
|
||||||
mu.id, mu.text, mu.context, mu.event_date, mu.occurred_start,
|
mu.id, mu.text, mu.context, mu.event_date, mu.occurred_start,
|
||||||
mu.occurred_end, mu.mentioned_at,
|
mu.occurred_end, mu.mentioned_at,
|
||||||
mu.fact_type, mu.document_id, mu.chunk_id, mu.tags, mu.proof_count,
|
mu.fact_type, mu.document_id, mu.chunk_id, mu.tags, mu.proof_count, mu.room, mu.hall, mu.layer,
|
||||||
ml.weight AS score,
|
ml.weight AS score,
|
||||||
'causal'::text AS source
|
'causal'::text AS source
|
||||||
FROM {ml} ml
|
FROM {ml} ml
|
||||||
|
|
@ -480,7 +480,7 @@ class LinkExpansionRetriever(GraphRetriever):
|
||||||
SELECT
|
SELECT
|
||||||
mu.id, mu.text, mu.context, mu.event_date, mu.occurred_start,
|
mu.id, mu.text, mu.context, mu.event_date, mu.occurred_start,
|
||||||
mu.occurred_end, mu.mentioned_at,
|
mu.occurred_end, mu.mentioned_at,
|
||||||
mu.fact_type, mu.document_id, mu.chunk_id, mu.tags, mu.proof_count,
|
mu.fact_type, mu.document_id, mu.chunk_id, mu.tags, mu.proof_count, mu.room, mu.hall, mu.layer,
|
||||||
(SELECT COUNT(DISTINCT s) FROM unnest(mu.source_memory_ids) s WHERE s = ANY(ca.source_ids))::float AS score
|
(SELECT COUNT(DISTINCT s) FROM unnest(mu.source_memory_ids) s WHERE s = ANY(ca.source_ids))::float AS score
|
||||||
FROM {fq_table("memory_units")} mu, connected_array ca
|
FROM {fq_table("memory_units")} mu, connected_array ca
|
||||||
WHERE mu.fact_type = 'observation'
|
WHERE mu.fact_type = 'observation'
|
||||||
|
|
@ -504,13 +504,13 @@ class LinkExpansionRetriever(GraphRetriever):
|
||||||
SELECT
|
SELECT
|
||||||
id, text, context, event_date, occurred_start,
|
id, text, context, event_date, occurred_start,
|
||||||
occurred_end, mentioned_at,
|
occurred_end, mentioned_at,
|
||||||
fact_type, document_id, chunk_id, tags, proof_count,
|
fact_type, document_id, chunk_id, tags, proof_count, room, hall, layer,
|
||||||
MAX(weight) AS score,
|
MAX(weight) AS score,
|
||||||
'semantic'::text AS source
|
'semantic'::text AS source
|
||||||
FROM (
|
FROM (
|
||||||
SELECT mu.id, mu.text, mu.context, mu.event_date, mu.occurred_start,
|
SELECT mu.id, mu.text, mu.context, mu.event_date, mu.occurred_start,
|
||||||
mu.occurred_end, mu.mentioned_at, mu.fact_type, mu.document_id,
|
mu.occurred_end, mu.mentioned_at, mu.fact_type, mu.document_id,
|
||||||
mu.chunk_id, mu.tags, mu.proof_count, ml.weight
|
mu.chunk_id, mu.tags, mu.proof_count, mu.room, mu.hall, mu.layer, ml.weight
|
||||||
FROM {ml} ml JOIN {mu} mu ON mu.id = ml.to_unit_id
|
FROM {ml} ml JOIN {mu} mu ON mu.id = ml.to_unit_id
|
||||||
WHERE ml.from_unit_id = ANY($1::uuid[])
|
WHERE ml.from_unit_id = ANY($1::uuid[])
|
||||||
AND ml.link_type = 'semantic' AND mu.fact_type = 'observation'
|
AND ml.link_type = 'semantic' AND mu.fact_type = 'observation'
|
||||||
|
|
@ -518,21 +518,21 @@ class LinkExpansionRetriever(GraphRetriever):
|
||||||
UNION ALL
|
UNION ALL
|
||||||
SELECT mu.id, mu.text, mu.context, mu.event_date, mu.occurred_start,
|
SELECT mu.id, mu.text, mu.context, mu.event_date, mu.occurred_start,
|
||||||
mu.occurred_end, mu.mentioned_at, mu.fact_type, mu.document_id,
|
mu.occurred_end, mu.mentioned_at, mu.fact_type, mu.document_id,
|
||||||
mu.chunk_id, mu.tags, mu.proof_count, ml.weight
|
mu.chunk_id, mu.tags, mu.proof_count, mu.room, mu.hall, mu.layer, ml.weight
|
||||||
FROM {ml} ml JOIN {mu} mu ON mu.id = ml.from_unit_id
|
FROM {ml} ml JOIN {mu} mu ON mu.id = ml.from_unit_id
|
||||||
WHERE ml.to_unit_id = ANY($1::uuid[])
|
WHERE ml.to_unit_id = ANY($1::uuid[])
|
||||||
AND ml.link_type = 'semantic' AND mu.fact_type = 'observation'
|
AND ml.link_type = 'semantic' AND mu.fact_type = 'observation'
|
||||||
AND mu.id != ALL($1::uuid[])
|
AND mu.id != ALL($1::uuid[])
|
||||||
) sem_raw
|
) sem_raw
|
||||||
GROUP BY id, text, context, event_date, occurred_start, occurred_end,
|
GROUP BY id, text, context, event_date, occurred_start, occurred_end,
|
||||||
mentioned_at, fact_type, document_id, chunk_id, tags, proof_count
|
mentioned_at, fact_type, document_id, chunk_id, tags, proof_count, room, hall, layer
|
||||||
ORDER BY score DESC LIMIT $2
|
ORDER BY score DESC LIMIT $2
|
||||||
),
|
),
|
||||||
causal_expanded AS (
|
causal_expanded AS (
|
||||||
SELECT DISTINCT ON (mu.id)
|
SELECT DISTINCT ON (mu.id)
|
||||||
mu.id, mu.text, mu.context, mu.event_date, mu.occurred_start,
|
mu.id, mu.text, mu.context, mu.event_date, mu.occurred_start,
|
||||||
mu.occurred_end, mu.mentioned_at, mu.fact_type, mu.document_id,
|
mu.occurred_end, mu.mentioned_at, mu.fact_type, mu.document_id,
|
||||||
mu.chunk_id, mu.tags, mu.proof_count, ml.weight AS score, 'causal'::text AS source
|
mu.chunk_id, mu.tags, mu.proof_count, mu.room, mu.hall, mu.layer, ml.weight AS score, 'causal'::text AS source
|
||||||
FROM {ml} ml JOIN {mu} mu ON ml.to_unit_id = mu.id
|
FROM {ml} ml JOIN {mu} mu ON ml.to_unit_id = mu.id
|
||||||
WHERE ml.from_unit_id = ANY($1::uuid[])
|
WHERE ml.from_unit_id = ANY($1::uuid[])
|
||||||
AND ml.link_type IN ('causes', 'caused_by', 'enables', 'prevents')
|
AND ml.link_type IN ('causes', 'caused_by', 'enables', 'prevents')
|
||||||
|
|
|
||||||
|
|
@ -98,6 +98,9 @@ async def retrieve_semantic_bm25_combined(
|
||||||
tags: list[str] | None = None,
|
tags: list[str] | None = None,
|
||||||
tags_match: TagsMatch = "any",
|
tags_match: TagsMatch = "any",
|
||||||
tag_groups: list[TagGroup] | None = None,
|
tag_groups: list[TagGroup] | None = None,
|
||||||
|
room: list[str] | None = None,
|
||||||
|
hall: list[str] | None = None,
|
||||||
|
max_layer: str = "L3",
|
||||||
) -> dict[str, tuple[list[RetrievalResult], list[RetrievalResult]]]:
|
) -> dict[str, tuple[list[RetrievalResult], list[RetrievalResult]]]:
|
||||||
"""
|
"""
|
||||||
Combined semantic + BM25 retrieval for multiple fact types in a single query.
|
Combined semantic + BM25 retrieval for multiple fact types in a single query.
|
||||||
|
|
@ -141,7 +144,7 @@ async def retrieve_semantic_bm25_combined(
|
||||||
|
|
||||||
cols = (
|
cols = (
|
||||||
"id, text, context, event_date, occurred_start, occurred_end, mentioned_at, "
|
"id, text, context, event_date, occurred_start, occurred_end, mentioned_at, "
|
||||||
"fact_type, document_id, chunk_id, tags, metadata, proof_count"
|
"fact_type, document_id, chunk_id, tags, metadata, proof_count, room, hall, layer"
|
||||||
)
|
)
|
||||||
table = fq_table("memory_units")
|
table = fq_table("memory_units")
|
||||||
|
|
||||||
|
|
@ -161,7 +164,31 @@ async def retrieve_semantic_bm25_combined(
|
||||||
|
|
||||||
# tag_groups params start immediately after the tags param slot
|
# tag_groups params start immediately after the tags param slot
|
||||||
tag_groups_param_start = tags_param_idx + (1 if tags else 0)
|
tag_groups_param_start = tags_param_idx + (1 if tags else 0)
|
||||||
groups_clause, groups_params, _ = build_tag_groups_where_clause(tag_groups, tag_groups_param_start)
|
groups_clause, groups_params, next_param = build_tag_groups_where_clause(tag_groups, tag_groups_param_start)
|
||||||
|
|
||||||
|
# ADR-145: Room/Hall filtering — applied BEFORE semantic search for accuracy boost
|
||||||
|
room_clause = ""
|
||||||
|
room_params = []
|
||||||
|
if room:
|
||||||
|
room_clause = f"AND room = ANY(${next_param}::text[])"
|
||||||
|
room_params = [room]
|
||||||
|
next_param += 1
|
||||||
|
hall_clause = ""
|
||||||
|
hall_params = []
|
||||||
|
if hall:
|
||||||
|
hall_clause = f"AND hall = ANY(${next_param}::text[])"
|
||||||
|
hall_params = [hall]
|
||||||
|
next_param += 1
|
||||||
|
|
||||||
|
# ADR-145: Layer filtering — cascade from L0 to max_layer
|
||||||
|
layer_order = {"L0": 0, "L1": 1, "L2": 2, "L3": 3}
|
||||||
|
layer_clause = ""
|
||||||
|
layer_params = []
|
||||||
|
if max_layer and max_layer != "L3":
|
||||||
|
allowed_layers = [l for l, v in layer_order.items() if v <= layer_order.get(max_layer, 3)]
|
||||||
|
layer_clause = f"AND COALESCE(layer, 'L2') = ANY(${next_param}::text[])"
|
||||||
|
layer_params = [allowed_layers]
|
||||||
|
next_param += 1
|
||||||
|
|
||||||
# --- Semantic UNION ALL arms (one per fact_type) ---
|
# --- Semantic UNION ALL arms (one per fact_type) ---
|
||||||
# Each arm has its own ORDER BY embedding <=> $1 LIMIT {hnsw_fetch}, which
|
# Each arm has its own ORDER BY embedding <=> $1 LIMIT {hnsw_fetch}, which
|
||||||
|
|
@ -266,6 +293,9 @@ async def retrieve_temporal_combined(
|
||||||
tags: list[str] | None = None,
|
tags: list[str] | None = None,
|
||||||
tags_match: TagsMatch = "any",
|
tags_match: TagsMatch = "any",
|
||||||
tag_groups: list[TagGroup] | None = None,
|
tag_groups: list[TagGroup] | None = None,
|
||||||
|
room: list[str] | None = None,
|
||||||
|
hall: list[str] | None = None,
|
||||||
|
max_layer: str = "L3",
|
||||||
) -> dict[str, list[RetrievalResult]]:
|
) -> dict[str, list[RetrievalResult]]:
|
||||||
"""
|
"""
|
||||||
Temporal retrieval for multiple fact types in a single query.
|
Temporal retrieval for multiple fact types in a single query.
|
||||||
|
|
@ -298,7 +328,32 @@ async def retrieve_temporal_combined(
|
||||||
# Entry point query: fixed params are $1-$6, tags at $7
|
# Entry point query: fixed params are $1-$6, tags at $7
|
||||||
tags_clause = build_tags_where_clause_simple(tags, 7, match=tags_match)
|
tags_clause = build_tags_where_clause_simple(tags, 7, match=tags_match)
|
||||||
tag_groups_param_start = 7 + (1 if tags else 0)
|
tag_groups_param_start = 7 + (1 if tags else 0)
|
||||||
groups_clause, groups_params, _ = build_tag_groups_where_clause(tag_groups, tag_groups_param_start)
|
groups_clause, groups_params, next_param = build_tag_groups_where_clause(tag_groups, tag_groups_param_start)
|
||||||
|
|
||||||
|
# ADR-145: Room/Hall filtering
|
||||||
|
room_clause = ""
|
||||||
|
room_params_list = []
|
||||||
|
if room:
|
||||||
|
room_clause = f"AND room = ANY(${next_param}::text[])"
|
||||||
|
room_params_list = [room]
|
||||||
|
next_param += 1
|
||||||
|
hall_clause = ""
|
||||||
|
hall_params_list = []
|
||||||
|
if hall:
|
||||||
|
hall_clause = f"AND hall = ANY(${next_param}::text[])"
|
||||||
|
hall_params_list = [hall]
|
||||||
|
next_param += 1
|
||||||
|
|
||||||
|
# ADR-145: Layer filtering — cascade from L0 to max_layer
|
||||||
|
layer_order = {"L0": 0, "L1": 1, "L2": 2, "L3": 3}
|
||||||
|
layer_clause = ""
|
||||||
|
layer_params_list = []
|
||||||
|
if max_layer and max_layer != "L3":
|
||||||
|
allowed_layers = [l for l, v in layer_order.items() if v <= layer_order.get(max_layer, 3)]
|
||||||
|
layer_clause = f"AND COALESCE(layer, 'L2') = ANY(${next_param}::text[])"
|
||||||
|
layer_params_list = [allowed_layers]
|
||||||
|
next_param += 1
|
||||||
|
|
||||||
params: list = [query_emb_str, bank_id, fact_types, start_date, end_date, semantic_threshold]
|
params: list = [query_emb_str, bank_id, fact_types, start_date, end_date, semantic_threshold]
|
||||||
if tags:
|
if tags:
|
||||||
params.append(tags)
|
params.append(tags)
|
||||||
|
|
@ -336,7 +391,7 @@ async def retrieve_temporal_combined(
|
||||||
{groups_clause}
|
{groups_clause}
|
||||||
),
|
),
|
||||||
sim_ranked AS (
|
sim_ranked AS (
|
||||||
SELECT mu.id, mu.text, mu.context, mu.event_date, mu.occurred_start, mu.occurred_end, mu.mentioned_at, mu.fact_type, mu.proof_count, mu.document_id, mu.chunk_id, mu.tags, mu.metadata,
|
SELECT mu.id, mu.text, mu.context, mu.event_date, mu.occurred_start, mu.occurred_end, mu.mentioned_at, mu.fact_type, mu.proof_count, mu.document_id, mu.chunk_id, mu.tags, mu.metadata, mu.room, mu.hall, mu.layer,
|
||||||
1 - (mu.embedding <=> $1::vector) AS similarity,
|
1 - (mu.embedding <=> $1::vector) AS similarity,
|
||||||
ROW_NUMBER() OVER (PARTITION BY mu.fact_type ORDER BY mu.embedding <=> $1::vector) AS sim_rn
|
ROW_NUMBER() OVER (PARTITION BY mu.fact_type ORDER BY mu.embedding <=> $1::vector) AS sim_rn
|
||||||
FROM date_ranked dr
|
FROM date_ranked dr
|
||||||
|
|
@ -344,7 +399,7 @@ async def retrieve_temporal_combined(
|
||||||
WHERE dr.rn <= 50
|
WHERE dr.rn <= 50
|
||||||
AND (1 - (mu.embedding <=> $1::vector)) >= $6
|
AND (1 - (mu.embedding <=> $1::vector)) >= $6
|
||||||
)
|
)
|
||||||
SELECT id, text, context, event_date, occurred_start, occurred_end, mentioned_at, fact_type, proof_count, document_id, chunk_id, tags, metadata, similarity
|
SELECT id, text, context, event_date, occurred_start, occurred_end, mentioned_at, fact_type, proof_count, document_id, chunk_id, tags, metadata, room, hall, layer, similarity
|
||||||
FROM sim_ranked
|
FROM sim_ranked
|
||||||
WHERE sim_rn <= 10
|
WHERE sim_rn <= 10
|
||||||
""",
|
""",
|
||||||
|
|
@ -442,7 +497,7 @@ async def retrieve_temporal_combined(
|
||||||
# bank_id on memory_units lets the planner use idx_memory_units_bank_fact_type.
|
# bank_id on memory_units lets the planner use idx_memory_units_bank_fact_type.
|
||||||
neighbors = await conn.fetch(
|
neighbors = await conn.fetch(
|
||||||
f"""
|
f"""
|
||||||
SELECT src.from_unit_id, mu.id, mu.text, mu.context, mu.event_date, mu.occurred_start, mu.occurred_end, mu.mentioned_at, mu.fact_type, mu.document_id, mu.chunk_id, mu.tags, mu.metadata,
|
SELECT src.from_unit_id, mu.id, mu.text, mu.context, mu.event_date, mu.occurred_start, mu.occurred_end, mu.mentioned_at, mu.fact_type, mu.document_id, mu.chunk_id, mu.tags, mu.metadata, mu.room, mu.hall,
|
||||||
l.weight, l.link_type,
|
l.weight, l.link_type,
|
||||||
1 - (mu.embedding <=> $1::vector) AS similarity
|
1 - (mu.embedding <=> $1::vector) AS similarity
|
||||||
FROM unnest($2::uuid[]) AS src(from_unit_id)
|
FROM unnest($2::uuid[]) AS src(from_unit_id)
|
||||||
|
|
@ -536,6 +591,9 @@ async def retrieve_all_fact_types_parallel(
|
||||||
tags: list[str] | None = None,
|
tags: list[str] | None = None,
|
||||||
tags_match: TagsMatch = "any",
|
tags_match: TagsMatch = "any",
|
||||||
tag_groups: list[TagGroup] | None = None,
|
tag_groups: list[TagGroup] | None = None,
|
||||||
|
room: list[str] | None = None,
|
||||||
|
hall: list[str] | None = None,
|
||||||
|
max_layer: str = "L3",
|
||||||
) -> MultiFactTypeRetrievalResult:
|
) -> MultiFactTypeRetrievalResult:
|
||||||
"""
|
"""
|
||||||
Optimized retrieval for multiple fact types using batched queries.
|
Optimized retrieval for multiple fact types using batched queries.
|
||||||
|
|
@ -594,6 +652,9 @@ async def retrieve_all_fact_types_parallel(
|
||||||
tags=tags,
|
tags=tags,
|
||||||
tags_match=tags_match,
|
tags_match=tags_match,
|
||||||
tag_groups=tag_groups,
|
tag_groups=tag_groups,
|
||||||
|
room=room,
|
||||||
|
hall=hall,
|
||||||
|
max_layer=max_layer,
|
||||||
)
|
)
|
||||||
semantic_bm25_time = time.time() - semantic_bm25_start
|
semantic_bm25_time = time.time() - semantic_bm25_start
|
||||||
|
|
||||||
|
|
@ -613,6 +674,9 @@ async def retrieve_all_fact_types_parallel(
|
||||||
tags=tags,
|
tags=tags,
|
||||||
tags_match=tags_match,
|
tags_match=tags_match,
|
||||||
tag_groups=tag_groups,
|
tag_groups=tag_groups,
|
||||||
|
room=room,
|
||||||
|
hall=hall,
|
||||||
|
max_layer=max_layer,
|
||||||
)
|
)
|
||||||
temporal_time = time.time() - temporal_start
|
temporal_time = time.time() - temporal_start
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -49,6 +49,9 @@ class RetrievalResult:
|
||||||
tags: list[str] | None = None # Visibility scope tags
|
tags: list[str] | None = None # Visibility scope tags
|
||||||
metadata: dict[str, str] | None = None # User-provided metadata
|
metadata: dict[str, str] | None = None # User-provided metadata
|
||||||
proof_count: int | None = None # Number of supporting memories (observations only)
|
proof_count: int | None = None # Number of supporting memories (observations only)
|
||||||
|
room: str | None = None # Topic classification (ADR-145 RCLL)
|
||||||
|
hall: str | None = None # Knowledge type classification (ADR-145 RCLL)
|
||||||
|
layer: str | None = None # Memory layer (ADR-145: L0-L3)
|
||||||
|
|
||||||
# Retrieval-specific scores (only one will be set depending on retrieval method)
|
# Retrieval-specific scores (only one will be set depending on retrieval method)
|
||||||
similarity: float | None = None # Semantic retrieval
|
similarity: float | None = None # Semantic retrieval
|
||||||
|
|
@ -74,6 +77,9 @@ class RetrievalResult:
|
||||||
tags=row.get("tags"),
|
tags=row.get("tags"),
|
||||||
metadata=row.get("metadata"),
|
metadata=row.get("metadata"),
|
||||||
proof_count=row.get("proof_count"),
|
proof_count=row.get("proof_count"),
|
||||||
|
room=row.get("room"),
|
||||||
|
hall=row.get("hall"),
|
||||||
|
layer=row.get("layer"),
|
||||||
similarity=row.get("similarity"),
|
similarity=row.get("similarity"),
|
||||||
bm25_score=row.get("bm25_score"),
|
bm25_score=row.get("bm25_score"),
|
||||||
activation=row.get("activation"),
|
activation=row.get("activation"),
|
||||||
|
|
@ -158,6 +164,9 @@ class ScoredResult:
|
||||||
"chunk_id": self.retrieval.chunk_id,
|
"chunk_id": self.retrieval.chunk_id,
|
||||||
"tags": self.retrieval.tags,
|
"tags": self.retrieval.tags,
|
||||||
"metadata": self.retrieval.metadata,
|
"metadata": self.retrieval.metadata,
|
||||||
|
"room": self.retrieval.room,
|
||||||
|
"hall": self.retrieval.hall,
|
||||||
|
"layer": self.retrieval.layer,
|
||||||
"semantic_similarity": self.retrieval.similarity,
|
"semantic_similarity": self.retrieval.similarity,
|
||||||
"bm25_score": self.retrieval.bm25_score,
|
"bm25_score": self.retrieval.bm25_score,
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -100,6 +100,9 @@ class MemoryUnit(Base):
|
||||||
unit_metadata: Mapped[dict] = mapped_column(
|
unit_metadata: Mapped[dict] = mapped_column(
|
||||||
"metadata", JSONB, server_default=sql_text("'{}'::jsonb")
|
"metadata", JSONB, server_default=sql_text("'{}'::jsonb")
|
||||||
) # User-defined metadata (str->str)
|
) # User-defined metadata (str->str)
|
||||||
|
room: Mapped[str | None] = mapped_column(Text) # Topic classification (ADR-145 RCLL)
|
||||||
|
hall: Mapped[str | None] = mapped_column(Text) # Knowledge type classification (ADR-145 RCLL)
|
||||||
|
layer: Mapped[str | None] = mapped_column(Text, server_default="L2") # Memory layer (ADR-145 L0-L3)
|
||||||
created_at: Mapped[datetime] = mapped_column(TIMESTAMP(timezone=True), server_default=func.now())
|
created_at: Mapped[datetime] = mapped_column(TIMESTAMP(timezone=True), server_default=func.now())
|
||||||
updated_at: Mapped[datetime] = mapped_column(TIMESTAMP(timezone=True), server_default=func.now())
|
updated_at: Mapped[datetime] = mapped_column(TIMESTAMP(timezone=True), server_default=func.now())
|
||||||
|
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue