From 5de793eec737b4961d14b0f11f7df1fab923dc05 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=B2=20Boschi?= Date: Fri, 13 Mar 2026 14:30:11 +0100 Subject: [PATCH] feat: compound tag filtering via tag_groups (#562) * feat: add compound tag filtering via tag_groups Adds tag_groups to RecallRequest and ReflectRequest to express arbitrary boolean tag predicates: leaf {tags, match}, and/or/not compounds. Top-level groups are AND-ed. Existing tags/tags_match unchanged. Examples: Step filter AND user scope: tag_groups: [{tags: ["step:5","step:8"], match: "any_strict"}, {tags: ["user:alice"], match: "all_strict"}] Exclusion: tag_groups: [{tags: ["user:alice"], match: "all_strict"}, {not: {tags: ["archived"], match: "any_strict"}}] - Recursive SQL builder (build_tag_groups_where_clause) threads through all 4 retrieval strategies (semantic/BM25, temporal, graph, MPFP) - Python-side filter (filter_results_by_tag_groups) for post-traversal - 22 new unit tests - OpenAPI spec + all clients regenerated (Rust, Python, TypeScript, Go) * fix: add tag_groups: None to Rust CLI struct initializers * fix: add tag_groups: None to Rust client test RecallRequest initializer * feat: reject tags+tag_groups together, add tag_groups integration tests - Add model_validator to RecallRequest and ReflectRequest that returns 422 when both `tags` and `tag_groups` are set (mutually exclusive) - Add 5 integration tests for tag_groups compound filtering: * validation: 422 when both fields are set * AND filter: two leaf groups (step scope AND user scope) * OR compound: user:alice OR user:bob * NOT compound: user:alice AND NOT archived * Nested: user:alice AND (step:5 OR step:8) * ci: trigger CI run --- hindsight-api-slim/hindsight_api/api/http.py | 28 +- .../hindsight_api/engine/memory_engine.py | 10 +- .../hindsight_api/engine/reflect/tools.py | 12 +- .../engine/search/graph_retrieval.py | 24 +- .../engine/search/link_expansion_retrieval.py | 14 +- .../engine/search/mpfp_retrieval.py | 24 +- .../hindsight_api/engine/search/retrieval.py | 31 +- .../hindsight_api/engine/search/tags.py | 220 ++++++- .../tests/test_tags_visibility.py | 547 +++++++++++++++++- hindsight-cli/src/commands/explore.rs | 2 + hindsight-cli/src/commands/memory.rs | 2 + hindsight-clients/go/api/openapi.yaml | 70 +++ hindsight-clients/go/model_not.go | 143 +++++ hindsight-clients/go/model_recall_request.go | 37 ++ .../model_recall_request_tag_groups_inner.go | 143 +++++ hindsight-clients/go/model_reflect_request.go | 37 ++ hindsight-clients/go/model_tag_group_and.go | 158 +++++ hindsight-clients/go/model_tag_group_leaf.go | 198 +++++++ hindsight-clients/go/model_tag_group_not.go | 158 +++++ hindsight-clients/go/model_tag_group_or.go | 158 +++++ .../python/.openapi-generator/FILES | 6 + .../python/hindsight_client_api/__init__.py | 6 + .../hindsight_client_api/models/__init__.py | 6 + .../hindsight_client_api/models/model_not.py | 166 ++++++ .../models/recall_request.py | 19 +- .../models/recall_request_tag_groups_inner.py | 166 ++++++ .../models/reflect_request.py | 19 +- .../models/tag_group_and.py | 97 ++++ .../models/tag_group_leaf.py | 99 ++++ .../models/tag_group_not.py | 93 +++ .../models/tag_group_or.py | 97 ++++ hindsight-clients/rust/src/lib.rs | 1 + .../typescript/generated/types.gen.ts | 68 +++ hindsight-docs/docs/developer/api/recall.mdx | 60 ++ hindsight-docs/static/openapi.json | 171 ++++++ 35 files changed, 3069 insertions(+), 21 deletions(-) create mode 100644 hindsight-clients/go/model_not.go create mode 100644 hindsight-clients/go/model_recall_request_tag_groups_inner.go create mode 100644 hindsight-clients/go/model_tag_group_and.go create mode 100644 hindsight-clients/go/model_tag_group_leaf.go create mode 100644 hindsight-clients/go/model_tag_group_not.go create mode 100644 hindsight-clients/go/model_tag_group_or.go create mode 100644 hindsight-clients/python/hindsight_client_api/models/model_not.py create mode 100644 hindsight-clients/python/hindsight_client_api/models/recall_request_tag_groups_inner.py create mode 100644 hindsight-clients/python/hindsight_client_api/models/tag_group_and.py create mode 100644 hindsight-clients/python/hindsight_client_api/models/tag_group_leaf.py create mode 100644 hindsight-clients/python/hindsight_client_api/models/tag_group_not.py create mode 100644 hindsight-clients/python/hindsight_client_api/models/tag_group_or.py diff --git a/hindsight-api-slim/hindsight_api/api/http.py b/hindsight-api-slim/hindsight_api/api/http.py index 401f88d7..f7a36137 100644 --- a/hindsight-api-slim/hindsight_api/api/http.py +++ b/hindsight-api-slim/hindsight_api/api/http.py @@ -34,7 +34,7 @@ def _parse_metadata(metadata: Any) -> dict[str, Any]: from typing import Callable -from pydantic import BaseModel, ConfigDict, Field, field_validator +from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator from hindsight_api import MemoryEngine @@ -73,7 +73,7 @@ def FieldWithDefault(default_factory: Callable, **kwargs) -> Any: from hindsight_api.config import get_config from hindsight_api.engine.memory_engine import Budget, _current_schema, _get_tiktoken_encoding, fq_table from hindsight_api.engine.response_models import VALID_RECALL_FACT_TYPES, MemoryFact, TokenUsage -from hindsight_api.engine.search.tags import TagsMatch +from hindsight_api.engine.search.tags import TagGroup, TagsMatch from hindsight_api.extensions import HttpExtension, OperationValidationError, load_extension from hindsight_api.metrics import create_metrics_collector, get_metrics_collector, initialize_metrics from hindsight_api.models import RequestContext @@ -163,6 +163,17 @@ class RecallRequest(BaseModel): description="How to match tags: 'any' (OR, includes untagged), 'all' (AND, includes untagged), " "'any_strict' (OR, excludes untagged), 'all_strict' (AND, excludes untagged).", ) + tag_groups: list[TagGroup] | None = Field( + default=None, + description="Compound tag filter using boolean groups. Groups in the list are AND-ed. " + "Each group is a leaf {tags, match} or compound {and: [...]}, {or: [...]}, {not: ...}.", + ) + + @model_validator(mode="after") + def validate_tags_exclusive(self) -> "RecallRequest": + if self.tags is not None and self.tag_groups is not None: + raise ValueError("'tags' and 'tag_groups' are mutually exclusive. Use 'tag_groups' for compound filtering.") + return self class RecallResult(BaseModel): @@ -639,6 +650,17 @@ class ReflectRequest(BaseModel): description="How to match tags: 'any' (OR, includes untagged), 'all' (AND, includes untagged), " "'any_strict' (OR, excludes untagged), 'all_strict' (AND, excludes untagged).", ) + tag_groups: list[TagGroup] | None = Field( + default=None, + description="Compound tag filter using boolean groups. Groups in the list are AND-ed. " + "Each group is a leaf {tags, match} or compound {and: [...]}, {or: [...]}, {not: ...}.", + ) + + @model_validator(mode="after") + def validate_tags_exclusive(self) -> "ReflectRequest": + if self.tags is not None and self.tag_groups is not None: + raise ValueError("'tags' and 'tag_groups' are mutually exclusive. Use 'tag_groups' for compound filtering.") + return self class ReflectFact(BaseModel): @@ -2324,6 +2346,7 @@ def _register_routes(app: FastAPI): request_context=request_context, tags=request.tags, tags_match=request.tags_match, + tag_groups=request.tag_groups, ) # Convert core MemoryFact objects to API RecallResult objects (excluding internal metrics) @@ -2459,6 +2482,7 @@ def _register_routes(app: FastAPI): request_context=request_context, tags=request.tags, tags_match=request.tags_match, + tag_groups=request.tag_groups, ) # Build based_on (memories + mental_models + directives) if facts are requested diff --git a/hindsight-api-slim/hindsight_api/engine/memory_engine.py b/hindsight-api-slim/hindsight_api/engine/memory_engine.py index 142a8432..d8ae44f7 100644 --- a/hindsight-api-slim/hindsight_api/engine/memory_engine.py +++ b/hindsight-api-slim/hindsight_api/engine/memory_engine.py @@ -184,7 +184,7 @@ from .retain import bank_utils, embedding_utils from .retain.types import RetainContentDict from .search import think_utils from .search.reranking import CrossEncoderReranker, apply_combined_scoring -from .search.tags import TagsMatch, build_tags_where_clause +from .search.tags import TagGroup, TagsMatch, build_tags_where_clause from .task_backend import BrokerTaskBackend, SyncTaskBackend, TaskBackend @@ -2300,6 +2300,7 @@ class MemoryEngine(MemoryEngineInterface): request_context: "RequestContext", tags: list[str] | None = None, tags_match: TagsMatch = "any", + tag_groups: list[TagGroup] | None = None, _connection_budget: int | None = None, _quiet: bool = False, ) -> RecallResultModel: @@ -2434,6 +2435,7 @@ class MemoryEngine(MemoryEngineInterface): semaphore_wait=semaphore_wait, tags=tags, tags_match=tags_match, + tag_groups=tag_groups, connection_budget=_connection_budget, quiet=_quiet, include_source_facts=include_source_facts, @@ -2561,6 +2563,7 @@ class MemoryEngine(MemoryEngineInterface): semaphore_wait: float = 0.0, tags: list[str] | None = None, tags_match: TagsMatch = "any", + tag_groups: list[TagGroup] | None = None, connection_budget: int | None = None, quiet: bool = False, include_source_facts: bool = False, @@ -2680,6 +2683,7 @@ class MemoryEngine(MemoryEngineInterface): self.query_analyzer, tags=tags, tags_match=tags_match, + tag_groups=tag_groups, ) parallel_duration = time.time() - parallel_start finally: @@ -5040,6 +5044,7 @@ class MemoryEngine(MemoryEngineInterface): request_context: "RequestContext", tags: list[str] | None = None, tags_match: TagsMatch = "any", + tag_groups: list[TagGroup] | None = None, exclude_mental_model_ids: list[str] | None = None, _skip_span: bool = False, ) -> ReflectResult: @@ -5142,6 +5147,7 @@ class MemoryEngine(MemoryEngineInterface): max_results=max_results, tags=tags, tags_match=tags_match, + tag_groups=tag_groups, exclude_ids=exclude_mental_model_ids, pending_consolidation=pending_consolidation, ) @@ -5155,6 +5161,7 @@ class MemoryEngine(MemoryEngineInterface): max_tokens=max_tokens, tags=tags, tags_match=tags_match, + tag_groups=tag_groups, last_consolidated_at=last_consolidated_at, pending_consolidation=pending_consolidation, ) @@ -5168,6 +5175,7 @@ class MemoryEngine(MemoryEngineInterface): max_tokens=max_tokens, tags=tags, tags_match=tags_match, + tag_groups=tag_groups, max_chunk_tokens=max_chunk_tokens, ) diff --git a/hindsight-api-slim/hindsight_api/engine/reflect/tools.py b/hindsight-api-slim/hindsight_api/engine/reflect/tools.py index 00488712..a55d9833 100644 --- a/hindsight-api-slim/hindsight_api/engine/reflect/tools.py +++ b/hindsight-api-slim/hindsight_api/engine/reflect/tools.py @@ -29,6 +29,7 @@ async def tool_search_mental_models( max_results: int = 5, tags: list[str] | None = None, tags_match: str = "any", + tag_groups: "list | None" = None, exclude_ids: list[str] | None = None, pending_consolidation: int = 0, ) -> dict[str, Any]: @@ -52,7 +53,7 @@ async def tool_search_mental_models( Dict with matching mental models including content and freshness info """ from ..memory_engine import fq_table - from ..search.tags import build_tags_where_clause + from ..search.tags import build_tag_groups_where_clause, build_tags_where_clause # Build filters dynamically filters = "" @@ -65,6 +66,11 @@ async def tool_search_mental_models( filters += f" {tag_clause}" params.extend(tag_params) + if tag_groups: + groups_clause, groups_params, next_param = build_tag_groups_where_clause(tag_groups, next_param) + filters += f" {groups_clause}" + params.extend(groups_params) + if exclude_ids: filters += f" AND id != ALL(${next_param}::text[])" params.append(exclude_ids) @@ -125,6 +131,7 @@ async def tool_search_observations( max_tokens: int = 5000, tags: list[str] | None = None, tags_match: str = "any", + tag_groups: "list | None" = None, last_consolidated_at: datetime | None = None, pending_consolidation: int = 0, ) -> dict[str, Any]: @@ -157,6 +164,7 @@ async def tool_search_observations( request_context=request_context, tags=tags, tags_match=tags_match, + tag_groups=tag_groups, include_source_facts=True, max_source_facts_tokens=-1, # No token limit — include all source facts _connection_budget=1, @@ -189,6 +197,7 @@ async def tool_recall( max_tokens: int = 2048, tags: list[str] | None = None, tags_match: str = "any", + tag_groups: "list | None" = None, connection_budget: int = 1, max_chunk_tokens: int = 1000, ) -> dict[str, Any]: @@ -222,6 +231,7 @@ async def tool_recall( request_context=request_context, tags=tags, tags_match=tags_match, + tag_groups=tag_groups, _connection_budget=connection_budget, _quiet=True, # Suppress logging for internal operations include_chunks=include_chunks, diff --git a/hindsight-api-slim/hindsight_api/engine/search/graph_retrieval.py b/hindsight-api-slim/hindsight_api/engine/search/graph_retrieval.py index acef2243..ce60963f 100644 --- a/hindsight-api-slim/hindsight_api/engine/search/graph_retrieval.py +++ b/hindsight-api-slim/hindsight_api/engine/search/graph_retrieval.py @@ -11,7 +11,7 @@ from abc import ABC, abstractmethod from ..db_utils import acquire_with_retry from ..memory_engine import fq_table -from .tags import TagsMatch, filter_results_by_tags +from .tags import TagGroup, TagsMatch, filter_results_by_tag_groups, filter_results_by_tags from .types import MPFPTimings, RetrievalResult logger = logging.getLogger(__name__) @@ -46,6 +46,7 @@ class GraphRetriever(ABC): adjacency=None, # TypedAdjacency, optional pre-loaded graph tags: list[str] | None = None, # Visibility scope tags for filtering tags_match: TagsMatch = "any", # How to match tags: 'any' (OR) or 'all' (AND) + tag_groups: list[TagGroup] | None = None, # Compound boolean tag filter groups ) -> tuple[list[RetrievalResult], MPFPTimings | None]: """ Retrieve relevant facts via graph traversal. @@ -120,6 +121,7 @@ class BFSGraphRetriever(GraphRetriever): adjacency=None, # Not used by BFS tags: list[str] | None = None, tags_match: TagsMatch = "any", + tag_groups: list[TagGroup] | None = None, ) -> tuple[list[RetrievalResult], MPFPTimings | None]: """ Retrieve facts using BFS spreading activation. @@ -136,7 +138,14 @@ class BFSGraphRetriever(GraphRetriever): """ async with acquire_with_retry(pool) as conn: results = await self._retrieve_with_conn( - conn, query_embedding_str, bank_id, fact_type, budget, tags=tags, tags_match=tags_match + conn, + query_embedding_str, + bank_id, + fact_type, + budget, + tags=tags, + tags_match=tags_match, + tag_groups=tag_groups, ) return results, None @@ -149,14 +158,18 @@ class BFSGraphRetriever(GraphRetriever): budget: int, tags: list[str] | None = None, tags_match: TagsMatch = "any", + tag_groups: list[TagGroup] | None = None, ) -> list[RetrievalResult]: """Internal implementation with connection.""" - from .tags import build_tags_where_clause_simple + from .tags import build_tag_groups_where_clause, build_tags_where_clause_simple tags_clause = build_tags_where_clause_simple(tags, 6, match=tags_match) + tag_groups_param_start = 6 + (1 if tags else 0) + groups_clause, groups_params, _ = build_tag_groups_where_clause(tag_groups, tag_groups_param_start) params = [query_embedding_str, bank_id, fact_type, self.entry_point_threshold, self.entry_point_limit] if tags: params.append(tags) + params.extend(groups_params) # Step 1: Find entry points entry_points = await conn.fetch( @@ -170,6 +183,7 @@ class BFSGraphRetriever(GraphRetriever): AND fact_type = $3 AND (1 - (embedding <=> $1::vector)) >= $4 {tags_clause} + {groups_clause} ORDER BY embedding <=> $1::vector LIMIT $5 """, @@ -261,4 +275,8 @@ class BFSGraphRetriever(GraphRetriever): if tags: results = filter_results_by_tags(results, tags, match=tags_match) + # Apply compound tag group filtering (post-traversal) + if tag_groups: + results = filter_results_by_tag_groups(results, tag_groups) + return results diff --git a/hindsight-api-slim/hindsight_api/engine/search/link_expansion_retrieval.py b/hindsight-api-slim/hindsight_api/engine/search/link_expansion_retrieval.py index a761546b..5e4652ec 100644 --- a/hindsight-api-slim/hindsight_api/engine/search/link_expansion_retrieval.py +++ b/hindsight-api-slim/hindsight_api/engine/search/link_expansion_retrieval.py @@ -28,7 +28,7 @@ import time from ..db_utils import acquire_with_retry from ..memory_engine import fq_table from .graph_retrieval import GraphRetriever -from .tags import TagsMatch, filter_results_by_tags +from .tags import TagGroup, TagsMatch, filter_results_by_tag_groups, filter_results_by_tags from .types import MPFPTimings, RetrievalResult logger = logging.getLogger(__name__) @@ -43,14 +43,18 @@ async def _find_semantic_seeds( threshold: float = 0.3, tags: list[str] | None = None, tags_match: TagsMatch = "any", + tag_groups: list[TagGroup] | None = None, ) -> list[RetrievalResult]: """Find semantic seeds via embedding search.""" - from .tags import build_tags_where_clause_simple + from .tags import build_tag_groups_where_clause, build_tags_where_clause_simple tags_clause = build_tags_where_clause_simple(tags, 6, match=tags_match) + tag_groups_param_start = 6 + (1 if tags else 0) + groups_clause, groups_params, _ = build_tag_groups_where_clause(tag_groups, tag_groups_param_start) params = [query_embedding_str, bank_id, fact_type, threshold, limit] if tags: params.append(tags) + params.extend(groups_params) rows = await conn.fetch( f""" @@ -63,6 +67,7 @@ async def _find_semantic_seeds( AND fact_type = $3 AND (1 - (embedding <=> $1::vector)) >= $4 {tags_clause} + {groups_clause} ORDER BY embedding <=> $1::vector LIMIT $5 """, @@ -110,6 +115,7 @@ class LinkExpansionRetriever(GraphRetriever): adjacency=None, tags: list[str] | None = None, tags_match: TagsMatch = "any", + tag_groups: list[TagGroup] | None = None, ) -> tuple[list[RetrievalResult], MPFPTimings | None]: """ Retrieve facts by expanding links from seeds. @@ -147,6 +153,7 @@ class LinkExpansionRetriever(GraphRetriever): threshold=0.3, tags=tags, tags_match=tags_match, + tag_groups=tag_groups, ) timings.seeds_time = time.time() - seeds_start logger.debug( @@ -221,6 +228,9 @@ class LinkExpansionRetriever(GraphRetriever): if tags: results = filter_results_by_tags(results, tags, match=tags_match) + if tag_groups: + results = filter_results_by_tag_groups(results, tag_groups) + timings.result_count = len(results) timings.traverse = time.time() - start_time diff --git a/hindsight-api-slim/hindsight_api/engine/search/mpfp_retrieval.py b/hindsight-api-slim/hindsight_api/engine/search/mpfp_retrieval.py index 01d00e1e..ad53914b 100644 --- a/hindsight-api-slim/hindsight_api/engine/search/mpfp_retrieval.py +++ b/hindsight-api-slim/hindsight_api/engine/search/mpfp_retrieval.py @@ -23,7 +23,7 @@ from dataclasses import dataclass, field from ..db_utils import acquire_with_retry from ..memory_engine import fq_table from .graph_retrieval import GraphRetriever -from .tags import TagsMatch +from .tags import TagGroup, TagsMatch from .types import MPFPTimings, RetrievalResult logger = logging.getLogger(__name__) @@ -506,6 +506,7 @@ class MPFPGraphRetriever(GraphRetriever): adjacency=None, # Ignored - kept for interface compatibility tags: list[str] | None = None, tags_match: TagsMatch = "any", + tag_groups: list[TagGroup] | None = None, ) -> tuple[list[RetrievalResult], MPFPTimings | None]: """ Retrieve facts using MPFP algorithm with lazy edge loading. @@ -537,7 +538,13 @@ class MPFPGraphRetriever(GraphRetriever): if not semantic_seed_nodes: seeds_start = time.time() semantic_seed_nodes = await self._find_semantic_seeds( - pool, query_embedding_str, bank_id, fact_type, tags=tags, tags_match=tags_match + pool, + query_embedding_str, + bank_id, + fact_type, + tags=tags, + tags_match=tags_match, + tag_groups=tag_groups, ) timings.seeds_time = time.time() - seeds_start logger.debug( @@ -616,6 +623,12 @@ class MPFPGraphRetriever(GraphRetriever): results = filter_results_by_tags(results, tags, match=tags_match) + # Apply compound tag group filtering (post-traversal) + if tag_groups: + from .tags import filter_results_by_tag_groups + + results = filter_results_by_tag_groups(results, tag_groups) + timings.result_count = len(results) # Add activation scores from fusion @@ -656,14 +669,18 @@ class MPFPGraphRetriever(GraphRetriever): threshold: float = 0.3, tags: list[str] | None = None, tags_match: TagsMatch = "any", + tag_groups: list[TagGroup] | None = None, ) -> list[SeedNode]: """Fallback: find semantic seeds via embedding search.""" - from .tags import build_tags_where_clause_simple + from .tags import build_tag_groups_where_clause, build_tags_where_clause_simple tags_clause = build_tags_where_clause_simple(tags, 6, match=tags_match) + tag_groups_param_start = 6 + (1 if tags else 0) + groups_clause, groups_params, _ = build_tag_groups_where_clause(tag_groups, tag_groups_param_start) params = [query_embedding_str, bank_id, fact_type, threshold, limit] if tags: params.append(tags) + params.extend(groups_params) async with acquire_with_retry(pool) as conn: rows = await conn.fetch( @@ -675,6 +692,7 @@ class MPFPGraphRetriever(GraphRetriever): AND fact_type = $3 AND (1 - (embedding <=> $1::vector)) >= $4 {tags_clause} + {groups_clause} ORDER BY embedding <=> $1::vector LIMIT $5 """, diff --git a/hindsight-api-slim/hindsight_api/engine/search/retrieval.py b/hindsight-api-slim/hindsight_api/engine/search/retrieval.py index 57f7963e..e4c43e5d 100644 --- a/hindsight-api-slim/hindsight_api/engine/search/retrieval.py +++ b/hindsight-api-slim/hindsight_api/engine/search/retrieval.py @@ -20,7 +20,7 @@ from ..memory_engine import fq_table from .graph_retrieval import BFSGraphRetriever, GraphRetriever from .link_expansion_retrieval import LinkExpansionRetriever from .mpfp_retrieval import MPFPGraphRetriever -from .tags import TagsMatch, build_tags_where_clause_simple +from .tags import TagGroup, TagsMatch, build_tag_groups_where_clause, build_tags_where_clause_simple from .types import MPFPTimings, RetrievalResult logger = logging.getLogger(__name__) @@ -94,6 +94,7 @@ async def retrieve_semantic_bm25_combined( limit: int, tags: list[str] | None = None, tags_match: TagsMatch = "any", + tag_groups: list[TagGroup] | None = None, ) -> dict[str, tuple[list[RetrievalResult], list[RetrievalResult]]]: """ Combined semantic + BM25 retrieval for multiple fact types in a single query. @@ -150,9 +151,14 @@ async def retrieve_semantic_bm25_combined( # $3 = limit (BM25 LIMIT; semantic uses inlined hnsw_fetch literal) # $4 = bm25_text (only when tokens present) # $N = tags (N=4 when no tokens, N=5 when tokens present) + # $M+ = tag_groups params (one per leaf, starting after tags param) tags_param_idx = 5 if tokens else 4 tags_clause = build_tags_where_clause_simple(tags, tags_param_idx, match=tags_match) + # tag_groups params start immediately after the tags param slot + tag_groups_param_start = tags_param_idx + (1 if tags else 0) + groups_clause, groups_params, _ = build_tag_groups_where_clause(tag_groups, tag_groups_param_start) + # --- Semantic UNION ALL arms (one per fact_type) --- # Each arm has its own ORDER BY embedding <=> $1 LIMIT {hnsw_fetch}, which # lets the planner use the partial HNSW index for that fact_type. @@ -169,6 +175,7 @@ async def retrieve_semantic_bm25_combined( f" AND embedding IS NOT NULL" f" AND (1 - (embedding <=> $1::vector)) >= 0.3" f" {tags_clause}" + f" {groups_clause}" f" ORDER BY embedding <=> $1::vector" f" LIMIT {hnsw_fetch})" ) @@ -208,6 +215,7 @@ async def retrieve_semantic_bm25_combined( f" AND fact_type = '{ft}'" f" {bm25_where_filter}" f" {tags_clause}" + f" {groups_clause}" f" ORDER BY {bm25_order_by}" f" LIMIT $3)" ) @@ -219,6 +227,7 @@ async def retrieve_semantic_bm25_combined( params.append(bm25_text_param) if tags: params.append(tags) + params.extend(groups_params) rows = await conn.fetch(query, *params) @@ -251,6 +260,7 @@ async def retrieve_temporal_combined( semantic_threshold: float = 0.1, tags: list[str] | None = None, tags_match: TagsMatch = "any", + tag_groups: list[TagGroup] | None = None, ) -> dict[str, list[RetrievalResult]]: """ Temporal retrieval for multiple fact types in a single query. @@ -280,10 +290,14 @@ async def retrieve_temporal_combined( end_date = end_date.replace(tzinfo=UTC) # Build tags clause + # Entry point query: fixed params are $1-$6, tags at $7 tags_clause = build_tags_where_clause_simple(tags, 7, match=tags_match) - params = [query_emb_str, bank_id, fact_types, start_date, end_date, semantic_threshold] + tag_groups_param_start = 7 + (1 if tags else 0) + groups_clause, groups_params, _ = build_tag_groups_where_clause(tag_groups, tag_groups_param_start) + params: list = [query_emb_str, bank_id, fact_types, start_date, end_date, semantic_threshold] if tags: params.append(tags) + params.extend(groups_params) # Two-phase entry point query: # Phase 1 (date_ranked): rank by date only — no embedding computation — for all units in @@ -314,6 +328,7 @@ async def retrieve_temporal_combined( (occurred_end IS NOT NULL AND occurred_end BETWEEN $4 AND $5) ) {tags_clause} + {groups_clause} ), sim_ranked AS ( SELECT mu.id, mu.text, mu.context, mu.event_date, mu.occurred_start, mu.occurred_end, mu.mentioned_at, mu.fact_type, mu.document_id, mu.chunk_id, mu.tags, @@ -400,16 +415,21 @@ async def retrieve_temporal_combined( # Build tags clause for spreading (use param 7 since 1-6 are used) spreading_tags_clause = build_tags_where_clause_simple(tags, 7, table_alias="mu.", match=tags_match) + spreading_groups_param_start = 7 + (1 if tags else 0) + spreading_groups_clause, spreading_groups_params, _ = build_tag_groups_where_clause( + tag_groups, spreading_groups_param_start, table_alias="mu." + ) while frontier and budget_remaining > 0 and iteration < max_iterations: iteration += 1 batch_ids = frontier[:batch_size] frontier = frontier[batch_size:] - # $1=query_emb, $2=batch_ids, $3=fact_type, $4=threshold, $5=per_source_limit, $6=bank_id, $7=tags + # $1=query_emb, $2=batch_ids, $3=fact_type, $4=threshold, $5=per_source_limit, $6=bank_id, $7=tags, $M+=tag_groups spreading_params = [query_emb_str, batch_ids, ft, semantic_threshold, per_source_limit, bank_id] if tags: spreading_params.append(tags) + spreading_params.extend(spreading_groups_params) # LATERAL join: for each source node, fetch top-K neighbors by weight using # the existing idx_memory_links_from_type_weight index with early-exit semantics. @@ -436,6 +456,7 @@ async def retrieve_temporal_combined( AND mu.embedding IS NOT NULL AND (1 - (mu.embedding <=> $1::vector)) >= $4 {spreading_tags_clause} + {spreading_groups_clause} """, *spreading_params, ) @@ -509,6 +530,7 @@ async def retrieve_all_fact_types_parallel( graph_retriever: GraphRetriever | None = None, tags: list[str] | None = None, tags_match: TagsMatch = "any", + tag_groups: list[TagGroup] | None = None, ) -> MultiFactTypeRetrievalResult: """ Optimized retrieval for multiple fact types using batched queries. @@ -566,6 +588,7 @@ async def retrieve_all_fact_types_parallel( thinking_budget, tags=tags, tags_match=tags_match, + tag_groups=tag_groups, ) semantic_bm25_time = time.time() - semantic_bm25_start @@ -584,6 +607,7 @@ async def retrieve_all_fact_types_parallel( semantic_threshold=0.1, tags=tags, tags_match=tags_match, + tag_groups=tag_groups, ) temporal_time = time.time() - temporal_start @@ -604,6 +628,7 @@ async def retrieve_all_fact_types_parallel( temporal_seeds=None, tags=tags, tags_match=tags_match, + tag_groups=tag_groups, ) return ft, results, time.time() - graph_start, mpfp_timing diff --git a/hindsight-api-slim/hindsight_api/engine/search/tags.py b/hindsight-api-slim/hindsight_api/engine/search/tags.py index 5417a5a9..4f876eab 100644 --- a/hindsight-api-slim/hindsight_api/engine/search/tags.py +++ b/hindsight-api-slim/hindsight_api/engine/search/tags.py @@ -12,7 +12,11 @@ OR matching (any/any_strict): Memory matches if ANY of its tags overlap with req AND matching (all/all_strict): Memory matches if ALL request tags are present in its tags """ -from typing import Literal +from __future__ import annotations + +from typing import Annotated, Literal + +from pydantic import BaseModel, ConfigDict, Field TagsMatch = Literal["any", "all", "any_strict", "all_strict"] @@ -170,3 +174,217 @@ def filter_results_by_tags( filtered.append(result) return filtered + + +# ============================================================================= +# Compound tag group models (recursive boolean expressions) +# ============================================================================= + + +class TagGroupLeaf(BaseModel): + """A leaf tag filter: matches memories by tag list and match mode.""" + + tags: list[str] + match: TagsMatch = "any_strict" + + +class TagGroupAnd(BaseModel): + """Compound AND group: all child filters must match.""" + + model_config = ConfigDict(populate_by_name=True) + filters: list[TagGroup] = Field(alias="and") + + +class TagGroupOr(BaseModel): + """Compound OR group: at least one child filter must match.""" + + model_config = ConfigDict(populate_by_name=True) + filters: list[TagGroup] = Field(alias="or") + + +class TagGroupNot(BaseModel): + """Compound NOT group: child filter must NOT match.""" + + model_config = ConfigDict(populate_by_name=True) + filter: TagGroup = Field(alias="not") + + +# TagGroup is a discriminated union; Pydantic will try left-to-right. +# TagGroupLeaf is identified by the presence of 'tags'. +# TagGroupAnd / TagGroupOr / TagGroupNot are compound (no 'tags' key). +TagGroup = Annotated[ + TagGroupLeaf | TagGroupAnd | TagGroupOr | TagGroupNot, + Field(union_mode="left_to_right"), +] + +# Rebuild forward-reference models so recursive TagGroup is resolved. +TagGroupAnd.model_rebuild() +TagGroupOr.model_rebuild() +TagGroupNot.model_rebuild() + + +# ============================================================================= +# SQL builder for compound tag groups +# ============================================================================= + + +def _build_group_clause( + group: TagGroup, + param_offset: int, + table_alias: str, +) -> tuple[str, list, int]: + """ + Recursively build an inner SQL clause (no leading AND/OR) for a single TagGroup. + + Returns: + (inner_clause, params, next_param_offset) + """ + if isinstance(group, TagGroupLeaf): + column = f"{table_alias}tags" if table_alias else "tags" + operator, include_untagged = _parse_tags_match(group.match) + if include_untagged: + clause = f"({column} IS NULL OR {column} = '{{}}' OR {column} {operator} ${param_offset})" + else: + clause = f"({column} IS NOT NULL AND {column} != '{{}}' AND {column} {operator} ${param_offset})" + return clause, [group.tags], param_offset + 1 + + elif isinstance(group, TagGroupAnd): + parts = [] + params: list = [] + offset = param_offset + for child in group.filters: + child_clause, child_params, offset = _build_group_clause(child, offset, table_alias) + parts.append(child_clause) + params.extend(child_params) + inner = " AND ".join(parts) + return f"({inner})", params, offset + + elif isinstance(group, TagGroupOr): + parts = [] + params = [] + offset = param_offset + for child in group.filters: + child_clause, child_params, offset = _build_group_clause(child, offset, table_alias) + parts.append(child_clause) + params.extend(child_params) + inner = " OR ".join(parts) + return f"({inner})", params, offset + + elif isinstance(group, TagGroupNot): + child_clause, child_params, next_offset = _build_group_clause(group.filter, param_offset, table_alias) + return f"NOT {child_clause}", child_params, next_offset + + else: + # Should never happen with proper Pydantic validation + return "", [], param_offset + + +def build_tag_groups_where_clause( + tag_groups: list[TagGroup] | None, + param_offset: int, + table_alias: str = "", +) -> tuple[str, list, int]: + """ + Build a SQL WHERE clause for compound tag group filtering. + + Top-level groups are AND-ed together. Each group is a recursive boolean + expression (leaf, and, or, not). + + Args: + tag_groups: List of TagGroup objects. If None or empty, returns empty clause. + param_offset: Starting parameter number for SQL placeholders. + table_alias: Optional table alias prefix (e.g., "mu." for "memory_units mu"). + + Returns: + Tuple of (sql_clause, params, next_param_offset): + - sql_clause: SQL WHERE clause string starting with "AND" (or empty string) + - params: List of parameter values to bind (one per leaf node) + - next_param_offset: Next available parameter number + + Example: + >>> groups = [TagGroupLeaf(tags=["user:alice"], match="all_strict")] + >>> clause, params, next_offset = build_tag_groups_where_clause(groups, 3) + >>> print(clause) # "AND (tags IS NOT NULL AND tags != '{}' AND tags @> $3)" + """ + if not tag_groups: + return "", [], param_offset + + all_params: list = [] + all_clauses: list[str] = [] + offset = param_offset + + for group in tag_groups: + inner_clause, group_params, offset = _build_group_clause(group, offset, table_alias) + all_clauses.append(inner_clause) + all_params.extend(group_params) + + combined = " AND ".join(all_clauses) + return f"AND {combined}", all_params, offset + + +# ============================================================================= +# Python-side filter for compound tag groups (post-retrieval filtering) +# ============================================================================= + + +def _match_group(result: object, group: TagGroup) -> bool: + """ + Recursively evaluate a TagGroup against a retrieval result. + + Args: + result: Any object with a 'tags' attribute (list[str] or None). + group: The TagGroup to evaluate. + + Returns: + True if the result matches the group, False otherwise. + """ + if isinstance(group, TagGroupLeaf): + result_tags = getattr(result, "tags", None) + is_untagged = result_tags is None or len(result_tags) == 0 + _, include_untagged = _parse_tags_match(group.match) + is_any_match = group.match in ("any", "any_strict") + tags_set = set(group.tags) + + if is_untagged: + return include_untagged + else: + result_tags_set = set(result_tags) + if is_any_match: + return bool(result_tags_set & tags_set) + else: + return tags_set <= result_tags_set + + elif isinstance(group, TagGroupAnd): + return all(_match_group(result, child) for child in group.filters) + + elif isinstance(group, TagGroupOr): + return any(_match_group(result, child) for child in group.filters) + + elif isinstance(group, TagGroupNot): + return not _match_group(result, group.filter) + + else: + return True + + +def filter_results_by_tag_groups( + results: list, + tag_groups: list[TagGroup] | None, +) -> list: + """ + Filter retrieval results by compound tag groups in Python (for post-processing). + + Used when SQL filtering isn't possible (e.g., graph traversal results). + Top-level groups are AND-ed together. + + Args: + results: List of RetrievalResult objects with a 'tags' attribute. + tag_groups: List of TagGroup objects. If None or empty, returns all results. + + Returns: + Filtered list of results where ALL top-level groups match. + """ + if not tag_groups: + return results + + return [r for r in results if all(_match_group(r, group) for group in tag_groups)] diff --git a/hindsight-api-slim/tests/test_tags_visibility.py b/hindsight-api-slim/tests/test_tags_visibility.py index 053e5798..ba17a50b 100644 --- a/hindsight-api-slim/tests/test_tags_visibility.py +++ b/hindsight-api-slim/tests/test_tags_visibility.py @@ -16,7 +16,16 @@ import pytest import pytest_asyncio from hindsight_api.api import create_app -from hindsight_api.engine.search.tags import build_tags_where_clause_simple, filter_results_by_tags +from hindsight_api.engine.search.tags import ( + TagGroupAnd, + TagGroupLeaf, + TagGroupNot, + TagGroupOr, + build_tag_groups_where_clause, + build_tags_where_clause_simple, + filter_results_by_tag_groups, + filter_results_by_tags, +) # ============================================================================ # Unit Tests for tags SQL builder @@ -263,6 +272,327 @@ class TestFilterResultsByTags: assert missing_session not in filtered +# ============================================================================ +# Unit Tests for build_tag_groups_where_clause (SQL builder) +# ============================================================================ + + +class TestBuildTagGroupsWhereClause: + """Unit tests for the compound tag group SQL builder.""" + + def test_none_returns_empty(self): + """None tag_groups returns empty clause.""" + clause, params, next_offset = build_tag_groups_where_clause(None, 3) + assert clause == "" + assert params == [] + assert next_offset == 3 + + def test_empty_list_returns_empty(self): + """Empty tag_groups list returns empty clause.""" + clause, params, next_offset = build_tag_groups_where_clause([], 3) + assert clause == "" + assert params == [] + assert next_offset == 3 + + def test_single_leaf_any_strict(self): + """Single any_strict leaf generates correct SQL.""" + groups = [TagGroupLeaf(tags=["step:5", "step:8"], match="any_strict")] + clause, params, next_offset = build_tag_groups_where_clause(groups, 3) + assert clause.startswith("AND ") + assert "$3" in clause + assert "IS NOT NULL" in clause + assert "!= '{}'" in clause + assert "&&" in clause + assert params == [["step:5", "step:8"]] + assert next_offset == 4 + + def test_single_leaf_all_strict(self): + """Single all_strict leaf generates @> operator.""" + groups = [TagGroupLeaf(tags=["user:alice"], match="all_strict")] + clause, params, next_offset = build_tag_groups_where_clause(groups, 1) + assert "@>" in clause + assert "IS NOT NULL" in clause + assert params == [["user:alice"]] + assert next_offset == 2 + + def test_single_leaf_any_includes_untagged(self): + """Single any (non-strict) leaf generates NULL-inclusive clause.""" + groups = [TagGroupLeaf(tags=["user:alice"], match="any")] + clause, params, next_offset = build_tag_groups_where_clause(groups, 1) + assert "IS NULL" in clause + assert "= '{}'" in clause + assert "&&" in clause + assert params == [["user:alice"]] + assert next_offset == 2 + + def test_and_of_two_leaves(self): + """AND of two leaves generates AND-joined clause.""" + groups = [ + TagGroupAnd.model_validate( + {"and": [ + {"tags": ["step:5"], "match": "any_strict"}, + {"tags": ["user:ep_42"], "match": "all_strict"}, + ]} + ) + ] + clause, params, next_offset = build_tag_groups_where_clause(groups, 3) + assert "AND" in clause + assert "$3" in clause + assert "$4" in clause + assert len(params) == 2 + assert params[0] == ["step:5"] + assert params[1] == ["user:ep_42"] + assert next_offset == 5 + + def test_or_of_two_leaves(self): + """OR of two leaves generates OR-joined clause.""" + groups = [ + TagGroupOr.model_validate( + {"or": [ + {"tags": ["step:5"], "match": "any_strict"}, + {"tags": ["priority:high"], "match": "all_strict"}, + ]} + ) + ] + clause, params, next_offset = build_tag_groups_where_clause(groups, 1) + assert "OR" in clause + assert "$1" in clause + assert "$2" in clause + assert len(params) == 2 + assert next_offset == 3 + + def test_not_wraps_with_not(self): + """NOT group wraps child clause with NOT.""" + groups = [ + TagGroupNot.model_validate( + {"not": {"tags": ["archived"], "match": "any_strict"}} + ) + ] + clause, params, next_offset = build_tag_groups_where_clause(groups, 2) + assert "NOT" in clause + assert "$2" in clause + assert len(params) == 1 + assert next_offset == 3 + + def test_nested_and_containing_or(self): + """AND containing an OR generates correct nested SQL.""" + groups = [ + TagGroupAnd.model_validate( + {"and": [ + {"tags": ["user:alice"], "match": "all_strict"}, + {"or": [ + {"tags": ["step:5"], "match": "any_strict"}, + {"tags": ["priority:high"], "match": "all_strict"}, + ]}, + ]} + ) + ] + clause, params, next_offset = build_tag_groups_where_clause(groups, 1) + assert "AND" in clause + assert "OR" in clause + assert len(params) == 3 + assert next_offset == 4 + + def test_param_numbering_sequential(self): + """Params are numbered sequentially starting from param_offset.""" + groups = [ + TagGroupAnd.model_validate( + {"and": [ + {"tags": ["a"], "match": "any_strict"}, + {"tags": ["b"], "match": "any_strict"}, + {"tags": ["c"], "match": "any_strict"}, + ]} + ) + ] + clause, params, next_offset = build_tag_groups_where_clause(groups, 5) + assert "$5" in clause + assert "$6" in clause + assert "$7" in clause + assert next_offset == 8 + assert len(params) == 3 + + def test_table_alias_applied_to_leaves(self): + """Table alias is prefixed to column name in all leaf clauses.""" + groups = [TagGroupLeaf(tags=["user:alice"], match="any_strict")] + clause, params, next_offset = build_tag_groups_where_clause(groups, 1, table_alias="mu.") + assert "mu.tags" in clause + + def test_table_alias_propagates_to_nested(self): + """Table alias propagates to nested leaves (each leaf uses the alias).""" + groups = [ + TagGroupAnd.model_validate( + {"and": [ + {"tags": ["a"], "match": "any_strict"}, + {"tags": ["b"], "match": "any_strict"}, + ]} + ) + ] + clause, params, next_offset = build_tag_groups_where_clause(groups, 1, table_alias="mu.") + # Each leaf of type any_strict references mu.tags three times (IS NOT NULL, != '{}', &&) + # We verify that 'tags' without alias is NOT present, proving the alias is always used + assert "mu.tags" in clause + # No bare 'tags' keyword without the alias prefix (other than inside the alias itself) + import re + bare_tags = re.findall(r"(? None: + if args: + if len(args) > 1: + raise ValueError("If a position argument is used, only 1 is allowed to set `actual_instance`") + if kwargs: + raise ValueError("If a position argument is used, keyword arguments cannot be used.") + super().__init__(actual_instance=args[0]) + else: + super().__init__(**kwargs) + + @field_validator('actual_instance') + def actual_instance_must_validate_anyof(cls, v): + instance = ModelNot.model_construct() + error_messages = [] + # validate data type: TagGroupLeaf + if not isinstance(v, TagGroupLeaf): + error_messages.append(f"Error! Input type `{type(v)}` is not `TagGroupLeaf`") + else: + return v + + # validate data type: TagGroupAnd + if not isinstance(v, TagGroupAnd): + error_messages.append(f"Error! Input type `{type(v)}` is not `TagGroupAnd`") + else: + return v + + # validate data type: TagGroupOr + if not isinstance(v, TagGroupOr): + error_messages.append(f"Error! Input type `{type(v)}` is not `TagGroupOr`") + else: + return v + + # validate data type: TagGroupNot + if not isinstance(v, TagGroupNot): + error_messages.append(f"Error! Input type `{type(v)}` is not `TagGroupNot`") + else: + return v + + if error_messages: + # no match + raise ValueError("No match found when setting the actual_instance in ModelNot with anyOf schemas: TagGroupAnd, TagGroupLeaf, TagGroupNot, TagGroupOr. Details: " + ", ".join(error_messages)) + else: + return v + + @classmethod + def from_dict(cls, obj: Dict[str, Any]) -> Self: + return cls.from_json(json.dumps(obj)) + + @classmethod + def from_json(cls, json_str: str) -> Self: + """Returns the object represented by the json string""" + instance = cls.model_construct() + error_messages = [] + # anyof_schema_1_validator: Optional[TagGroupLeaf] = None + try: + instance.actual_instance = TagGroupLeaf.from_json(json_str) + return instance + except (ValidationError, ValueError) as e: + error_messages.append(str(e)) + # anyof_schema_2_validator: Optional[TagGroupAnd] = None + try: + instance.actual_instance = TagGroupAnd.from_json(json_str) + return instance + except (ValidationError, ValueError) as e: + error_messages.append(str(e)) + # anyof_schema_3_validator: Optional[TagGroupOr] = None + try: + instance.actual_instance = TagGroupOr.from_json(json_str) + return instance + except (ValidationError, ValueError) as e: + error_messages.append(str(e)) + # anyof_schema_4_validator: Optional[TagGroupNot] = None + try: + instance.actual_instance = TagGroupNot.from_json(json_str) + return instance + except (ValidationError, ValueError) as e: + error_messages.append(str(e)) + + if error_messages: + # no match + raise ValueError("No match found when deserializing the JSON string into ModelNot with anyOf schemas: TagGroupAnd, TagGroupLeaf, TagGroupNot, TagGroupOr. Details: " + ", ".join(error_messages)) + else: + return instance + + def to_json(self) -> str: + """Returns the JSON representation of the actual instance""" + if self.actual_instance is None: + return "null" + + if hasattr(self.actual_instance, "to_json") and callable(self.actual_instance.to_json): + return self.actual_instance.to_json() + else: + return json.dumps(self.actual_instance) + + def to_dict(self) -> Optional[Union[Dict[str, Any], TagGroupAnd, TagGroupLeaf, TagGroupNot, TagGroupOr]]: + """Returns the dict representation of the actual instance""" + if self.actual_instance is None: + return None + + if hasattr(self.actual_instance, "to_dict") and callable(self.actual_instance.to_dict): + return self.actual_instance.to_dict() + else: + return self.actual_instance + + def to_str(self) -> str: + """Returns the string representation of the actual instance""" + return pprint.pformat(self.model_dump()) + +from hindsight_client_api.models.tag_group_and import TagGroupAnd +from hindsight_client_api.models.tag_group_not import TagGroupNot +from hindsight_client_api.models.tag_group_or import TagGroupOr +# TODO: Rewrite to not use raise_errors +ModelNot.model_rebuild(raise_errors=False) + diff --git a/hindsight-clients/python/hindsight_client_api/models/recall_request.py b/hindsight-clients/python/hindsight_client_api/models/recall_request.py index 7bc5fa6f..f7bb3942 100644 --- a/hindsight-clients/python/hindsight_client_api/models/recall_request.py +++ b/hindsight-clients/python/hindsight_client_api/models/recall_request.py @@ -21,6 +21,7 @@ from pydantic import BaseModel, ConfigDict, Field, StrictBool, StrictInt, Strict from typing import Any, ClassVar, Dict, List, Optional from hindsight_client_api.models.budget import Budget from hindsight_client_api.models.include_options import IncludeOptions +from hindsight_client_api.models.recall_request_tag_groups_inner import RecallRequestTagGroupsInner from typing import Optional, Set from typing_extensions import Self @@ -37,7 +38,8 @@ class RecallRequest(BaseModel): include: Optional[IncludeOptions] = Field(default=None, description="Options for including additional data (entities are included by default)") tags: Optional[List[StrictStr]] = None tags_match: Optional[StrictStr] = Field(default='any', description="How to match tags: 'any' (OR, includes untagged), 'all' (AND, includes untagged), 'any_strict' (OR, excludes untagged), 'all_strict' (AND, excludes untagged).") - __properties: ClassVar[List[str]] = ["query", "types", "budget", "max_tokens", "trace", "query_timestamp", "include", "tags", "tags_match"] + tag_groups: Optional[List[RecallRequestTagGroupsInner]] = None + __properties: ClassVar[List[str]] = ["query", "types", "budget", "max_tokens", "trace", "query_timestamp", "include", "tags", "tags_match", "tag_groups"] @field_validator('tags_match') def tags_match_validate_enum(cls, value): @@ -91,6 +93,13 @@ class RecallRequest(BaseModel): # override the default output from pydantic by calling `to_dict()` of include if self.include: _dict['include'] = self.include.to_dict() + # override the default output from pydantic by calling `to_dict()` of each item in tag_groups (list) + _items = [] + if self.tag_groups: + for _item_tag_groups in self.tag_groups: + if _item_tag_groups: + _items.append(_item_tag_groups.to_dict()) + _dict['tag_groups'] = _items # set to None if types (nullable) is None # and model_fields_set contains the field if self.types is None and "types" in self.model_fields_set: @@ -106,6 +115,11 @@ class RecallRequest(BaseModel): if self.tags is None and "tags" in self.model_fields_set: _dict['tags'] = None + # set to None if tag_groups (nullable) is None + # and model_fields_set contains the field + if self.tag_groups is None and "tag_groups" in self.model_fields_set: + _dict['tag_groups'] = None + return _dict @classmethod @@ -126,7 +140,8 @@ class RecallRequest(BaseModel): "query_timestamp": obj.get("query_timestamp"), "include": IncludeOptions.from_dict(obj["include"]) if obj.get("include") is not None else None, "tags": obj.get("tags"), - "tags_match": obj.get("tags_match") if obj.get("tags_match") is not None else 'any' + "tags_match": obj.get("tags_match") if obj.get("tags_match") is not None else 'any', + "tag_groups": [RecallRequestTagGroupsInner.from_dict(_item) for _item in obj["tag_groups"]] if obj.get("tag_groups") is not None else None }) return _obj diff --git a/hindsight-clients/python/hindsight_client_api/models/recall_request_tag_groups_inner.py b/hindsight-clients/python/hindsight_client_api/models/recall_request_tag_groups_inner.py new file mode 100644 index 00000000..dd2dcc7f --- /dev/null +++ b/hindsight-clients/python/hindsight_client_api/models/recall_request_tag_groups_inner.py @@ -0,0 +1,166 @@ +# coding: utf-8 + +""" + Hindsight HTTP API + + HTTP API for Hindsight + + The version of the OpenAPI document: 0.4.17 + Generated by OpenAPI Generator (https://openapi-generator.tech) + + Do not edit the class manually. +""" # noqa: E501 + + +from __future__ import annotations +from inspect import getfullargspec +import json +import pprint +import re # noqa: F401 +from pydantic import BaseModel, ConfigDict, Field, StrictStr, ValidationError, field_validator +from typing import Optional +from hindsight_client_api.models.tag_group_leaf import TagGroupLeaf +from typing import Union, Any, List, Set, TYPE_CHECKING, Optional, Dict +from typing_extensions import Literal, Self +from pydantic import Field + +RECALLREQUESTTAGGROUPSINNER_ANY_OF_SCHEMAS = ["TagGroupAnd", "TagGroupLeaf", "TagGroupNot", "TagGroupOr"] + +class RecallRequestTagGroupsInner(BaseModel): + """ + RecallRequestTagGroupsInner + """ + + # data type: TagGroupLeaf + anyof_schema_1_validator: Optional[TagGroupLeaf] = None + # data type: TagGroupAnd + anyof_schema_2_validator: Optional[TagGroupAnd] = None + # data type: TagGroupOr + anyof_schema_3_validator: Optional[TagGroupOr] = None + # data type: TagGroupNot + anyof_schema_4_validator: Optional[TagGroupNot] = None + if TYPE_CHECKING: + actual_instance: Optional[Union[TagGroupAnd, TagGroupLeaf, TagGroupNot, TagGroupOr]] = None + else: + actual_instance: Any = None + any_of_schemas: Set[str] = { "TagGroupAnd", "TagGroupLeaf", "TagGroupNot", "TagGroupOr" } + + model_config = { + "validate_assignment": True, + "protected_namespaces": (), + } + + def __init__(self, *args, **kwargs) -> None: + if args: + if len(args) > 1: + raise ValueError("If a position argument is used, only 1 is allowed to set `actual_instance`") + if kwargs: + raise ValueError("If a position argument is used, keyword arguments cannot be used.") + super().__init__(actual_instance=args[0]) + else: + super().__init__(**kwargs) + + @field_validator('actual_instance') + def actual_instance_must_validate_anyof(cls, v): + instance = RecallRequestTagGroupsInner.model_construct() + error_messages = [] + # validate data type: TagGroupLeaf + if not isinstance(v, TagGroupLeaf): + error_messages.append(f"Error! Input type `{type(v)}` is not `TagGroupLeaf`") + else: + return v + + # validate data type: TagGroupAnd + if not isinstance(v, TagGroupAnd): + error_messages.append(f"Error! Input type `{type(v)}` is not `TagGroupAnd`") + else: + return v + + # validate data type: TagGroupOr + if not isinstance(v, TagGroupOr): + error_messages.append(f"Error! Input type `{type(v)}` is not `TagGroupOr`") + else: + return v + + # validate data type: TagGroupNot + if not isinstance(v, TagGroupNot): + error_messages.append(f"Error! Input type `{type(v)}` is not `TagGroupNot`") + else: + return v + + if error_messages: + # no match + raise ValueError("No match found when setting the actual_instance in RecallRequestTagGroupsInner with anyOf schemas: TagGroupAnd, TagGroupLeaf, TagGroupNot, TagGroupOr. Details: " + ", ".join(error_messages)) + else: + return v + + @classmethod + def from_dict(cls, obj: Dict[str, Any]) -> Self: + return cls.from_json(json.dumps(obj)) + + @classmethod + def from_json(cls, json_str: str) -> Self: + """Returns the object represented by the json string""" + instance = cls.model_construct() + error_messages = [] + # anyof_schema_1_validator: Optional[TagGroupLeaf] = None + try: + instance.actual_instance = TagGroupLeaf.from_json(json_str) + return instance + except (ValidationError, ValueError) as e: + error_messages.append(str(e)) + # anyof_schema_2_validator: Optional[TagGroupAnd] = None + try: + instance.actual_instance = TagGroupAnd.from_json(json_str) + return instance + except (ValidationError, ValueError) as e: + error_messages.append(str(e)) + # anyof_schema_3_validator: Optional[TagGroupOr] = None + try: + instance.actual_instance = TagGroupOr.from_json(json_str) + return instance + except (ValidationError, ValueError) as e: + error_messages.append(str(e)) + # anyof_schema_4_validator: Optional[TagGroupNot] = None + try: + instance.actual_instance = TagGroupNot.from_json(json_str) + return instance + except (ValidationError, ValueError) as e: + error_messages.append(str(e)) + + if error_messages: + # no match + raise ValueError("No match found when deserializing the JSON string into RecallRequestTagGroupsInner with anyOf schemas: TagGroupAnd, TagGroupLeaf, TagGroupNot, TagGroupOr. Details: " + ", ".join(error_messages)) + else: + return instance + + def to_json(self) -> str: + """Returns the JSON representation of the actual instance""" + if self.actual_instance is None: + return "null" + + if hasattr(self.actual_instance, "to_json") and callable(self.actual_instance.to_json): + return self.actual_instance.to_json() + else: + return json.dumps(self.actual_instance) + + def to_dict(self) -> Optional[Union[Dict[str, Any], TagGroupAnd, TagGroupLeaf, TagGroupNot, TagGroupOr]]: + """Returns the dict representation of the actual instance""" + if self.actual_instance is None: + return None + + if hasattr(self.actual_instance, "to_dict") and callable(self.actual_instance.to_dict): + return self.actual_instance.to_dict() + else: + return self.actual_instance + + def to_str(self) -> str: + """Returns the string representation of the actual instance""" + return pprint.pformat(self.model_dump()) + +from hindsight_client_api.models.tag_group_and import TagGroupAnd +from hindsight_client_api.models.tag_group_not import TagGroupNot +from hindsight_client_api.models.tag_group_or import TagGroupOr +# TODO: Rewrite to not use raise_errors +RecallRequestTagGroupsInner.model_rebuild(raise_errors=False) + diff --git a/hindsight-clients/python/hindsight_client_api/models/reflect_request.py b/hindsight-clients/python/hindsight_client_api/models/reflect_request.py index 30a3ed8d..e96ed3c5 100644 --- a/hindsight-clients/python/hindsight_client_api/models/reflect_request.py +++ b/hindsight-clients/python/hindsight_client_api/models/reflect_request.py @@ -20,6 +20,7 @@ import json from pydantic import BaseModel, ConfigDict, Field, StrictInt, StrictStr, field_validator from typing import Any, ClassVar, Dict, List, Optional from hindsight_client_api.models.budget import Budget +from hindsight_client_api.models.recall_request_tag_groups_inner import RecallRequestTagGroupsInner from hindsight_client_api.models.reflect_include_options import ReflectIncludeOptions from typing import Optional, Set from typing_extensions import Self @@ -36,7 +37,8 @@ class ReflectRequest(BaseModel): response_schema: Optional[Dict[str, Any]] = None tags: Optional[List[StrictStr]] = None tags_match: Optional[StrictStr] = Field(default='any', description="How to match tags: 'any' (OR, includes untagged), 'all' (AND, includes untagged), 'any_strict' (OR, excludes untagged), 'all_strict' (AND, excludes untagged).") - __properties: ClassVar[List[str]] = ["query", "budget", "context", "max_tokens", "include", "response_schema", "tags", "tags_match"] + tag_groups: Optional[List[RecallRequestTagGroupsInner]] = None + __properties: ClassVar[List[str]] = ["query", "budget", "context", "max_tokens", "include", "response_schema", "tags", "tags_match", "tag_groups"] @field_validator('tags_match') def tags_match_validate_enum(cls, value): @@ -90,6 +92,13 @@ class ReflectRequest(BaseModel): # override the default output from pydantic by calling `to_dict()` of include if self.include: _dict['include'] = self.include.to_dict() + # override the default output from pydantic by calling `to_dict()` of each item in tag_groups (list) + _items = [] + if self.tag_groups: + for _item_tag_groups in self.tag_groups: + if _item_tag_groups: + _items.append(_item_tag_groups.to_dict()) + _dict['tag_groups'] = _items # set to None if context (nullable) is None # and model_fields_set contains the field if self.context is None and "context" in self.model_fields_set: @@ -105,6 +114,11 @@ class ReflectRequest(BaseModel): if self.tags is None and "tags" in self.model_fields_set: _dict['tags'] = None + # set to None if tag_groups (nullable) is None + # and model_fields_set contains the field + if self.tag_groups is None and "tag_groups" in self.model_fields_set: + _dict['tag_groups'] = None + return _dict @classmethod @@ -124,7 +138,8 @@ class ReflectRequest(BaseModel): "include": ReflectIncludeOptions.from_dict(obj["include"]) if obj.get("include") is not None else None, "response_schema": obj.get("response_schema"), "tags": obj.get("tags"), - "tags_match": obj.get("tags_match") if obj.get("tags_match") is not None else 'any' + "tags_match": obj.get("tags_match") if obj.get("tags_match") is not None else 'any', + "tag_groups": [RecallRequestTagGroupsInner.from_dict(_item) for _item in obj["tag_groups"]] if obj.get("tag_groups") is not None else None }) return _obj diff --git a/hindsight-clients/python/hindsight_client_api/models/tag_group_and.py b/hindsight-clients/python/hindsight_client_api/models/tag_group_and.py new file mode 100644 index 00000000..fd27981b --- /dev/null +++ b/hindsight-clients/python/hindsight_client_api/models/tag_group_and.py @@ -0,0 +1,97 @@ +# coding: utf-8 + +""" + Hindsight HTTP API + + HTTP API for Hindsight + + The version of the OpenAPI document: 0.4.17 + Generated by OpenAPI Generator (https://openapi-generator.tech) + + Do not edit the class manually. +""" # noqa: E501 + + +from __future__ import annotations +import pprint +import re # noqa: F401 +import json + +from pydantic import BaseModel, ConfigDict, Field +from typing import Any, ClassVar, Dict, List +from typing import Optional, Set +from typing_extensions import Self + +class TagGroupAnd(BaseModel): + """ + Compound AND group: all child filters must match. + """ # noqa: E501 + var_and: List[RecallRequestTagGroupsInner] = Field(alias="and") + __properties: ClassVar[List[str]] = ["and"] + + model_config = ConfigDict( + populate_by_name=True, + validate_assignment=True, + protected_namespaces=(), + ) + + + def to_str(self) -> str: + """Returns the string representation of the model using alias""" + return pprint.pformat(self.model_dump(by_alias=True)) + + def to_json(self) -> str: + """Returns the JSON representation of the model using alias""" + # TODO: pydantic v2: use .model_dump_json(by_alias=True, exclude_unset=True) instead + return json.dumps(self.to_dict()) + + @classmethod + def from_json(cls, json_str: str) -> Optional[Self]: + """Create an instance of TagGroupAnd from a JSON string""" + return cls.from_dict(json.loads(json_str)) + + def to_dict(self) -> Dict[str, Any]: + """Return the dictionary representation of the model using alias. + + This has the following differences from calling pydantic's + `self.model_dump(by_alias=True)`: + + * `None` is only added to the output dict for nullable fields that + were set at model initialization. Other fields with value `None` + are ignored. + """ + excluded_fields: Set[str] = set([ + ]) + + _dict = self.model_dump( + by_alias=True, + exclude=excluded_fields, + exclude_none=True, + ) + # override the default output from pydantic by calling `to_dict()` of each item in var_and (list) + _items = [] + if self.var_and: + for _item_var_and in self.var_and: + if _item_var_and: + _items.append(_item_var_and.to_dict()) + _dict['and'] = _items + return _dict + + @classmethod + def from_dict(cls, obj: Optional[Dict[str, Any]]) -> Optional[Self]: + """Create an instance of TagGroupAnd from a dict""" + if obj is None: + return None + + if not isinstance(obj, dict): + return cls.model_validate(obj) + + _obj = cls.model_validate({ + "and": [RecallRequestTagGroupsInner.from_dict(_item) for _item in obj["and"]] if obj.get("and") is not None else None + }) + return _obj + +from hindsight_client_api.models.recall_request_tag_groups_inner import RecallRequestTagGroupsInner +# TODO: Rewrite to not use raise_errors +TagGroupAnd.model_rebuild(raise_errors=False) + diff --git a/hindsight-clients/python/hindsight_client_api/models/tag_group_leaf.py b/hindsight-clients/python/hindsight_client_api/models/tag_group_leaf.py new file mode 100644 index 00000000..9e8b0548 --- /dev/null +++ b/hindsight-clients/python/hindsight_client_api/models/tag_group_leaf.py @@ -0,0 +1,99 @@ +# coding: utf-8 + +""" + Hindsight HTTP API + + HTTP API for Hindsight + + The version of the OpenAPI document: 0.4.17 + Generated by OpenAPI Generator (https://openapi-generator.tech) + + Do not edit the class manually. +""" # noqa: E501 + + +from __future__ import annotations +import pprint +import re # noqa: F401 +import json + +from pydantic import BaseModel, ConfigDict, StrictStr, field_validator +from typing import Any, ClassVar, Dict, List, Optional +from typing import Optional, Set +from typing_extensions import Self + +class TagGroupLeaf(BaseModel): + """ + A leaf tag filter: matches memories by tag list and match mode. + """ # noqa: E501 + tags: List[StrictStr] + match: Optional[StrictStr] = 'any_strict' + __properties: ClassVar[List[str]] = ["tags", "match"] + + @field_validator('match') + def match_validate_enum(cls, value): + """Validates the enum""" + if value is None: + return value + + if value not in set(['any', 'all', 'any_strict', 'all_strict']): + raise ValueError("must be one of enum values ('any', 'all', 'any_strict', 'all_strict')") + return value + + model_config = ConfigDict( + populate_by_name=True, + validate_assignment=True, + protected_namespaces=(), + ) + + + def to_str(self) -> str: + """Returns the string representation of the model using alias""" + return pprint.pformat(self.model_dump(by_alias=True)) + + def to_json(self) -> str: + """Returns the JSON representation of the model using alias""" + # TODO: pydantic v2: use .model_dump_json(by_alias=True, exclude_unset=True) instead + return json.dumps(self.to_dict()) + + @classmethod + def from_json(cls, json_str: str) -> Optional[Self]: + """Create an instance of TagGroupLeaf from a JSON string""" + return cls.from_dict(json.loads(json_str)) + + def to_dict(self) -> Dict[str, Any]: + """Return the dictionary representation of the model using alias. + + This has the following differences from calling pydantic's + `self.model_dump(by_alias=True)`: + + * `None` is only added to the output dict for nullable fields that + were set at model initialization. Other fields with value `None` + are ignored. + """ + excluded_fields: Set[str] = set([ + ]) + + _dict = self.model_dump( + by_alias=True, + exclude=excluded_fields, + exclude_none=True, + ) + return _dict + + @classmethod + def from_dict(cls, obj: Optional[Dict[str, Any]]) -> Optional[Self]: + """Create an instance of TagGroupLeaf from a dict""" + if obj is None: + return None + + if not isinstance(obj, dict): + return cls.model_validate(obj) + + _obj = cls.model_validate({ + "tags": obj.get("tags"), + "match": obj.get("match") if obj.get("match") is not None else 'any_strict' + }) + return _obj + + diff --git a/hindsight-clients/python/hindsight_client_api/models/tag_group_not.py b/hindsight-clients/python/hindsight_client_api/models/tag_group_not.py new file mode 100644 index 00000000..03d06c88 --- /dev/null +++ b/hindsight-clients/python/hindsight_client_api/models/tag_group_not.py @@ -0,0 +1,93 @@ +# coding: utf-8 + +""" + Hindsight HTTP API + + HTTP API for Hindsight + + The version of the OpenAPI document: 0.4.17 + Generated by OpenAPI Generator (https://openapi-generator.tech) + + Do not edit the class manually. +""" # noqa: E501 + + +from __future__ import annotations +import pprint +import re # noqa: F401 +import json + +from pydantic import BaseModel, ConfigDict, Field +from typing import Any, ClassVar, Dict, List +from typing import Optional, Set +from typing_extensions import Self + +class TagGroupNot(BaseModel): + """ + Compound NOT group: child filter must NOT match. + """ # noqa: E501 + var_not: ModelNot = Field(alias="not") + __properties: ClassVar[List[str]] = ["not"] + + model_config = ConfigDict( + populate_by_name=True, + validate_assignment=True, + protected_namespaces=(), + ) + + + def to_str(self) -> str: + """Returns the string representation of the model using alias""" + return pprint.pformat(self.model_dump(by_alias=True)) + + def to_json(self) -> str: + """Returns the JSON representation of the model using alias""" + # TODO: pydantic v2: use .model_dump_json(by_alias=True, exclude_unset=True) instead + return json.dumps(self.to_dict()) + + @classmethod + def from_json(cls, json_str: str) -> Optional[Self]: + """Create an instance of TagGroupNot from a JSON string""" + return cls.from_dict(json.loads(json_str)) + + def to_dict(self) -> Dict[str, Any]: + """Return the dictionary representation of the model using alias. + + This has the following differences from calling pydantic's + `self.model_dump(by_alias=True)`: + + * `None` is only added to the output dict for nullable fields that + were set at model initialization. Other fields with value `None` + are ignored. + """ + excluded_fields: Set[str] = set([ + ]) + + _dict = self.model_dump( + by_alias=True, + exclude=excluded_fields, + exclude_none=True, + ) + # override the default output from pydantic by calling `to_dict()` of var_not + if self.var_not: + _dict['not'] = self.var_not.to_dict() + return _dict + + @classmethod + def from_dict(cls, obj: Optional[Dict[str, Any]]) -> Optional[Self]: + """Create an instance of TagGroupNot from a dict""" + if obj is None: + return None + + if not isinstance(obj, dict): + return cls.model_validate(obj) + + _obj = cls.model_validate({ + "not": ModelNot.from_dict(obj["not"]) if obj.get("not") is not None else None + }) + return _obj + +from hindsight_client_api.models.model_not import ModelNot +# TODO: Rewrite to not use raise_errors +TagGroupNot.model_rebuild(raise_errors=False) + diff --git a/hindsight-clients/python/hindsight_client_api/models/tag_group_or.py b/hindsight-clients/python/hindsight_client_api/models/tag_group_or.py new file mode 100644 index 00000000..a9c68fd7 --- /dev/null +++ b/hindsight-clients/python/hindsight_client_api/models/tag_group_or.py @@ -0,0 +1,97 @@ +# coding: utf-8 + +""" + Hindsight HTTP API + + HTTP API for Hindsight + + The version of the OpenAPI document: 0.4.17 + Generated by OpenAPI Generator (https://openapi-generator.tech) + + Do not edit the class manually. +""" # noqa: E501 + + +from __future__ import annotations +import pprint +import re # noqa: F401 +import json + +from pydantic import BaseModel, ConfigDict, Field +from typing import Any, ClassVar, Dict, List +from typing import Optional, Set +from typing_extensions import Self + +class TagGroupOr(BaseModel): + """ + Compound OR group: at least one child filter must match. + """ # noqa: E501 + var_or: List[RecallRequestTagGroupsInner] = Field(alias="or") + __properties: ClassVar[List[str]] = ["or"] + + model_config = ConfigDict( + populate_by_name=True, + validate_assignment=True, + protected_namespaces=(), + ) + + + def to_str(self) -> str: + """Returns the string representation of the model using alias""" + return pprint.pformat(self.model_dump(by_alias=True)) + + def to_json(self) -> str: + """Returns the JSON representation of the model using alias""" + # TODO: pydantic v2: use .model_dump_json(by_alias=True, exclude_unset=True) instead + return json.dumps(self.to_dict()) + + @classmethod + def from_json(cls, json_str: str) -> Optional[Self]: + """Create an instance of TagGroupOr from a JSON string""" + return cls.from_dict(json.loads(json_str)) + + def to_dict(self) -> Dict[str, Any]: + """Return the dictionary representation of the model using alias. + + This has the following differences from calling pydantic's + `self.model_dump(by_alias=True)`: + + * `None` is only added to the output dict for nullable fields that + were set at model initialization. Other fields with value `None` + are ignored. + """ + excluded_fields: Set[str] = set([ + ]) + + _dict = self.model_dump( + by_alias=True, + exclude=excluded_fields, + exclude_none=True, + ) + # override the default output from pydantic by calling `to_dict()` of each item in var_or (list) + _items = [] + if self.var_or: + for _item_var_or in self.var_or: + if _item_var_or: + _items.append(_item_var_or.to_dict()) + _dict['or'] = _items + return _dict + + @classmethod + def from_dict(cls, obj: Optional[Dict[str, Any]]) -> Optional[Self]: + """Create an instance of TagGroupOr from a dict""" + if obj is None: + return None + + if not isinstance(obj, dict): + return cls.model_validate(obj) + + _obj = cls.model_validate({ + "or": [RecallRequestTagGroupsInner.from_dict(_item) for _item in obj["or"]] if obj.get("or") is not None else None + }) + return _obj + +from hindsight_client_api.models.recall_request_tag_groups_inner import RecallRequestTagGroupsInner +# TODO: Rewrite to not use raise_errors +TagGroupOr.model_rebuild(raise_errors=False) + diff --git a/hindsight-clients/rust/src/lib.rs b/hindsight-clients/rust/src/lib.rs index 9c72f544..7632ce74 100644 --- a/hindsight-clients/rust/src/lib.rs +++ b/hindsight-clients/rust/src/lib.rs @@ -103,6 +103,7 @@ mod tests { types: None, tags: None, tags_match: types::TagsMatch::Any, + tag_groups: None, }; let recall_response = client .recall_memories(&bank_id, None, &recall_request) diff --git a/hindsight-clients/typescript/generated/types.gen.ts b/hindsight-clients/typescript/generated/types.gen.ts index 03bcbd82..c9de2955 100644 --- a/hindsight-clients/typescript/generated/types.gen.ts +++ b/hindsight-clients/typescript/generated/types.gen.ts @@ -1488,6 +1488,14 @@ export type RecallRequest = { * How to match tags: 'any' (OR, includes untagged), 'all' (AND, includes untagged), 'any_strict' (OR, excludes untagged), 'all_strict' (AND, excludes untagged). */ tags_match?: "any" | "all" | "any_strict" | "all_strict"; + /** + * Tag Groups + * + * Compound tag filter using boolean groups. Groups in the list are AND-ed. Each group is a leaf {tags, match} or compound {and: [...]}, {or: [...]}, {not: ...}. + */ + tag_groups?: Array< + TagGroupLeaf | TagGroupAnd | TagGroupOr | TagGroupNot + > | null; }; /** @@ -1791,6 +1799,14 @@ export type ReflectRequest = { * How to match tags: 'any' (OR, includes untagged), 'all' (AND, includes untagged), 'any_strict' (OR, excludes untagged), 'all_strict' (AND, excludes untagged). */ tags_match?: "any" | "all" | "any_strict" | "all_strict"; + /** + * Tag Groups + * + * Compound tag filter using boolean groups. Groups in the list are AND-ed. Each group is a leaf {tags, match} or compound {and: [...]}, {or: [...]}, {not: ...}. + */ + tag_groups?: Array< + TagGroupLeaf | TagGroupAnd | TagGroupOr | TagGroupNot + > | null; }; /** @@ -1991,6 +2007,58 @@ export type SourceFactsIncludeOptions = { max_tokens_per_observation?: number; }; +/** + * TagGroupAnd + * + * Compound AND group: all child filters must match. + */ +export type TagGroupAnd = { + /** + * And + */ + and: Array; +}; + +/** + * TagGroupLeaf + * + * A leaf tag filter: matches memories by tag list and match mode. + */ +export type TagGroupLeaf = { + /** + * Tags + */ + tags: Array; + /** + * Match + */ + match?: "any" | "all" | "any_strict" | "all_strict"; +}; + +/** + * TagGroupNot + * + * Compound NOT group: child filter must NOT match. + */ +export type TagGroupNot = { + /** + * Not + */ + not: TagGroupLeaf | TagGroupAnd | TagGroupOr | TagGroupNot; +}; + +/** + * TagGroupOr + * + * Compound OR group: at least one child filter must match. + */ +export type TagGroupOr = { + /** + * Or + */ + or: Array; +}; + /** * TagItem * diff --git a/hindsight-docs/docs/developer/api/recall.mdx b/hindsight-docs/docs/developer/api/recall.mdx index a6ef4f4b..8fee9e41 100644 --- a/hindsight-docs/docs/developer/api/recall.mdx +++ b/hindsight-docs/docs/developer/api/recall.mdx @@ -184,6 +184,66 @@ Use this for strict scope enforcement where a memory must explicitly belong to * A memory with tags `["user:alice", "team", "project:x"]` will still match a filter of `["user:alice", "team"]` under `all_strict` — extra tags on the memory are not a problem. The filter only requires the memory to contain **at least** the specified tags. ::: +### tag_groups + +`tag_groups` is a list of compound boolean tag filters. The groups in the list are AND-ed together at the top level. Each group is a recursive boolean expression: a **leaf** node `{tags, match}`, or a **compound** node `{and: [...]}`, `{or: [...]}`, or `{not: ...}`. + +`tag_groups` and `tags` / `tags_match` can be used simultaneously — they are AND-ed together. + +#### Leaf node + +```json +{ "tags": ["step:5", "step:8"], "match": "any_strict" } +``` + +`match` accepts the same values as `tags_match`: `any`, `all`, `any_strict`, `all_strict`. Defaults to `any_strict`. + +#### Compound nodes + +```json +{ "and": [ , , ... ] } +{ "or": [ , , ... ] } +{ "not": } +``` + +#### Examples + +**Step filter AND user scope** — two top-level groups AND-ed: + +```json +{ + "tag_groups": [ + { "tags": ["step:5", "step:8", "step:12"], "match": "any_strict" }, + { "tags": ["user:ep_42"], "match": "all_strict" } + ] +} +``` + +**Nested OR inside AND** — user must match, plus either step OR priority: + +```json +{ + "tag_groups": [ + { "tags": ["user:alice"], "match": "all_strict" }, + { "or": [ + { "tags": ["step:5"], "match": "any_strict" }, + { "tags": ["priority:high"], "match": "all_strict" } + ]} + ] +} +``` + +**Exclusion** — user must match, but archived memories are excluded: + +```json +{ + "tag_groups": [ + { "tags": ["user:alice"], "match": "all_strict" }, + { "not": { "tags": ["archived"], "match": "any_strict" } } + ] +} +``` + ### trace When set to `true`, the response includes a detailed debug trace covering the query embedding, entry points, per-strategy retrieval results, RRF fusion candidates, reranked results, temporal constraints detected, and per-phase timings. Has no effect on the retrieval logic itself. Useful for understanding why specific memories were or were not returned. diff --git a/hindsight-docs/static/openapi.json b/hindsight-docs/static/openapi.json index 58f75395..5a48c542 100644 --- a/hindsight-docs/static/openapi.json +++ b/hindsight-docs/static/openapi.json @@ -6543,6 +6543,34 @@ "title": "Tags Match", "description": "How to match tags: 'any' (OR, includes untagged), 'all' (AND, includes untagged), 'any_strict' (OR, excludes untagged), 'all_strict' (AND, excludes untagged).", "default": "any" + }, + "tag_groups": { + "anyOf": [ + { + "items": { + "anyOf": [ + { + "$ref": "#/components/schemas/TagGroupLeaf" + }, + { + "$ref": "#/components/schemas/TagGroupAnd" + }, + { + "$ref": "#/components/schemas/TagGroupOr" + }, + { + "$ref": "#/components/schemas/TagGroupNot" + } + ] + }, + "type": "array" + }, + { + "type": "null" + } + ], + "title": "Tag Groups", + "description": "Compound tag filter using boolean groups. Groups in the list are AND-ed. Each group is a leaf {tags, match} or compound {and: [...]}, {or: [...]}, {not: ...}." } }, "type": "object", @@ -7155,6 +7183,34 @@ "title": "Tags Match", "description": "How to match tags: 'any' (OR, includes untagged), 'all' (AND, includes untagged), 'any_strict' (OR, excludes untagged), 'all_strict' (AND, excludes untagged).", "default": "any" + }, + "tag_groups": { + "anyOf": [ + { + "items": { + "anyOf": [ + { + "$ref": "#/components/schemas/TagGroupLeaf" + }, + { + "$ref": "#/components/schemas/TagGroupAnd" + }, + { + "$ref": "#/components/schemas/TagGroupOr" + }, + { + "$ref": "#/components/schemas/TagGroupNot" + } + ] + }, + "type": "array" + }, + { + "type": "null" + } + ], + "title": "Tag Groups", + "description": "Compound tag filter using boolean groups. Groups in the list are AND-ed. Each group is a leaf {tags, match} or compound {and: [...]}, {or: [...]}, {not: ...}." } }, "type": "object", @@ -7547,6 +7603,121 @@ "title": "SourceFactsIncludeOptions", "description": "Options for including source facts for observation-type results." }, + "TagGroupAnd": { + "properties": { + "and": { + "items": { + "anyOf": [ + { + "$ref": "#/components/schemas/TagGroupLeaf" + }, + { + "$ref": "#/components/schemas/TagGroupAnd" + }, + { + "$ref": "#/components/schemas/TagGroupOr" + }, + { + "$ref": "#/components/schemas/TagGroupNot" + } + ] + }, + "type": "array", + "title": "And" + } + }, + "type": "object", + "required": [ + "and" + ], + "title": "TagGroupAnd", + "description": "Compound AND group: all child filters must match." + }, + "TagGroupLeaf": { + "properties": { + "tags": { + "items": { + "type": "string" + }, + "type": "array", + "title": "Tags" + }, + "match": { + "type": "string", + "enum": [ + "any", + "all", + "any_strict", + "all_strict" + ], + "title": "Match", + "default": "any_strict" + } + }, + "type": "object", + "required": [ + "tags" + ], + "title": "TagGroupLeaf", + "description": "A leaf tag filter: matches memories by tag list and match mode." + }, + "TagGroupNot": { + "properties": { + "not": { + "anyOf": [ + { + "$ref": "#/components/schemas/TagGroupLeaf" + }, + { + "$ref": "#/components/schemas/TagGroupAnd" + }, + { + "$ref": "#/components/schemas/TagGroupOr" + }, + { + "$ref": "#/components/schemas/TagGroupNot" + } + ], + "title": "Not" + } + }, + "type": "object", + "required": [ + "not" + ], + "title": "TagGroupNot", + "description": "Compound NOT group: child filter must NOT match." + }, + "TagGroupOr": { + "properties": { + "or": { + "items": { + "anyOf": [ + { + "$ref": "#/components/schemas/TagGroupLeaf" + }, + { + "$ref": "#/components/schemas/TagGroupAnd" + }, + { + "$ref": "#/components/schemas/TagGroupOr" + }, + { + "$ref": "#/components/schemas/TagGroupNot" + } + ] + }, + "type": "array", + "title": "Or" + } + }, + "type": "object", + "required": [ + "or" + ], + "title": "TagGroupOr", + "description": "Compound OR group: at least one child filter must match." + }, "TagItem": { "properties": { "tag": {