From 627ec5d524d753d2e9f22771f3de432908c152c5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=B2=20Boschi?= Date: Tue, 31 Mar 2026 11:42:02 +0200 Subject: [PATCH] feat: expose document_metadata in API and control plane (#798) * feat: expose document_metadata in API and control plane Add document_metadata (sourced from retain_params.metadata) to both list and get document endpoints. Display it in the control plane documents table and detail panel. Drop the unused metadata column from the documents table (was always stored as empty {}). * fix: code review fixes for document_metadata feature - Remove unnecessary `import json as _json` (json already imported at module level) - Simplify redundant truthiness checks in retain_params parsing - Regenerate OpenAPI spec and client SDKs (Python, TypeScript, Go) - Add tests for document_metadata in get_document and list_documents * feat(ui): improve documents table and detail panel - Relative timestamps with full date on hover - Remove context column from table - Metadata shown as k=v badges (blue, like tags) - Size in bytes instead of chars - Document IDs wrap instead of truncating - Detail panel wider (560px) - Retain params: context, event_date, metadata badges --- ...f8a9b0c1_drop_documents_metadata_column.py | 34 ++++++ hindsight-api-slim/hindsight_api/api/http.py | 4 + .../hindsight_api/engine/memory_engine.py | 27 ++++- .../engine/retain/fact_storage.py | 6 +- .../tests/test_document_tracking.py | 64 ++++++++++ hindsight-clients/go/api/openapi.yaml | 12 ++ .../go/model_document_response.go | 74 ++++++++++++ .../models/document_response.py | 18 ++- .../typescript/generated/types.gen.ts | 16 +++ .../src/components/documents-view.tsx | 111 +++++++++++++----- hindsight-docs/static/openapi.json | 34 ++++++ 11 files changed, 361 insertions(+), 39 deletions(-) create mode 100644 hindsight-api-slim/hindsight_api/alembic/versions/d6e7f8a9b0c1_drop_documents_metadata_column.py diff --git a/hindsight-api-slim/hindsight_api/alembic/versions/d6e7f8a9b0c1_drop_documents_metadata_column.py b/hindsight-api-slim/hindsight_api/alembic/versions/d6e7f8a9b0c1_drop_documents_metadata_column.py new file mode 100644 index 00000000..97d28d74 --- /dev/null +++ b/hindsight-api-slim/hindsight_api/alembic/versions/d6e7f8a9b0c1_drop_documents_metadata_column.py @@ -0,0 +1,34 @@ +"""Drop unused metadata column from documents table + +Revision ID: d6e7f8a9b0c1 +Revises: c2d3e4f5g6h7, c5d6e7f8a9b0 +Create Date: 2026-03-30 + +The metadata column on documents was always stored as an empty dict {}. +Actual document metadata is stored inside retain_params.metadata. +""" + +from collections.abc import Sequence + +from alembic import context, op + +revision: str = "d6e7f8a9b0c1" +down_revision: str | Sequence[str] | None = ("c2d3e4f5g6h7", "c5d6e7f8a9b0") +branch_labels: str | Sequence[str] | None = None +depends_on: str | Sequence[str] | None = None + + +def _get_schema_prefix() -> str: + """Get schema prefix for table names (required for multi-tenant support).""" + schema = context.config.get_main_option("target_schema") + return f'"{schema}".' if schema else "" + + +def upgrade() -> None: + schema = _get_schema_prefix() + op.execute(f"ALTER TABLE {schema}documents DROP COLUMN IF EXISTS metadata") + + +def downgrade() -> None: + schema = _get_schema_prefix() + op.execute(f"ALTER TABLE {schema}documents ADD COLUMN IF NOT EXISTS metadata jsonb DEFAULT '{{}}'") diff --git a/hindsight-api-slim/hindsight_api/api/http.py b/hindsight-api-slim/hindsight_api/api/http.py index 89681012..b431865f 100644 --- a/hindsight-api-slim/hindsight_api/api/http.py +++ b/hindsight-api-slim/hindsight_api/api/http.py @@ -1277,6 +1277,8 @@ class DocumentResponse(BaseModel): "updated_at": "2024-01-15T10:30:00Z", "memory_unit_count": 15, "tags": ["user_a", "session_123"], + "document_metadata": {"source": "slack", "channel": "#general"}, + "retain_params": {"context": "Team meeting notes", "event_date": "2024-01-15"}, } } ) @@ -1289,6 +1291,8 @@ class DocumentResponse(BaseModel): updated_at: str memory_unit_count: int tags: list[str] = FieldWithDefault(list, description="Tags associated with this document") + document_metadata: dict[str, Any] | None = Field(default=None, description="Document metadata") + retain_params: dict[str, Any] | None = Field(default=None, description="Parameters used during retain") class UpdateDocumentRequest(BaseModel): diff --git a/hindsight-api-slim/hindsight_api/engine/memory_engine.py b/hindsight-api-slim/hindsight_api/engine/memory_engine.py index db1aacbe..44eb69ae 100644 --- a/hindsight-api-slim/hindsight_api/engine/memory_engine.py +++ b/hindsight-api-slim/hindsight_api/engine/memory_engine.py @@ -3509,11 +3509,13 @@ class MemoryEngine(MemoryEngineInterface): doc = await conn.fetchrow( f""" SELECT d.id, d.bank_id, d.original_text, d.content_hash, - d.created_at, d.updated_at, d.tags, COUNT(mu.id) as unit_count + d.created_at, d.updated_at, d.tags, d.retain_params, + COUNT(mu.id) as unit_count FROM {fq_table("documents")} d LEFT JOIN {fq_table("memory_units")} mu ON mu.document_id = d.id WHERE d.id = $1 AND d.bank_id = $2 - GROUP BY d.id, d.bank_id, d.original_text, d.content_hash, d.created_at, d.updated_at, d.tags + GROUP BY d.id, d.bank_id, d.original_text, d.content_hash, + d.created_at, d.updated_at, d.tags, d.retain_params """, document_id, bank_id, @@ -3522,6 +3524,14 @@ class MemoryEngine(MemoryEngineInterface): if not doc: return None + retain_params_raw = doc["retain_params"] + retain_params_parsed = ( + json.loads(retain_params_raw) if isinstance(retain_params_raw, str) else retain_params_raw + ) + + # document_metadata is sourced from retain_params.metadata + document_metadata = retain_params_parsed.get("metadata") if retain_params_parsed else None + return { "id": doc["id"], "bank_id": doc["bank_id"], @@ -3531,6 +3541,8 @@ class MemoryEngine(MemoryEngineInterface): "created_at": doc["created_at"].isoformat() if doc["created_at"] else None, "updated_at": doc["updated_at"].isoformat() if doc["updated_at"] else None, "tags": list(doc["tags"]) if doc["tags"] else [], + "document_metadata": document_metadata or None, + "retain_params": retain_params_parsed or None, } async def delete_document( @@ -4950,6 +4962,14 @@ class MemoryEngine(MemoryEngineInterface): bank_id_val = row["bank_id"] unit_count = count_map.get((doc_id, bank_id_val), 0) + retain_params_val = row["retain_params"] + retain_params_val = ( + json.loads(retain_params_val) if isinstance(retain_params_val, str) else retain_params_val + ) + + # document_metadata is sourced from retain_params.metadata + document_metadata = retain_params_val.get("metadata") if retain_params_val else None + items.append( { "id": doc_id, @@ -4959,7 +4979,8 @@ class MemoryEngine(MemoryEngineInterface): "updated_at": row["updated_at"].isoformat() if row["updated_at"] else "", "text_length": row["text_length"] or 0, "memory_unit_count": unit_count, - "retain_params": row["retain_params"] if row["retain_params"] else None, + "retain_params": retain_params_val or None, + "document_metadata": document_metadata or None, "tags": row["tags"] if row["tags"] else [], } ) diff --git a/hindsight-api-slim/hindsight_api/engine/retain/fact_storage.py b/hindsight-api-slim/hindsight_api/engine/retain/fact_storage.py index a98181f7..beb127f0 100644 --- a/hindsight-api-slim/hindsight_api/engine/retain/fact_storage.py +++ b/hindsight-api-slim/hindsight_api/engine/retain/fact_storage.py @@ -288,12 +288,11 @@ async def _upsert_document_row( """Insert or update a document row.""" await conn.execute( f""" - INSERT INTO {fq_table("documents")} (id, bank_id, original_text, content_hash, metadata, retain_params, tags) - VALUES ($1, $2, $3, $4, $5, $6, $7) + INSERT INTO {fq_table("documents")} (id, bank_id, original_text, content_hash, retain_params, tags) + VALUES ($1, $2, $3, $4, $5, $6) ON CONFLICT (id, bank_id) DO UPDATE SET original_text = EXCLUDED.original_text, content_hash = EXCLUDED.content_hash, - metadata = EXCLUDED.metadata, retain_params = EXCLUDED.retain_params, tags = EXCLUDED.tags, updated_at = NOW() @@ -302,7 +301,6 @@ async def _upsert_document_row( bank_id, combined_content, content_hash, - json.dumps({}), # Empty metadata dict json.dumps(retain_params) if retain_params else None, document_tags or [], ) diff --git a/hindsight-api-slim/tests/test_document_tracking.py b/hindsight-api-slim/tests/test_document_tracking.py index e251d885..dcf40ae9 100644 --- a/hindsight-api-slim/tests/test_document_tracking.py +++ b/hindsight-api-slim/tests/test_document_tracking.py @@ -141,6 +141,70 @@ async def test_memory_without_document(memory, request_context): await memory.delete_bank(bank_id, request_context=request_context) +@pytest.mark.asyncio +async def test_document_metadata_from_retain_params(memory, request_context): + """Test that document_metadata is returned from retain_params.metadata in both get and list.""" + bank_id = f"test_doc_meta_{datetime.now(timezone.utc).timestamp()}" + + try: + document_id = "doc-with-metadata" + metadata = {"source": "slack", "channel": "#general"} + + await memory.retain_batch_async( + bank_id=bank_id, + contents=[{"content": "Alice works at Google.", "context": "Team meeting", "metadata": metadata}], + document_id=document_id, + request_context=request_context, + ) + + # get_document should include document_metadata + doc = await memory.get_document(document_id, bank_id, request_context=request_context) + assert doc is not None + assert doc["document_metadata"] == metadata + assert doc["retain_params"] is not None + assert doc["retain_params"]["metadata"] == metadata + + # list_documents should also include document_metadata + docs_list = await memory.list_documents( + bank_id=bank_id, search_query=None, limit=100, offset=0, request_context=request_context + ) + listed_doc = next(d for d in docs_list["items"] if d["id"] == document_id) + assert listed_doc["document_metadata"] == metadata + + finally: + await memory.delete_bank(bank_id, request_context=request_context) + + +@pytest.mark.asyncio +async def test_document_without_metadata(memory, request_context): + """Test that document_metadata is None when no metadata was provided during retain.""" + bank_id = f"test_doc_no_meta_{datetime.now(timezone.utc).timestamp()}" + + try: + document_id = "doc-no-metadata" + + await memory.retain_async( + bank_id=bank_id, + content="Bob works at Microsoft.", + context="Meeting", + document_id=document_id, + request_context=request_context, + ) + + doc = await memory.get_document(document_id, bank_id, request_context=request_context) + assert doc is not None + assert doc["document_metadata"] is None + + docs_list = await memory.list_documents( + bank_id=bank_id, search_query=None, limit=100, offset=0, request_context=request_context + ) + listed_doc = next(d for d in docs_list["items"] if d["id"] == document_id) + assert listed_doc["document_metadata"] is None + + finally: + await memory.delete_bank(bank_id, request_context=request_context) + + @pytest.mark.asyncio async def test_document_persisted_with_zero_facts(memory, request_context): """ diff --git a/hindsight-clients/go/api/openapi.yaml b/hindsight-clients/go/api/openapi.yaml index e111b616..54dc3408 100644 --- a/hindsight-clients/go/api/openapi.yaml +++ b/hindsight-clients/go/api/openapi.yaml @@ -3910,9 +3910,15 @@ components: bank_id: user123 content_hash: abc123 created_at: 2024-01-15T10:30:00Z + document_metadata: + channel: '#general' + source: slack id: session_1 memory_unit_count: 15 original_text: Full document text here... + retain_params: + context: Team meeting notes + event_date: 2024-01-15 tags: - user_a - session_123 @@ -3945,6 +3951,12 @@ components: items: type: string type: array + document_metadata: + additionalProperties: {} + nullable: true + retain_params: + additionalProperties: {} + nullable: true required: - bank_id - content_hash diff --git a/hindsight-clients/go/model_document_response.go b/hindsight-clients/go/model_document_response.go index 942d3619..ce1917a5 100644 --- a/hindsight-clients/go/model_document_response.go +++ b/hindsight-clients/go/model_document_response.go @@ -30,6 +30,8 @@ type DocumentResponse struct { MemoryUnitCount int32 `json:"memory_unit_count"` // Tags associated with this document Tags []string `json:"tags,omitempty"` + DocumentMetadata map[string]interface{} `json:"document_metadata,omitempty"` + RetainParams map[string]interface{} `json:"retain_params,omitempty"` } type _DocumentResponse DocumentResponse @@ -260,6 +262,72 @@ func (o *DocumentResponse) SetTags(v []string) { o.Tags = v } +// GetDocumentMetadata returns the DocumentMetadata field value if set, zero value otherwise (both if not set or set to explicit null). +func (o *DocumentResponse) GetDocumentMetadata() map[string]interface{} { + if o == nil { + var ret map[string]interface{} + return ret + } + return o.DocumentMetadata +} + +// GetDocumentMetadataOk returns a tuple with the DocumentMetadata field value if set, nil otherwise +// and a boolean to check if the value has been set. +// NOTE: If the value is an explicit nil, `nil, true` will be returned +func (o *DocumentResponse) GetDocumentMetadataOk() (map[string]interface{}, bool) { + if o == nil || IsNil(o.DocumentMetadata) { + return map[string]interface{}{}, false + } + return o.DocumentMetadata, true +} + +// HasDocumentMetadata returns a boolean if a field has been set. +func (o *DocumentResponse) HasDocumentMetadata() bool { + if o != nil && !IsNil(o.DocumentMetadata) { + return true + } + + return false +} + +// SetDocumentMetadata gets a reference to the given map[string]interface{} and assigns it to the DocumentMetadata field. +func (o *DocumentResponse) SetDocumentMetadata(v map[string]interface{}) { + o.DocumentMetadata = v +} + +// GetRetainParams returns the RetainParams field value if set, zero value otherwise (both if not set or set to explicit null). +func (o *DocumentResponse) GetRetainParams() map[string]interface{} { + if o == nil { + var ret map[string]interface{} + return ret + } + return o.RetainParams +} + +// GetRetainParamsOk returns a tuple with the RetainParams field value if set, nil otherwise +// and a boolean to check if the value has been set. +// NOTE: If the value is an explicit nil, `nil, true` will be returned +func (o *DocumentResponse) GetRetainParamsOk() (map[string]interface{}, bool) { + if o == nil || IsNil(o.RetainParams) { + return map[string]interface{}{}, false + } + return o.RetainParams, true +} + +// HasRetainParams returns a boolean if a field has been set. +func (o *DocumentResponse) HasRetainParams() bool { + if o != nil && !IsNil(o.RetainParams) { + return true + } + + return false +} + +// SetRetainParams gets a reference to the given map[string]interface{} and assigns it to the RetainParams field. +func (o *DocumentResponse) SetRetainParams(v map[string]interface{}) { + o.RetainParams = v +} + func (o DocumentResponse) MarshalJSON() ([]byte, error) { toSerialize,err := o.ToMap() if err != nil { @@ -280,6 +348,12 @@ func (o DocumentResponse) ToMap() (map[string]interface{}, error) { if !IsNil(o.Tags) { toSerialize["tags"] = o.Tags } + if o.DocumentMetadata != nil { + toSerialize["document_metadata"] = o.DocumentMetadata + } + if o.RetainParams != nil { + toSerialize["retain_params"] = o.RetainParams + } return toSerialize, nil } diff --git a/hindsight-clients/python/hindsight_client_api/models/document_response.py b/hindsight-clients/python/hindsight_client_api/models/document_response.py index 1ffe5a09..4b734e58 100644 --- a/hindsight-clients/python/hindsight_client_api/models/document_response.py +++ b/hindsight-clients/python/hindsight_client_api/models/document_response.py @@ -34,7 +34,9 @@ class DocumentResponse(BaseModel): updated_at: StrictStr memory_unit_count: StrictInt tags: Optional[List[StrictStr]] = Field(default=None, description="Tags associated with this document") - __properties: ClassVar[List[str]] = ["id", "bank_id", "original_text", "content_hash", "created_at", "updated_at", "memory_unit_count", "tags"] + document_metadata: Optional[Dict[str, Any]] = None + retain_params: Optional[Dict[str, Any]] = None + __properties: ClassVar[List[str]] = ["id", "bank_id", "original_text", "content_hash", "created_at", "updated_at", "memory_unit_count", "tags", "document_metadata", "retain_params"] model_config = ConfigDict( populate_by_name=True, @@ -80,6 +82,16 @@ class DocumentResponse(BaseModel): if self.content_hash is None and "content_hash" in self.model_fields_set: _dict['content_hash'] = None + # set to None if document_metadata (nullable) is None + # and model_fields_set contains the field + if self.document_metadata is None and "document_metadata" in self.model_fields_set: + _dict['document_metadata'] = None + + # set to None if retain_params (nullable) is None + # and model_fields_set contains the field + if self.retain_params is None and "retain_params" in self.model_fields_set: + _dict['retain_params'] = None + return _dict @classmethod @@ -99,7 +111,9 @@ class DocumentResponse(BaseModel): "created_at": obj.get("created_at"), "updated_at": obj.get("updated_at"), "memory_unit_count": obj.get("memory_unit_count"), - "tags": obj.get("tags") + "tags": obj.get("tags"), + "document_metadata": obj.get("document_metadata"), + "retain_params": obj.get("retain_params") }) return _obj diff --git a/hindsight-clients/typescript/generated/types.gen.ts b/hindsight-clients/typescript/generated/types.gen.ts index 0ccde400..9e73b5d4 100644 --- a/hindsight-clients/typescript/generated/types.gen.ts +++ b/hindsight-clients/typescript/generated/types.gen.ts @@ -954,6 +954,22 @@ export type DocumentResponse = { * Tags associated with this document */ tags?: Array; + /** + * Document Metadata + * + * Document metadata + */ + document_metadata?: { + [key: string]: unknown; + } | null; + /** + * Retain Params + * + * Parameters used during retain + */ + retain_params?: { + [key: string]: unknown; + } | null; }; /** diff --git a/hindsight-control-plane/src/components/documents-view.tsx b/hindsight-control-plane/src/components/documents-view.tsx index 68fbc492..1307b6c9 100644 --- a/hindsight-control-plane/src/components/documents-view.tsx +++ b/hindsight-control-plane/src/components/documents-view.tsx @@ -36,6 +36,48 @@ import { const ITEMS_PER_PAGE = 50; +function formatRelativeTime(dateStr: string): string { + const now = Date.now(); + const then = new Date(dateStr).getTime(); + const seconds = Math.floor((now - then) / 1000); + if (seconds < 60) return "just now"; + const minutes = Math.floor(seconds / 60); + if (minutes < 60) return `${minutes}m ago`; + const hours = Math.floor(minutes / 60); + if (hours < 24) return `${hours}h ago`; + const days = Math.floor(hours / 24); + if (days < 30) return `${days}d ago`; + const months = Math.floor(days / 30); + if (months < 12) return `${months}mo ago`; + return `${Math.floor(months / 12)}y ago`; +} + +function formatBytes(bytes: number): string { + if (bytes < 1024) return `${bytes} B`; + if (bytes < 1024 * 1024) return `${(bytes / 1024).toFixed(1)} KB`; + return `${(bytes / (1024 * 1024)).toFixed(1)} MB`; +} + +function MetadataBadges({ metadata }: { metadata: Record }) { + const entries = Object.entries(metadata); + if (entries.length === 0) return -; + return ( +
+ {entries.slice(0, 3).map(([k, v]) => ( + + {k}={String(v)} + + ))} + {entries.length > 3 && ( + +{entries.length - 3} + )} +
+ ); +} + export function DocumentsView() { const { currentBank } = useBank(); const [documents, setDocuments] = useState([]); @@ -238,8 +280,8 @@ export function DocumentsView() { Document ID Created Tags - Context - Text Length + Metadata + Size Memory Units @@ -251,11 +293,14 @@ export function DocumentsView() { className={`cursor-pointer hover:bg-muted/50 ${selectedDocument?.id === doc.id ? "bg-primary/10" : ""}`} onClick={() => viewDocumentText(doc.id)} > - - {doc.id.length > 30 ? doc.id.substring(0, 30) + "..." : doc.id} + + {doc.id} - - {doc.created_at ? new Date(doc.created_at).toLocaleString() : "N/A"} + + {doc.created_at ? formatRelativeTime(doc.created_at) : "N/A"} {doc.tags && doc.tags.length > 0 ? ( @@ -279,10 +324,15 @@ export function DocumentsView() { )} - {doc.retain_params?.context || "-"} + {doc.document_metadata && + Object.keys(doc.document_metadata).length > 0 ? ( + + ) : ( + "-" + )} - {doc.text_length?.toLocaleString()} chars + {formatBytes(doc.text_length || 0)} {doc.memory_unit_count} @@ -362,7 +412,7 @@ export function DocumentsView() { {/* Document Detail Panel - Fixed on Right */} {documents.length > 0 && selectedDocument && ( -
+
{/* Header with close button */}
@@ -424,46 +474,47 @@ export function DocumentsView() {
)} - {/* Text Length */} + {/* Text Size */} {selectedDocument.original_text && (
- Text Length + Size
- {selectedDocument.original_text.length.toLocaleString()} characters + {formatBytes(new Blob([selectedDocument.original_text]).size)}
)} {/* Retain Parameters */} {selectedDocument.retain_params && ( -
-
+
+
Retain Parameters
-
- {selectedDocument.retain_params.context && ( -
- Context:{" "} + {selectedDocument.retain_params.context && ( +
+
Context
+
{selectedDocument.retain_params.context}
- )} - {selectedDocument.retain_params.event_date && ( -
- Event Date:{" "} +
+ )} + {selectedDocument.retain_params.event_date && ( +
+
Event Date
+
{new Date(selectedDocument.retain_params.event_date).toLocaleString()}
- )} - {selectedDocument.retain_params.metadata && ( -
- Metadata: - 
-                            {JSON.stringify(selectedDocument.retain_params.metadata, null, 2)}
-
+
+ )} + {selectedDocument.retain_params.metadata && + Object.keys(selectedDocument.retain_params.metadata).length > 0 && ( +
+
Metadata
+
)} -
)} diff --git a/hindsight-docs/static/openapi.json b/hindsight-docs/static/openapi.json index 2111c994..ca5fb5e5 100644 --- a/hindsight-docs/static/openapi.json +++ b/hindsight-docs/static/openapi.json @@ -5815,6 +5815,32 @@ "title": "Tags", "description": "Tags associated with this document", "default": [] + }, + "document_metadata": { + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], + "title": "Document Metadata", + "description": "Document metadata" + }, + "retain_params": { + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], + "title": "Retain Params", + "description": "Parameters used during retain" } }, "type": "object", @@ -5833,9 +5859,17 @@ "bank_id": "user123", "content_hash": "abc123", "created_at": "2024-01-15T10:30:00Z", + "document_metadata": { + "channel": "#general", + "source": "slack" + }, "id": "session_1", "memory_unit_count": 15, "original_text": "Full document text here...", + "retain_params": { + "context": "Team meeting notes", + "event_date": "2024-01-15" + }, "tags": [ "user_a", "session_123"