diff --git a/hindsight-api-slim/hindsight_api/alembic/versions/d6e7f8a9b0c1_drop_documents_metadata_column.py b/hindsight-api-slim/hindsight_api/alembic/versions/d6e7f8a9b0c1_drop_documents_metadata_column.py new file mode 100644 index 00000000..97d28d74 --- /dev/null +++ b/hindsight-api-slim/hindsight_api/alembic/versions/d6e7f8a9b0c1_drop_documents_metadata_column.py @@ -0,0 +1,34 @@ +"""Drop unused metadata column from documents table + +Revision ID: d6e7f8a9b0c1 +Revises: c2d3e4f5g6h7, c5d6e7f8a9b0 +Create Date: 2026-03-30 + +The metadata column on documents was always stored as an empty dict {}. +Actual document metadata is stored inside retain_params.metadata. +""" + +from collections.abc import Sequence + +from alembic import context, op + +revision: str = "d6e7f8a9b0c1" +down_revision: str | Sequence[str] | None = ("c2d3e4f5g6h7", "c5d6e7f8a9b0") +branch_labels: str | Sequence[str] | None = None +depends_on: str | Sequence[str] | None = None + + +def _get_schema_prefix() -> str: + """Get schema prefix for table names (required for multi-tenant support).""" + schema = context.config.get_main_option("target_schema") + return f'"{schema}".' if schema else "" + + +def upgrade() -> None: + schema = _get_schema_prefix() + op.execute(f"ALTER TABLE {schema}documents DROP COLUMN IF EXISTS metadata") + + +def downgrade() -> None: + schema = _get_schema_prefix() + op.execute(f"ALTER TABLE {schema}documents ADD COLUMN IF NOT EXISTS metadata jsonb DEFAULT '{{}}'") diff --git a/hindsight-api-slim/hindsight_api/api/http.py b/hindsight-api-slim/hindsight_api/api/http.py index 89681012..b431865f 100644 --- a/hindsight-api-slim/hindsight_api/api/http.py +++ b/hindsight-api-slim/hindsight_api/api/http.py @@ -1277,6 +1277,8 @@ class DocumentResponse(BaseModel): "updated_at": "2024-01-15T10:30:00Z", "memory_unit_count": 15, "tags": ["user_a", "session_123"], + "document_metadata": {"source": "slack", "channel": "#general"}, + "retain_params": {"context": "Team meeting notes", "event_date": "2024-01-15"}, } } ) @@ -1289,6 +1291,8 @@ class DocumentResponse(BaseModel): updated_at: str memory_unit_count: int tags: list[str] = FieldWithDefault(list, description="Tags associated with this document") + document_metadata: dict[str, Any] | None = Field(default=None, description="Document metadata") + retain_params: dict[str, Any] | None = Field(default=None, description="Parameters used during retain") class UpdateDocumentRequest(BaseModel): diff --git a/hindsight-api-slim/hindsight_api/engine/memory_engine.py b/hindsight-api-slim/hindsight_api/engine/memory_engine.py index db1aacbe..44eb69ae 100644 --- a/hindsight-api-slim/hindsight_api/engine/memory_engine.py +++ b/hindsight-api-slim/hindsight_api/engine/memory_engine.py @@ -3509,11 +3509,13 @@ class MemoryEngine(MemoryEngineInterface): doc = await conn.fetchrow( f""" SELECT d.id, d.bank_id, d.original_text, d.content_hash, - d.created_at, d.updated_at, d.tags, COUNT(mu.id) as unit_count + d.created_at, d.updated_at, d.tags, d.retain_params, + COUNT(mu.id) as unit_count FROM {fq_table("documents")} d LEFT JOIN {fq_table("memory_units")} mu ON mu.document_id = d.id WHERE d.id = $1 AND d.bank_id = $2 - GROUP BY d.id, d.bank_id, d.original_text, d.content_hash, d.created_at, d.updated_at, d.tags + GROUP BY d.id, d.bank_id, d.original_text, d.content_hash, + d.created_at, d.updated_at, d.tags, d.retain_params """, document_id, bank_id, @@ -3522,6 +3524,14 @@ class MemoryEngine(MemoryEngineInterface): if not doc: return None + retain_params_raw = doc["retain_params"] + retain_params_parsed = ( + json.loads(retain_params_raw) if isinstance(retain_params_raw, str) else retain_params_raw + ) + + # document_metadata is sourced from retain_params.metadata + document_metadata = retain_params_parsed.get("metadata") if retain_params_parsed else None + return { "id": doc["id"], "bank_id": doc["bank_id"], @@ -3531,6 +3541,8 @@ class MemoryEngine(MemoryEngineInterface): "created_at": doc["created_at"].isoformat() if doc["created_at"] else None, "updated_at": doc["updated_at"].isoformat() if doc["updated_at"] else None, "tags": list(doc["tags"]) if doc["tags"] else [], + "document_metadata": document_metadata or None, + "retain_params": retain_params_parsed or None, } async def delete_document( @@ -4950,6 +4962,14 @@ class MemoryEngine(MemoryEngineInterface): bank_id_val = row["bank_id"] unit_count = count_map.get((doc_id, bank_id_val), 0) + retain_params_val = row["retain_params"] + retain_params_val = ( + json.loads(retain_params_val) if isinstance(retain_params_val, str) else retain_params_val + ) + + # document_metadata is sourced from retain_params.metadata + document_metadata = retain_params_val.get("metadata") if retain_params_val else None + items.append( { "id": doc_id, @@ -4959,7 +4979,8 @@ class MemoryEngine(MemoryEngineInterface): "updated_at": row["updated_at"].isoformat() if row["updated_at"] else "", "text_length": row["text_length"] or 0, "memory_unit_count": unit_count, - "retain_params": row["retain_params"] if row["retain_params"] else None, + "retain_params": retain_params_val or None, + "document_metadata": document_metadata or None, "tags": row["tags"] if row["tags"] else [], } ) diff --git a/hindsight-api-slim/hindsight_api/engine/retain/fact_storage.py b/hindsight-api-slim/hindsight_api/engine/retain/fact_storage.py index a98181f7..beb127f0 100644 --- a/hindsight-api-slim/hindsight_api/engine/retain/fact_storage.py +++ b/hindsight-api-slim/hindsight_api/engine/retain/fact_storage.py @@ -288,12 +288,11 @@ async def _upsert_document_row( """Insert or update a document row.""" await conn.execute( f""" - INSERT INTO {fq_table("documents")} (id, bank_id, original_text, content_hash, metadata, retain_params, tags) - VALUES ($1, $2, $3, $4, $5, $6, $7) + INSERT INTO {fq_table("documents")} (id, bank_id, original_text, content_hash, retain_params, tags) + VALUES ($1, $2, $3, $4, $5, $6) ON CONFLICT (id, bank_id) DO UPDATE SET original_text = EXCLUDED.original_text, content_hash = EXCLUDED.content_hash, - metadata = EXCLUDED.metadata, retain_params = EXCLUDED.retain_params, tags = EXCLUDED.tags, updated_at = NOW() @@ -302,7 +301,6 @@ async def _upsert_document_row( bank_id, combined_content, content_hash, - json.dumps({}), # Empty metadata dict json.dumps(retain_params) if retain_params else None, document_tags or [], ) diff --git a/hindsight-api-slim/tests/test_document_tracking.py b/hindsight-api-slim/tests/test_document_tracking.py index e251d885..dcf40ae9 100644 --- a/hindsight-api-slim/tests/test_document_tracking.py +++ b/hindsight-api-slim/tests/test_document_tracking.py @@ -141,6 +141,70 @@ async def test_memory_without_document(memory, request_context): await memory.delete_bank(bank_id, request_context=request_context) +@pytest.mark.asyncio +async def test_document_metadata_from_retain_params(memory, request_context): + """Test that document_metadata is returned from retain_params.metadata in both get and list.""" + bank_id = f"test_doc_meta_{datetime.now(timezone.utc).timestamp()}" + + try: + document_id = "doc-with-metadata" + metadata = {"source": "slack", "channel": "#general"} + + await memory.retain_batch_async( + bank_id=bank_id, + contents=[{"content": "Alice works at Google.", "context": "Team meeting", "metadata": metadata}], + document_id=document_id, + request_context=request_context, + ) + + # get_document should include document_metadata + doc = await memory.get_document(document_id, bank_id, request_context=request_context) + assert doc is not None + assert doc["document_metadata"] == metadata + assert doc["retain_params"] is not None + assert doc["retain_params"]["metadata"] == metadata + + # list_documents should also include document_metadata + docs_list = await memory.list_documents( + bank_id=bank_id, search_query=None, limit=100, offset=0, request_context=request_context + ) + listed_doc = next(d for d in docs_list["items"] if d["id"] == document_id) + assert listed_doc["document_metadata"] == metadata + + finally: + await memory.delete_bank(bank_id, request_context=request_context) + + +@pytest.mark.asyncio +async def test_document_without_metadata(memory, request_context): + """Test that document_metadata is None when no metadata was provided during retain.""" + bank_id = f"test_doc_no_meta_{datetime.now(timezone.utc).timestamp()}" + + try: + document_id = "doc-no-metadata" + + await memory.retain_async( + bank_id=bank_id, + content="Bob works at Microsoft.", + context="Meeting", + document_id=document_id, + request_context=request_context, + ) + + doc = await memory.get_document(document_id, bank_id, request_context=request_context) + assert doc is not None + assert doc["document_metadata"] is None + + docs_list = await memory.list_documents( + bank_id=bank_id, search_query=None, limit=100, offset=0, request_context=request_context + ) + listed_doc = next(d for d in docs_list["items"] if d["id"] == document_id) + assert listed_doc["document_metadata"] is None + + finally: + await memory.delete_bank(bank_id, request_context=request_context) + + @pytest.mark.asyncio async def test_document_persisted_with_zero_facts(memory, request_context): """ diff --git a/hindsight-clients/go/api/openapi.yaml b/hindsight-clients/go/api/openapi.yaml index e111b616..54dc3408 100644 --- a/hindsight-clients/go/api/openapi.yaml +++ b/hindsight-clients/go/api/openapi.yaml @@ -3910,9 +3910,15 @@ components: bank_id: user123 content_hash: abc123 created_at: 2024-01-15T10:30:00Z + document_metadata: + channel: '#general' + source: slack id: session_1 memory_unit_count: 15 original_text: Full document text here... + retain_params: + context: Team meeting notes + event_date: 2024-01-15 tags: - user_a - session_123 @@ -3945,6 +3951,12 @@ components: items: type: string type: array + document_metadata: + additionalProperties: {} + nullable: true + retain_params: + additionalProperties: {} + nullable: true required: - bank_id - content_hash diff --git a/hindsight-clients/go/model_document_response.go b/hindsight-clients/go/model_document_response.go index 942d3619..ce1917a5 100644 --- a/hindsight-clients/go/model_document_response.go +++ b/hindsight-clients/go/model_document_response.go @@ -30,6 +30,8 @@ type DocumentResponse struct { MemoryUnitCount int32 `json:"memory_unit_count"` // Tags associated with this document Tags []string `json:"tags,omitempty"` + DocumentMetadata map[string]interface{} `json:"document_metadata,omitempty"` + RetainParams map[string]interface{} `json:"retain_params,omitempty"` } type _DocumentResponse DocumentResponse @@ -260,6 +262,72 @@ func (o *DocumentResponse) SetTags(v []string) { o.Tags = v } +// GetDocumentMetadata returns the DocumentMetadata field value if set, zero value otherwise (both if not set or set to explicit null). +func (o *DocumentResponse) GetDocumentMetadata() map[string]interface{} { + if o == nil { + var ret map[string]interface{} + return ret + } + return o.DocumentMetadata +} + +// GetDocumentMetadataOk returns a tuple with the DocumentMetadata field value if set, nil otherwise +// and a boolean to check if the value has been set. +// NOTE: If the value is an explicit nil, `nil, true` will be returned +func (o *DocumentResponse) GetDocumentMetadataOk() (map[string]interface{}, bool) { + if o == nil || IsNil(o.DocumentMetadata) { + return map[string]interface{}{}, false + } + return o.DocumentMetadata, true +} + +// HasDocumentMetadata returns a boolean if a field has been set. +func (o *DocumentResponse) HasDocumentMetadata() bool { + if o != nil && !IsNil(o.DocumentMetadata) { + return true + } + + return false +} + +// SetDocumentMetadata gets a reference to the given map[string]interface{} and assigns it to the DocumentMetadata field. +func (o *DocumentResponse) SetDocumentMetadata(v map[string]interface{}) { + o.DocumentMetadata = v +} + +// GetRetainParams returns the RetainParams field value if set, zero value otherwise (both if not set or set to explicit null). +func (o *DocumentResponse) GetRetainParams() map[string]interface{} { + if o == nil { + var ret map[string]interface{} + return ret + } + return o.RetainParams +} + +// GetRetainParamsOk returns a tuple with the RetainParams field value if set, nil otherwise +// and a boolean to check if the value has been set. +// NOTE: If the value is an explicit nil, `nil, true` will be returned +func (o *DocumentResponse) GetRetainParamsOk() (map[string]interface{}, bool) { + if o == nil || IsNil(o.RetainParams) { + return map[string]interface{}{}, false + } + return o.RetainParams, true +} + +// HasRetainParams returns a boolean if a field has been set. +func (o *DocumentResponse) HasRetainParams() bool { + if o != nil && !IsNil(o.RetainParams) { + return true + } + + return false +} + +// SetRetainParams gets a reference to the given map[string]interface{} and assigns it to the RetainParams field. +func (o *DocumentResponse) SetRetainParams(v map[string]interface{}) { + o.RetainParams = v +} + func (o DocumentResponse) MarshalJSON() ([]byte, error) { toSerialize,err := o.ToMap() if err != nil { @@ -280,6 +348,12 @@ func (o DocumentResponse) ToMap() (map[string]interface{}, error) { if !IsNil(o.Tags) { toSerialize["tags"] = o.Tags } + if o.DocumentMetadata != nil { + toSerialize["document_metadata"] = o.DocumentMetadata + } + if o.RetainParams != nil { + toSerialize["retain_params"] = o.RetainParams + } return toSerialize, nil } diff --git a/hindsight-clients/python/hindsight_client_api/models/document_response.py b/hindsight-clients/python/hindsight_client_api/models/document_response.py index 1ffe5a09..4b734e58 100644 --- a/hindsight-clients/python/hindsight_client_api/models/document_response.py +++ b/hindsight-clients/python/hindsight_client_api/models/document_response.py @@ -34,7 +34,9 @@ class DocumentResponse(BaseModel): updated_at: StrictStr memory_unit_count: StrictInt tags: Optional[List[StrictStr]] = Field(default=None, description="Tags associated with this document") - __properties: ClassVar[List[str]] = ["id", "bank_id", "original_text", "content_hash", "created_at", "updated_at", "memory_unit_count", "tags"] + document_metadata: Optional[Dict[str, Any]] = None + retain_params: Optional[Dict[str, Any]] = None + __properties: ClassVar[List[str]] = ["id", "bank_id", "original_text", "content_hash", "created_at", "updated_at", "memory_unit_count", "tags", "document_metadata", "retain_params"] model_config = ConfigDict( populate_by_name=True, @@ -80,6 +82,16 @@ class DocumentResponse(BaseModel): if self.content_hash is None and "content_hash" in self.model_fields_set: _dict['content_hash'] = None + # set to None if document_metadata (nullable) is None + # and model_fields_set contains the field + if self.document_metadata is None and "document_metadata" in self.model_fields_set: + _dict['document_metadata'] = None + + # set to None if retain_params (nullable) is None + # and model_fields_set contains the field + if self.retain_params is None and "retain_params" in self.model_fields_set: + _dict['retain_params'] = None + return _dict @classmethod @@ -99,7 +111,9 @@ class DocumentResponse(BaseModel): "created_at": obj.get("created_at"), "updated_at": obj.get("updated_at"), "memory_unit_count": obj.get("memory_unit_count"), - "tags": obj.get("tags") + "tags": obj.get("tags"), + "document_metadata": obj.get("document_metadata"), + "retain_params": obj.get("retain_params") }) return _obj diff --git a/hindsight-clients/typescript/generated/types.gen.ts b/hindsight-clients/typescript/generated/types.gen.ts index 0ccde400..9e73b5d4 100644 --- a/hindsight-clients/typescript/generated/types.gen.ts +++ b/hindsight-clients/typescript/generated/types.gen.ts @@ -954,6 +954,22 @@ export type DocumentResponse = { * Tags associated with this document */ tags?: Array; + /** + * Document Metadata + * + * Document metadata + */ + document_metadata?: { + [key: string]: unknown; + } | null; + /** + * Retain Params + * + * Parameters used during retain + */ + retain_params?: { + [key: string]: unknown; + } | null; }; /** diff --git a/hindsight-control-plane/src/components/documents-view.tsx b/hindsight-control-plane/src/components/documents-view.tsx index 68fbc492..1307b6c9 100644 --- a/hindsight-control-plane/src/components/documents-view.tsx +++ b/hindsight-control-plane/src/components/documents-view.tsx @@ -36,6 +36,48 @@ import { const ITEMS_PER_PAGE = 50; +function formatRelativeTime(dateStr: string): string { + const now = Date.now(); + const then = new Date(dateStr).getTime(); + const seconds = Math.floor((now - then) / 1000); + if (seconds < 60) return "just now"; + const minutes = Math.floor(seconds / 60); + if (minutes < 60) return `${minutes}m ago`; + const hours = Math.floor(minutes / 60); + if (hours < 24) return `${hours}h ago`; + const days = Math.floor(hours / 24); + if (days < 30) return `${days}d ago`; + const months = Math.floor(days / 30); + if (months < 12) return `${months}mo ago`; + return `${Math.floor(months / 12)}y ago`; +} + +function formatBytes(bytes: number): string { + if (bytes < 1024) return `${bytes} B`; + if (bytes < 1024 * 1024) return `${(bytes / 1024).toFixed(1)} KB`; + return `${(bytes / (1024 * 1024)).toFixed(1)} MB`; +} + +function MetadataBadges({ metadata }: { metadata: Record }) { + const entries = Object.entries(metadata); + if (entries.length === 0) return -; + return ( +
+ {entries.slice(0, 3).map(([k, v]) => ( + + {k}={String(v)} + + ))} + {entries.length > 3 && ( + +{entries.length - 3} + )} +
+ ); +} + export function DocumentsView() { const { currentBank } = useBank(); const [documents, setDocuments] = useState([]); @@ -238,8 +280,8 @@ export function DocumentsView() { Document ID Created Tags - Context - Text Length + Metadata + Size Memory Units @@ -251,11 +293,14 @@ export function DocumentsView() { className={`cursor-pointer hover:bg-muted/50 ${selectedDocument?.id === doc.id ? "bg-primary/10" : ""}`} onClick={() => viewDocumentText(doc.id)} > - - {doc.id.length > 30 ? doc.id.substring(0, 30) + "..." : doc.id} + + {doc.id} - - {doc.created_at ? new Date(doc.created_at).toLocaleString() : "N/A"} + + {doc.created_at ? formatRelativeTime(doc.created_at) : "N/A"} {doc.tags && doc.tags.length > 0 ? ( @@ -279,10 +324,15 @@ export function DocumentsView() { )} - {doc.retain_params?.context || "-"} + {doc.document_metadata && + Object.keys(doc.document_metadata).length > 0 ? ( + + ) : ( + "-" + )} - {doc.text_length?.toLocaleString()} chars + {formatBytes(doc.text_length || 0)} {doc.memory_unit_count} @@ -362,7 +412,7 @@ export function DocumentsView() { {/* Document Detail Panel - Fixed on Right */} {documents.length > 0 && selectedDocument && ( -
+
{/* Header with close button */}
@@ -424,46 +474,47 @@ export function DocumentsView() {
)} - {/* Text Length */} + {/* Text Size */} {selectedDocument.original_text && (
- Text Length + Size
- {selectedDocument.original_text.length.toLocaleString()} characters + {formatBytes(new Blob([selectedDocument.original_text]).size)}
)} {/* Retain Parameters */} {selectedDocument.retain_params && ( -
-
+
+
Retain Parameters
-
- {selectedDocument.retain_params.context && ( -
- Context:{" "} + {selectedDocument.retain_params.context && ( +
+
Context
+
{selectedDocument.retain_params.context}
- )} - {selectedDocument.retain_params.event_date && ( -
- Event Date:{" "} +
+ )} + {selectedDocument.retain_params.event_date && ( +
+
Event Date
+
{new Date(selectedDocument.retain_params.event_date).toLocaleString()}
- )} - {selectedDocument.retain_params.metadata && ( -
- Metadata: - 
-                            {JSON.stringify(selectedDocument.retain_params.metadata, null, 2)}
-
+
+ )} + {selectedDocument.retain_params.metadata && + Object.keys(selectedDocument.retain_params.metadata).length > 0 && ( +
+
Metadata
+
)} -
)} diff --git a/hindsight-docs/static/openapi.json b/hindsight-docs/static/openapi.json index 2111c994..ca5fb5e5 100644 --- a/hindsight-docs/static/openapi.json +++ b/hindsight-docs/static/openapi.json @@ -5815,6 +5815,32 @@ "title": "Tags", "description": "Tags associated with this document", "default": [] + }, + "document_metadata": { + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], + "title": "Document Metadata", + "description": "Document metadata" + }, + "retain_params": { + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], + "title": "Retain Params", + "description": "Parameters used during retain" } }, "type": "object", @@ -5833,9 +5859,17 @@ "bank_id": "user123", "content_hash": "abc123", "created_at": "2024-01-15T10:30:00Z", + "document_metadata": { + "channel": "#general", + "source": "slack" + }, "id": "session_1", "memory_unit_count": 15, "original_text": "Full document text here...", + "retain_params": { + "context": "Team meeting notes", + "event_date": "2024-01-15" + }, "tags": [ "user_a", "session_123"