From f903948a263b683bc54257038aef8be690552efd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=B2=20Boschi?= Date: Mon, 2 Mar 2026 12:03:23 +0100 Subject: [PATCH] feat: support timestamp="unset" to retain content without a date (#465) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat: support timestamp="unset" to retain content without a date When callers retain timeless content (e.g. fictional documents, static reference material), passing timestamp="unset" now skips the utcnow() default so mentioned_at is stored as NULL instead of an artificial date. - HTTP: validate_timestamp recognises "unset" sentinel and threads it through api_retain as event_date=None (key present, value None), which the orchestrator distinguishes from key-absent (still defaults to now) - Orchestrator: new branching logic separates "key absent" → utcnow() from "key present but None" → no date - types.py: RetainContent.event_date and ProcessedFact.mentioned_at are now datetime | None; removed the unused _now_utc factory - fact_extraction.py: all event_date params accept datetime | None; _build_user_message emits "Event Date: Unknown" when None; removed mentioned_at from the Fact LLM response model (LLM never sets it) - embedding_processing: skip date suffix when fact_date is None - entity_resolver: COALESCE(event_date, now()) for first_seen/last_seen so entities table NOT NULL constraint is preserved - link_utils: skip temporal linking for units without event_date - Migration aa2b3c4d5e6f: DROP NOT NULL on memory_units.event_date - Tests: test_retain_no_timestamp and test_retain_omit_timestamp_defaults_to_now - Docs + OpenAPI + TypeScript client updated * refactor: replace _TIMESTAMP_UNKNOWN sentinel with plain string comparison The sentinel object() was only needed to distinguish "unset" from None at the boundary — but since the field type is datetime | str | None, "unset" can pass through the validator unchanged and be compared directly. * chore: regenerate OpenAPI spec and clients after timestamp type change timestamp field is now datetime | str | None to accept the "unset" sentinel value. --- .../aa2b3c4d5e6f_nullable_event_date.py | 36 +++++ hindsight-api/hindsight_api/api/http.py | 18 ++- .../hindsight_api/engine/entity_resolver.py | 4 +- .../engine/retain/embedding_processing.py | 9 +- .../engine/retain/fact_extraction.py | 34 ++-- .../hindsight_api/engine/retain/link_utils.py | 44 ++++-- .../engine/retain/orchestrator.py | 12 +- .../hindsight_api/engine/retain/types.py | 17 +- hindsight-api/tests/test_retain.py | 119 ++++++++++++++ hindsight-clients/go/api/openapi.yaml | 15 +- hindsight-clients/go/model_memory_item.go | 13 +- hindsight-clients/go/model_timestamp.go | 113 ++++++++++++++ .../python/.openapi-generator/FILES | 1 + .../python/hindsight_client_api/__init__.py | 1 + .../hindsight_client_api/models/__init__.py | 1 + .../models/memory_item.py | 9 +- .../hindsight_client_api/models/timestamp.py | 145 ++++++++++++++++++ .../typescript/generated/types.gen.ts | 2 + hindsight-docs/docs/developer/api/retain.mdx | 10 +- hindsight-docs/static/openapi.json | 6 +- 20 files changed, 540 insertions(+), 69 deletions(-) create mode 100644 hindsight-api/hindsight_api/alembic/versions/aa2b3c4d5e6f_nullable_event_date.py create mode 100644 hindsight-clients/go/model_timestamp.go create mode 100644 hindsight-clients/python/hindsight_client_api/models/timestamp.py diff --git a/hindsight-api/hindsight_api/alembic/versions/aa2b3c4d5e6f_nullable_event_date.py b/hindsight-api/hindsight_api/alembic/versions/aa2b3c4d5e6f_nullable_event_date.py new file mode 100644 index 00000000..b3444ee8 --- /dev/null +++ b/hindsight-api/hindsight_api/alembic/versions/aa2b3c4d5e6f_nullable_event_date.py @@ -0,0 +1,36 @@ +"""Make event_date nullable in memory_units to support timestamp-free content + +Revision ID: aa2b3c4d5e6f +Revises: z1u2v3w4x5y6 +Create Date: 2026-03-02 + +When callers retain content without a timestamp (e.g. fictional documents, static text), +the event_date column should be allowed to be NULL rather than defaulting to utcnow(). +""" + +from collections.abc import Sequence + +from alembic import context, op + +revision: str = "aa2b3c4d5e6f" +down_revision: str | Sequence[str] | None = "z1u2v3w4x5y6" +branch_labels: str | Sequence[str] | None = None +depends_on: str | Sequence[str] | None = None + + +def _get_schema_prefix() -> str: + """Get schema prefix for table names (required for multi-tenant support).""" + schema = context.config.get_main_option("target_schema") + return f'"{schema}".' if schema else "" + + +def upgrade() -> None: + schema = _get_schema_prefix() + op.execute(f"ALTER TABLE {schema}memory_units ALTER COLUMN event_date DROP NOT NULL") + + +def downgrade() -> None: + schema = _get_schema_prefix() + # Backfill NULLs with now() before restoring the NOT NULL constraint + op.execute(f"UPDATE {schema}memory_units SET event_date = now() WHERE event_date IS NULL") + op.execute(f"ALTER TABLE {schema}memory_units ALTER COLUMN event_date SET NOT NULL") diff --git a/hindsight-api/hindsight_api/api/http.py b/hindsight-api/hindsight_api/api/http.py index 1af29103..f4684901 100644 --- a/hindsight-api/hindsight_api/api/http.py +++ b/hindsight-api/hindsight_api/api/http.py @@ -383,7 +383,15 @@ class MemoryItem(BaseModel): ) content: str - timestamp: datetime | None = None + timestamp: datetime | str | None = Field( + default=None, + description=( + "When the content occurred. " + "Accepts an ISO 8601 datetime string (e.g. '2024-01-15T10:30:00Z'), null/omitted (defaults to now), " + "or the special string 'unset' to explicitly store without any timestamp " + "(use this for timeless content such as fictional documents or static reference material)." + ), + ) context: str | None = None metadata: dict[str, str] | None = None document_id: str | None = Field(default=None, description="Optional document ID for this memory item.") @@ -414,12 +422,14 @@ class MemoryItem(BaseModel): if isinstance(v, datetime): return v if isinstance(v, str): + if v.lower() == "unset": + return "unset" try: # Try parsing as ISO format return datetime.fromisoformat(v.replace("Z", "+00:00")) except ValueError as e: raise ValueError( - f"Invalid timestamp/event_date format: '{v}'. Expected ISO format like '2024-01-15T10:30:00' or '2024-01-15T10:30:00Z'" + f"Invalid timestamp/event_date format: '{v}'. Expected ISO format like '2024-01-15T10:30:00' or '2024-01-15T10:30:00Z', or the special value 'unset' to store without a timestamp." ) from e raise ValueError(f"timestamp must be a string or datetime, got {type(v).__name__}") @@ -3878,7 +3888,9 @@ def _register_routes(app: FastAPI): contents = [] for item in request.items: content_dict = {"content": item.content} - if item.timestamp: + if item.timestamp == "unset": + content_dict["event_date"] = None + elif item.timestamp: content_dict["event_date"] = item.timestamp if item.context: content_dict["context"] = item.context diff --git a/hindsight-api/hindsight_api/engine/entity_resolver.py b/hindsight-api/hindsight_api/engine/entity_resolver.py index 1acfe3da..1d4feb50 100644 --- a/hindsight-api/hindsight_api/engine/entity_resolver.py +++ b/hindsight-api/hindsight_api/engine/entity_resolver.py @@ -237,7 +237,7 @@ class EntityResolver: rows = await conn.fetch( f""" INSERT INTO {fq_table("entities")} (bank_id, canonical_name, first_seen, last_seen, mention_count) - SELECT $1, name, event_date, event_date, cnt + SELECT $1, name, COALESCE(event_date, now()), COALESCE(event_date, now()), cnt FROM unnest($2::text[], $3::timestamptz[], $4::int[]) AS t(name, event_date, cnt) ON CONFLICT (bank_id, LOWER(canonical_name)) DO UPDATE SET @@ -408,7 +408,7 @@ class EntityResolver: entity_id = await conn.fetchval( f""" INSERT INTO {fq_table("entities")} (bank_id, canonical_name, first_seen, last_seen, mention_count) - VALUES ($1, $2, $3, $4, 1) + VALUES ($1, $2, COALESCE($3, now()), COALESCE($4, now()), 1) ON CONFLICT (bank_id, LOWER(canonical_name)) DO UPDATE SET mention_count = {fq_table("entities")}.mention_count + 1, diff --git a/hindsight-api/hindsight_api/engine/retain/embedding_processing.py b/hindsight-api/hindsight_api/engine/retain/embedding_processing.py index 0ee63f6b..7956184e 100644 --- a/hindsight-api/hindsight_api/engine/retain/embedding_processing.py +++ b/hindsight-api/hindsight_api/engine/retain/embedding_processing.py @@ -29,9 +29,12 @@ def augment_texts_with_dates(facts: list[ExtractedFact], format_date_fn) -> list for fact in facts: # Use occurred_start as the representative date fact_date = fact.occurred_start or fact.mentioned_at - readable_date = format_date_fn(fact_date) - # Augment text with date for embedding (but store original text in DB) - augmented_text = f"{fact.fact_text} (happened in {readable_date})" + if fact_date is not None: + readable_date = format_date_fn(fact_date) + # Augment text with date for embedding (but store original text in DB) + augmented_text = f"{fact.fact_text} (happened in {readable_date})" + else: + augmented_text = fact.fact_text augmented_texts.append(augmented_text) return augmented_texts diff --git a/hindsight-api/hindsight_api/engine/retain/fact_extraction.py b/hindsight-api/hindsight_api/engine/retain/fact_extraction.py index 643a79f7..af480129 100644 --- a/hindsight-api/hindsight_api/engine/retain/fact_extraction.py +++ b/hindsight-api/hindsight_api/engine/retain/fact_extraction.py @@ -19,13 +19,16 @@ from ..llm_wrapper import LLMConfig, OutputTooLongError from ..response_models import TokenUsage -def _infer_temporal_date(fact_text: str, event_date: datetime) -> str | None: +def _infer_temporal_date(fact_text: str, event_date: datetime | None) -> str | None: """ Infer a temporal date from fact text when LLM didn't provide occurred_start. This is a fallback for when the LLM fails to extract temporal information from relative time expressions like "last night", "yesterday", etc. """ + if event_date is None: + return None + fact_lower = fact_text.lower() # Map relative time expressions to day offsets @@ -100,7 +103,6 @@ class Fact(BaseModel): # Optional temporal fields occurred_start: str | None = None occurred_end: str | None = None - mentioned_at: str | None = None # Optional location field where: str | None = Field( @@ -747,7 +749,7 @@ def _build_user_message( chunk: str, chunk_index: int, total_chunks: int, - event_date: datetime, + event_date: datetime | None, context: str, metadata: dict[str, str] | None = None, ) -> str: @@ -756,8 +758,12 @@ def _build_user_message( sanitized_chunk = _sanitize_text(chunk) sanitized_context = _sanitize_text(context) if context else "none" - event_date = parse_datetime_flexible(event_date) - event_date_formatted = event_date.strftime("%A, %B %d, %Y") + + if event_date is not None: + event_date = parse_datetime_flexible(event_date) + event_date_str = f"{event_date.strftime('%A, %B %d, %Y')} ({event_date.isoformat()})" + else: + event_date_str = "Unknown" metadata_section = "" if metadata: @@ -767,7 +773,7 @@ def _build_user_message( return f"""Extract facts from the following text chunk. Chunk: {chunk_index + 1}/{total_chunks} -Event Date: {event_date_formatted} ({event_date.isoformat()}) +Event Date: {event_date_str} Context: {sanitized_context}{metadata_section} Text: @@ -805,7 +811,7 @@ async def _extract_facts_from_chunk( chunk: str, chunk_index: int, total_chunks: int, - event_date: datetime, + event_date: datetime | None, context: str, llm_config: "LLMConfig", config, @@ -1043,8 +1049,9 @@ async def _extract_facts_from_chunk( if validated_relations: fact_data["causal_relations"] = validated_relations - # Always set mentioned_at to the event_date (when the conversation/document occurred) - fact_data["mentioned_at"] = event_date.isoformat() + # Set mentioned_at to the event_date (when the conversation/document occurred), + # or None when the caller opted into no timestamp. + fact_data["mentioned_at"] = event_date.isoformat() if event_date is not None else None # Build Fact model instance try: @@ -1107,7 +1114,7 @@ async def _extract_facts_with_auto_split( chunk: str, chunk_index: int, total_chunks: int, - event_date: datetime, + event_date: datetime | None, context: str, llm_config: LLMConfig, config, @@ -1226,7 +1233,7 @@ async def _extract_facts_with_auto_split( async def extract_facts_from_text( text: str, - event_date: datetime, + event_date: datetime | None, llm_config: LLMConfig, agent_name: str, config, @@ -1641,8 +1648,9 @@ async def extract_facts_from_contents_batch_api( if validated_relations: fact_data["causal_relations"] = validated_relations - # Always set mentioned_at - fact_data["mentioned_at"] = event_date.isoformat() + # Set mentioned_at to the event_date (when the conversation/document occurred), + # or None when the caller opted into no timestamp. + fact_data["mentioned_at"] = event_date.isoformat() if event_date is not None else None try: fact = Fact(fact=combined_text, fact_type=fact_type, **fact_data) diff --git a/hindsight-api/hindsight_api/engine/retain/link_utils.py b/hindsight-api/hindsight_api/engine/retain/link_utils.py index 79dce42e..3985d80b 100644 --- a/hindsight-api/hindsight_api/engine/retain/link_utils.py +++ b/hindsight-api/hindsight_api/engine/retain/link_utils.py @@ -47,6 +47,9 @@ def compute_temporal_links( links = [] for unit_id, unit_event_date in new_units.items(): + # Units without event_date can't form temporal links + if unit_event_date is None: + continue # Normalize unit_event_date for consistent comparison unit_event_date_norm = _normalize_datetime(unit_event_date) @@ -96,7 +99,11 @@ def compute_temporal_query_bounds( return None, None # Normalize all dates to be timezone-aware to avoid comparison issues - all_dates = [_normalize_datetime(d) for d in new_units.values()] + # Filter out None values — units without event_date can't form temporal links + all_dates = [_normalize_datetime(d) for d in new_units.values() if d is not None] + + if not all_dates: + return None, None try: min_date = min(all_dates) - timedelta(hours=time_window_hours) @@ -432,20 +439,23 @@ async def create_temporal_links_batch_per_fact( min_date, max_date = compute_temporal_query_bounds(new_units, time_window_hours) fetch_neighbors_start = time_mod.time() - all_candidates = await conn.fetch( - f""" - SELECT id, event_date - FROM {fq_table("memory_units")} - WHERE bank_id = $1 - AND event_date BETWEEN $2 AND $3 - AND id::text != ALL($4) - ORDER BY event_date DESC - """, - bank_id, - min_date, - max_date, - unit_ids, - ) + if min_date is not None and max_date is not None: + all_candidates = await conn.fetch( + f""" + SELECT id, event_date + FROM {fq_table("memory_units")} + WHERE bank_id = $1 + AND event_date BETWEEN $2 AND $3 + AND id::text != ALL($4) + ORDER BY event_date DESC + """, + bank_id, + min_date, + max_date, + unit_ids, + ) + else: + all_candidates = [] _log( log_buffer, f" [7.2] Fetch {len(all_candidates)} candidate neighbors (1 query): {time_mod.time() - fetch_neighbors_start:.3f}s", @@ -460,11 +470,15 @@ async def create_temporal_links_batch_per_fact( # Convert new_units dict to candidate format for within-batch linking new_unit_items = list(new_units.items()) for i, (unit_id, event_date) in enumerate(new_unit_items): + if event_date is None: + continue # Skip units without event_date for temporal linking unit_event_date_norm = _normalize_datetime(event_date) # Compare with other new units (only those after this one to avoid duplicates) for j in range(i + 1, len(new_unit_items)): other_id, other_event_date = new_unit_items[j] + if other_event_date is None: + continue # Skip units without event_date other_event_date_norm = _normalize_datetime(other_event_date) # Check if within time window diff --git a/hindsight-api/hindsight_api/engine/retain/orchestrator.py b/hindsight-api/hindsight_api/engine/retain/orchestrator.py index c84fb761..223707d3 100644 --- a/hindsight-api/hindsight_api/engine/retain/orchestrator.py +++ b/hindsight-api/hindsight_api/engine/retain/orchestrator.py @@ -128,12 +128,14 @@ async def retain_batch( item_tags = item.get("tags", []) or [] merged_tags = list(set(item_tags + (document_tags or []))) - # Handle event_date: parse flexibly (handles both datetime objects and ISO strings) - event_date_value = item.get("event_date") - if event_date_value: - event_date_value = parse_datetime_flexible(event_date_value) + # Handle event_date: distinguish "not provided" (default to now) from + # "explicitly None" (caller opted into no timestamp). + if "event_date" in item and item["event_date"] is None: + event_date_value = None # Caller explicitly signalled "unknown date" + elif item.get("event_date"): + event_date_value = parse_datetime_flexible(item["event_date"]) else: - event_date_value = utcnow() + event_date_value = utcnow() # Backward-compatible default content = RetainContent( content=item["content"], diff --git a/hindsight-api/hindsight_api/engine/retain/types.py b/hindsight-api/hindsight_api/engine/retain/types.py index b58811b4..e328cece 100644 --- a/hindsight-api/hindsight_api/engine/retain/types.py +++ b/hindsight-api/hindsight_api/engine/retain/types.py @@ -6,7 +6,7 @@ from content input to fact storage. """ from dataclasses import dataclass, field -from datetime import UTC, datetime +from datetime import datetime from typing import Literal, TypedDict from uuid import UUID @@ -29,7 +29,7 @@ class RetainContentDict(TypedDict, total=False): content: str # Required context: str - event_date: datetime + event_date: datetime | None metadata: dict[str, str] document_id: str entities: list[dict[str, str]] # [{"text": "...", "type": "..."}] @@ -39,11 +39,6 @@ class RetainContentDict(TypedDict, total=False): ) # Observation scopes for consolidation -def _now_utc() -> datetime: - """Factory function for default event_date.""" - return datetime.now(UTC) - - @dataclass class RetainContent: """ @@ -54,7 +49,7 @@ class RetainContent: content: str context: str = "" - event_date: datetime = field(default_factory=_now_utc) + event_date: datetime | None = None metadata: dict[str, str] = field(default_factory=dict) entities: list[dict[str, str]] = field(default_factory=list) # User-provided entities tags: list[str] = field(default_factory=list) # Visibility scope tags @@ -147,7 +142,7 @@ class ProcessedFact: # Temporal data occurred_start: datetime | None occurred_end: datetime | None - mentioned_at: datetime + mentioned_at: datetime | None # Context and metadata context: str @@ -200,12 +195,10 @@ class ProcessedFact: Returns: ProcessedFact ready for storage """ - from datetime import datetime - # Use occurred dates only if explicitly provided by LLM occurred_start = extracted_fact.occurred_start occurred_end = extracted_fact.occurred_end - mentioned_at = extracted_fact.mentioned_at or datetime.now(UTC) + mentioned_at = extracted_fact.mentioned_at # May be None when caller opted into no timestamp # Convert entity strings to EntityRef objects entities = [EntityRef(name=name) for name in extracted_fact.entities] diff --git a/hindsight-api/tests/test_retain.py b/hindsight-api/tests/test_retain.py index 3eb2ab88..cfc203ab 100644 --- a/hindsight-api/tests/test_retain.py +++ b/hindsight-api/tests/test_retain.py @@ -591,6 +591,125 @@ async def test_mentioned_at_from_context_string(memory, request_context): await memory.delete_bank(bank_id, request_context=request_context) +# ============================================================ +# No Timestamp Tests +# ============================================================ + + +@pytest.mark.asyncio +async def test_retain_no_timestamp(memory, request_context): + """ + Test retaining content with explicit "no timestamp" sentinel. + + When event_date=None is passed explicitly in the dict (i.e. caller opted into + no timestamp), mentioned_at should be NULL in the DB rather than defaulting to now(). + """ + bank_id = f"test_no_timestamp_{datetime.now(timezone.utc).timestamp()}" + + try: + # Use retain_batch_async with explicit event_date=None key to signal "no timestamp" + unit_ids_list = await memory.retain_batch_async( + bank_id=bank_id, + contents=[ + { + "content": "The capital of France is Paris. The Eiffel Tower is located in Paris.", + "context": "general knowledge", + "event_date": None, # Explicit sentinel: no timestamp + } + ], + request_context=request_context, + ) + + assert len(unit_ids_list) > 0, "Should create at least one batch result" + unit_ids = unit_ids_list[0] + assert len(unit_ids) > 0, "Should have extracted and stored facts" + + # Recall the facts + result = await memory.recall_async( + bank_id=bank_id, + query="Where is the Eiffel Tower?", + budget=Budget.LOW, + max_tokens=500, + fact_type=["world"], + request_context=request_context, + ) + + assert len(result.results) > 0, "Should recall the stored fact" + + # All temporal fields should be None for temporally agnostic content + for fact in result.results: + assert fact.mentioned_at is None, ( + f"mentioned_at should be None for no-timestamp content, got {fact.mentioned_at}" + ) + + print(f"\n✓ Test passed: mentioned_at is None for {len(result.results)} fact(s)") + + finally: + await memory.delete_bank(bank_id, request_context=request_context) + + +@pytest.mark.asyncio +async def test_retain_omit_timestamp_defaults_to_now(memory, request_context): + """ + Backward-compatibility regression test: omitting event_date still stores a real datetime. + + When event_date is absent from the content dict (key not present), the orchestrator + should default to utcnow() — preserving existing behavior. + """ + bank_id = f"test_default_timestamp_{datetime.now(timezone.utc).timestamp()}" + before = datetime.now(timezone.utc) + + try: + # Omit event_date entirely — should default to now() + unit_ids_list = await memory.retain_batch_async( + bank_id=bank_id, + contents=[ + { + "content": "Alice is a software engineer who loves Python.", + "context": "profile", + # event_date intentionally omitted + } + ], + request_context=request_context, + ) + + after = datetime.now(timezone.utc) + + assert len(unit_ids_list) > 0 + unit_ids = unit_ids_list[0] + assert len(unit_ids) > 0, "Should have extracted and stored facts" + + # Recall and verify mentioned_at is a real datetime close to now + result = await memory.recall_async( + bank_id=bank_id, + query="Who is Alice?", + budget=Budget.LOW, + max_tokens=500, + fact_type=["world"], + request_context=request_context, + ) + + assert len(result.results) > 0, "Should recall the fact" + fact = result.results[0] + + assert fact.mentioned_at is not None, "mentioned_at should be set when event_date is omitted" + + if isinstance(fact.mentioned_at, str): + mentioned_dt = datetime.fromisoformat(fact.mentioned_at.replace("Z", "+00:00")) + else: + mentioned_dt = fact.mentioned_at + + # Should be within 60s of when we ran the test + assert before <= mentioned_dt <= after + timedelta(seconds=60), ( + f"mentioned_at {mentioned_dt} should be close to now ({before} – {after})" + ) + + print(f"\n✓ Test passed: mentioned_at={mentioned_dt} is a real datetime (backward compat)") + + finally: + await memory.delete_bank(bank_id, request_context=request_context) + + # ============================================================ # Context Tracking Tests # ============================================================ diff --git a/hindsight-clients/go/api/openapi.yaml b/hindsight-clients/go/api/openapi.yaml index f64cb5d6..3a5a8285 100644 --- a/hindsight-clients/go/api/openapi.yaml +++ b/hindsight-clients/go/api/openapi.yaml @@ -3477,9 +3477,7 @@ components: title: Content type: string timestamp: - format: date-time - nullable: true - type: string + $ref: '#/components/schemas/Timestamp' context: nullable: true type: string @@ -4459,6 +4457,17 @@ components: - api_version - features title: VersionResponse + Timestamp: + anyOf: + - format: date-time + type: string + - type: string + description: "When the content occurred. Accepts an ISO 8601 datetime string\ + \ (e.g. '2024-01-15T10:30:00Z'), null/omitted (defaults to now), or the special\ + \ string 'unset' to explicitly store without any timestamp (use this for timeless\ + \ content such as fictional documents or static reference material)." + nullable: true + title: Timestamp ObservationScopes: anyOf: - enum: diff --git a/hindsight-clients/go/model_memory_item.go b/hindsight-clients/go/model_memory_item.go index 8ea9b60a..ab91c6b5 100644 --- a/hindsight-clients/go/model_memory_item.go +++ b/hindsight-clients/go/model_memory_item.go @@ -12,7 +12,6 @@ package hindsight import ( "encoding/json" - "time" "bytes" "fmt" ) @@ -23,7 +22,7 @@ var _ MappedNullable = &MemoryItem{} // MemoryItem Single memory item for retain. type MemoryItem struct { Content string `json:"content"` - Timestamp NullableTime `json:"timestamp,omitempty"` + Timestamp NullableTimestamp `json:"timestamp,omitempty"` Context NullableString `json:"context,omitempty"` Metadata map[string]string `json:"metadata,omitempty"` DocumentId NullableString `json:"document_id,omitempty"` @@ -77,9 +76,9 @@ func (o *MemoryItem) SetContent(v string) { } // GetTimestamp returns the Timestamp field value if set, zero value otherwise (both if not set or set to explicit null). -func (o *MemoryItem) GetTimestamp() time.Time { +func (o *MemoryItem) GetTimestamp() Timestamp { if o == nil || IsNil(o.Timestamp.Get()) { - var ret time.Time + var ret Timestamp return ret } return *o.Timestamp.Get() @@ -88,7 +87,7 @@ func (o *MemoryItem) GetTimestamp() time.Time { // GetTimestampOk returns a tuple with the Timestamp field value if set, nil otherwise // and a boolean to check if the value has been set. // NOTE: If the value is an explicit nil, `nil, true` will be returned -func (o *MemoryItem) GetTimestampOk() (*time.Time, bool) { +func (o *MemoryItem) GetTimestampOk() (*Timestamp, bool) { if o == nil { return nil, false } @@ -104,8 +103,8 @@ func (o *MemoryItem) HasTimestamp() bool { return false } -// SetTimestamp gets a reference to the given NullableTime and assigns it to the Timestamp field. -func (o *MemoryItem) SetTimestamp(v time.Time) { +// SetTimestamp gets a reference to the given NullableTimestamp and assigns it to the Timestamp field. +func (o *MemoryItem) SetTimestamp(v Timestamp) { o.Timestamp.Set(&v) } // SetTimestampNil sets the value for Timestamp to be an explicit nil diff --git a/hindsight-clients/go/model_timestamp.go b/hindsight-clients/go/model_timestamp.go new file mode 100644 index 00000000..fc15a3fc --- /dev/null +++ b/hindsight-clients/go/model_timestamp.go @@ -0,0 +1,113 @@ +/* +Hindsight HTTP API + +HTTP API for Hindsight + +API version: 0.4.14 +*/ + +// Code generated by OpenAPI Generator (https://openapi-generator.tech); DO NOT EDIT. + +package hindsight + +import ( + "encoding/json" + "time" + "fmt" +) + + +// Timestamp When the content occurred. Accepts an ISO 8601 datetime string (e.g. '2024-01-15T10:30:00Z'), null/omitted (defaults to now), or the special string 'unset' to explicitly store without any timestamp (use this for timeless content such as fictional documents or static reference material). +type Timestamp struct { + String *string + TimeTime *time.Time +} + +// Unmarshal JSON data into any of the pointers in the struct +func (dst *Timestamp) UnmarshalJSON(data []byte) error { + var err error + // this object is nullable so check if the payload is null or empty string + if string(data) == "" || string(data) == "{}" { + return nil + } + + // try to unmarshal JSON data into String + err = json.Unmarshal(data, &dst.String); + if err == nil { + jsonString, _ := json.Marshal(dst.String) + if string(jsonString) == "{}" { // empty struct + dst.String = nil + } else { + return nil // data stored in dst.String, return on the first match + } + } else { + dst.String = nil + } + + // try to unmarshal JSON data into TimeTime + err = json.Unmarshal(data, &dst.TimeTime); + if err == nil { + jsonTimeTime, _ := json.Marshal(dst.TimeTime) + if string(jsonTimeTime) == "{}" { // empty struct + dst.TimeTime = nil + } else { + return nil // data stored in dst.TimeTime, return on the first match + } + } else { + dst.TimeTime = nil + } + + return fmt.Errorf("data failed to match schemas in anyOf(Timestamp)") +} + +// Marshal data from the first non-nil pointers in the struct to JSON +func (src *Timestamp) MarshalJSON() ([]byte, error) { + if src.String != nil { + return json.Marshal(&src.String) + } + + if src.TimeTime != nil { + return json.Marshal(&src.TimeTime) + } + + return nil, nil // no data in anyOf schemas +} + + +type NullableTimestamp struct { + value *Timestamp + isSet bool +} + +func (v NullableTimestamp) Get() *Timestamp { + return v.value +} + +func (v *NullableTimestamp) Set(val *Timestamp) { + v.value = val + v.isSet = true +} + +func (v NullableTimestamp) IsSet() bool { + return v.isSet +} + +func (v *NullableTimestamp) Unset() { + v.value = nil + v.isSet = false +} + +func NewNullableTimestamp(val *Timestamp) *NullableTimestamp { + return &NullableTimestamp{value: val, isSet: true} +} + +func (v NullableTimestamp) MarshalJSON() ([]byte, error) { + return json.Marshal(v.value) +} + +func (v *NullableTimestamp) UnmarshalJSON(src []byte) error { + v.isSet = true + return json.Unmarshal(src, &v.value) +} + + diff --git a/hindsight-clients/python/.openapi-generator/FILES b/hindsight-clients/python/.openapi-generator/FILES index 32cdb78a..8a0f7715 100644 --- a/hindsight-clients/python/.openapi-generator/FILES +++ b/hindsight-clients/python/.openapi-generator/FILES @@ -81,6 +81,7 @@ hindsight_client_api/models/retain_request.py hindsight_client_api/models/retain_response.py hindsight_client_api/models/source_facts_include_options.py hindsight_client_api/models/tag_item.py +hindsight_client_api/models/timestamp.py hindsight_client_api/models/token_usage.py hindsight_client_api/models/tool_calls_include_options.py hindsight_client_api/models/update_directive_request.py diff --git a/hindsight-clients/python/hindsight_client_api/__init__.py b/hindsight-clients/python/hindsight_client_api/__init__.py index 550c4632..a1b13070 100644 --- a/hindsight-clients/python/hindsight_client_api/__init__.py +++ b/hindsight-clients/python/hindsight_client_api/__init__.py @@ -106,6 +106,7 @@ from hindsight_client_api.models.retain_request import RetainRequest from hindsight_client_api.models.retain_response import RetainResponse from hindsight_client_api.models.source_facts_include_options import SourceFactsIncludeOptions from hindsight_client_api.models.tag_item import TagItem +from hindsight_client_api.models.timestamp import Timestamp from hindsight_client_api.models.token_usage import TokenUsage from hindsight_client_api.models.tool_calls_include_options import ToolCallsIncludeOptions from hindsight_client_api.models.update_directive_request import UpdateDirectiveRequest diff --git a/hindsight-clients/python/hindsight_client_api/models/__init__.py b/hindsight-clients/python/hindsight_client_api/models/__init__.py index fe51bc0e..0d66df53 100644 --- a/hindsight-clients/python/hindsight_client_api/models/__init__.py +++ b/hindsight-clients/python/hindsight_client_api/models/__init__.py @@ -81,6 +81,7 @@ from hindsight_client_api.models.retain_request import RetainRequest from hindsight_client_api.models.retain_response import RetainResponse from hindsight_client_api.models.source_facts_include_options import SourceFactsIncludeOptions from hindsight_client_api.models.tag_item import TagItem +from hindsight_client_api.models.timestamp import Timestamp from hindsight_client_api.models.token_usage import TokenUsage from hindsight_client_api.models.tool_calls_include_options import ToolCallsIncludeOptions from hindsight_client_api.models.update_directive_request import UpdateDirectiveRequest diff --git a/hindsight-clients/python/hindsight_client_api/models/memory_item.py b/hindsight-clients/python/hindsight_client_api/models/memory_item.py index 789bfe7a..09576069 100644 --- a/hindsight-clients/python/hindsight_client_api/models/memory_item.py +++ b/hindsight-clients/python/hindsight_client_api/models/memory_item.py @@ -17,11 +17,11 @@ import pprint import re # noqa: F401 import json -from datetime import datetime from pydantic import BaseModel, ConfigDict, StrictStr from typing import Any, ClassVar, Dict, List, Optional from hindsight_client_api.models.entity_input import EntityInput from hindsight_client_api.models.observation_scopes import ObservationScopes +from hindsight_client_api.models.timestamp import Timestamp from typing import Optional, Set from typing_extensions import Self @@ -30,7 +30,7 @@ class MemoryItem(BaseModel): Single memory item for retain. """ # noqa: E501 content: StrictStr - timestamp: Optional[datetime] = None + timestamp: Optional[Timestamp] = None context: Optional[StrictStr] = None metadata: Optional[Dict[str, StrictStr]] = None document_id: Optional[StrictStr] = None @@ -78,6 +78,9 @@ class MemoryItem(BaseModel): exclude=excluded_fields, exclude_none=True, ) + # override the default output from pydantic by calling `to_dict()` of timestamp + if self.timestamp: + _dict['timestamp'] = self.timestamp.to_dict() # override the default output from pydantic by calling `to_dict()` of each item in entities (list) _items = [] if self.entities: @@ -136,7 +139,7 @@ class MemoryItem(BaseModel): _obj = cls.model_validate({ "content": obj.get("content"), - "timestamp": obj.get("timestamp"), + "timestamp": Timestamp.from_dict(obj["timestamp"]) if obj.get("timestamp") is not None else None, "context": obj.get("context"), "metadata": obj.get("metadata"), "document_id": obj.get("document_id"), diff --git a/hindsight-clients/python/hindsight_client_api/models/timestamp.py b/hindsight-clients/python/hindsight_client_api/models/timestamp.py new file mode 100644 index 00000000..ed4fd12b --- /dev/null +++ b/hindsight-clients/python/hindsight_client_api/models/timestamp.py @@ -0,0 +1,145 @@ +# coding: utf-8 + +""" + Hindsight HTTP API + + HTTP API for Hindsight + + The version of the OpenAPI document: 0.4.14 + Generated by OpenAPI Generator (https://openapi-generator.tech) + + Do not edit the class manually. +""" # noqa: E501 + + +from __future__ import annotations +from inspect import getfullargspec +import json +import pprint +import re # noqa: F401 +from datetime import datetime +from pydantic import BaseModel, ConfigDict, Field, StrictStr, ValidationError, field_validator +from typing import Optional +from typing import Union, Any, List, Set, TYPE_CHECKING, Optional, Dict +from typing_extensions import Literal, Self +from pydantic import Field + +TIMESTAMP_ANY_OF_SCHEMAS = ["datetime", "str"] + +class Timestamp(BaseModel): + """ + When the content occurred. Accepts an ISO 8601 datetime string (e.g. '2024-01-15T10:30:00Z'), null/omitted (defaults to now), or the special string 'unset' to explicitly store without any timestamp (use this for timeless content such as fictional documents or static reference material). + """ + + # data type: datetime + anyof_schema_1_validator: Optional[datetime] = None + # data type: str + anyof_schema_2_validator: Optional[StrictStr] = None + if TYPE_CHECKING: + actual_instance: Optional[Union[datetime, str]] = None + else: + actual_instance: Any = None + any_of_schemas: Set[str] = { "datetime", "str" } + + model_config = { + "validate_assignment": True, + "protected_namespaces": (), + } + + def __init__(self, *args, **kwargs) -> None: + if args: + if len(args) > 1: + raise ValueError("If a position argument is used, only 1 is allowed to set `actual_instance`") + if kwargs: + raise ValueError("If a position argument is used, keyword arguments cannot be used.") + super().__init__(actual_instance=args[0]) + else: + super().__init__(**kwargs) + + @field_validator('actual_instance') + def actual_instance_must_validate_anyof(cls, v): + if v is None: + return v + + instance = Timestamp.model_construct() + error_messages = [] + # validate data type: datetime + try: + instance.anyof_schema_1_validator = v + return v + except (ValidationError, ValueError) as e: + error_messages.append(str(e)) + # validate data type: str + try: + instance.anyof_schema_2_validator = v + return v + except (ValidationError, ValueError) as e: + error_messages.append(str(e)) + if error_messages: + # no match + raise ValueError("No match found when setting the actual_instance in Timestamp with anyOf schemas: datetime, str. Details: " + ", ".join(error_messages)) + else: + return v + + @classmethod + def from_dict(cls, obj: Dict[str, Any]) -> Self: + return cls.from_json(json.dumps(obj)) + + @classmethod + def from_json(cls, json_str: str) -> Self: + """Returns the object represented by the json string""" + instance = cls.model_construct() + if json_str is None: + return instance + + error_messages = [] + # deserialize data into datetime + try: + # validation + instance.anyof_schema_1_validator = json.loads(json_str) + # assign value to actual_instance + instance.actual_instance = instance.anyof_schema_1_validator + return instance + except (ValidationError, ValueError) as e: + error_messages.append(str(e)) + # deserialize data into str + try: + # validation + instance.anyof_schema_2_validator = json.loads(json_str) + # assign value to actual_instance + instance.actual_instance = instance.anyof_schema_2_validator + return instance + except (ValidationError, ValueError) as e: + error_messages.append(str(e)) + + if error_messages: + # no match + raise ValueError("No match found when deserializing the JSON string into Timestamp with anyOf schemas: datetime, str. Details: " + ", ".join(error_messages)) + else: + return instance + + def to_json(self) -> str: + """Returns the JSON representation of the actual instance""" + if self.actual_instance is None: + return "null" + + if hasattr(self.actual_instance, "to_json") and callable(self.actual_instance.to_json): + return self.actual_instance.to_json() + else: + return json.dumps(self.actual_instance) + + def to_dict(self) -> Optional[Union[Dict[str, Any], datetime, str]]: + """Returns the dict representation of the actual instance""" + if self.actual_instance is None: + return None + + if hasattr(self.actual_instance, "to_dict") and callable(self.actual_instance.to_dict): + return self.actual_instance.to_dict() + else: + return self.actual_instance + + def to_str(self) -> str: + """Returns the string representation of the actual instance""" + return pprint.pformat(self.model_dump()) + + diff --git a/hindsight-clients/typescript/generated/types.gen.ts b/hindsight-clients/typescript/generated/types.gen.ts index 41a4c989..5cfb72e8 100644 --- a/hindsight-clients/typescript/generated/types.gen.ts +++ b/hindsight-clients/typescript/generated/types.gen.ts @@ -1165,6 +1165,8 @@ export type MemoryItem = { content: string; /** * Timestamp + * + * When the content occurred. Accepts an ISO 8601 datetime string (e.g. '2024-01-15T10:30:00Z'), null/omitted (defaults to now), or the special string 'unset' to explicitly store without any timestamp (use this for timeless content such as fictional documents or static reference material). */ timestamp?: string | null; /** diff --git a/hindsight-docs/docs/developer/api/retain.mdx b/hindsight-docs/docs/developer/api/retain.mdx index 8f1af581..bd0df1e9 100644 --- a/hindsight-docs/docs/developer/api/retain.mdx +++ b/hindsight-docs/docs/developer/api/retain.mdx @@ -66,9 +66,15 @@ The raw text to store. This is the only required field. Hindsight chunks the con ### timestamp -When the event described in the content actually occurred. Accepts any ISO 8601 string (e.g., `"2024-01-15T10:30:00Z"`). If omitted, defaults to the current time at ingestion. +When the event described in the content actually occurred. Three forms are accepted: -The timestamp is injected verbatim into the LLM fact-extraction prompt so the model can resolve relative temporal references in the content — for example, if the content says "last Monday", the model uses the provided timestamp as the anchor to pin down the actual date. It also enables temporal recall queries like "What happened last spring?" to work correctly. +| Value | Behaviour | +|-------|-----------| +| Omitted / `null` | Defaults to the current time at ingestion. | +| ISO 8601 string (e.g. `"2024-01-15T10:30:00Z"`) | Uses the provided datetime. | +| `"unset"` | Stores the content **without any timestamp**. Use this for timeless material such as reference documents, books, or fictional content where no real event time exists. | + +The timestamp is injected into the LLM fact-extraction prompt so the model can resolve relative temporal references in the content — for example, if the content says "last Monday", the model uses the provided timestamp as the anchor to pin down the actual date. When `"unset"` is passed the prompt shows `Event Date: Unknown`, allowing the model to correctly return `N/A` for the `when` field of every extracted fact. Providing a real timestamp also enables temporal recall queries like "What happened last spring?" to work correctly. ### context diff --git a/hindsight-docs/static/openapi.json b/hindsight-docs/static/openapi.json index 94687d26..32803c75 100644 --- a/hindsight-docs/static/openapi.json +++ b/hindsight-docs/static/openapi.json @@ -5237,11 +5237,15 @@ "type": "string", "format": "date-time" }, + { + "type": "string" + }, { "type": "null" } ], - "title": "Timestamp" + "title": "Timestamp", + "description": "When the content occurred. Accepts an ISO 8601 datetime string (e.g. '2024-01-15T10:30:00Z'), null/omitted (defaults to now), or the special string 'unset' to explicitly store without any timestamp (use this for timeless content such as fictional documents or static reference material)." }, "context": { "anyOf": [