From 67c1a4295fe75710ae7aeb0ab87f6afa1149c411 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=B2=20Boschi?= Date: Thu, 8 Jan 2026 11:22:10 +0100 Subject: [PATCH] fix: ui shows only 1000 memories (#121) * fix: ui shows only 1000 memories * fix: ui shows only 1000 memories --- hindsight-api/hindsight_api/api/http.py | 11 +++-- .../hindsight_api/engine/interface.py | 4 +- .../hindsight_api/engine/memory_engine.py | 22 ++++++++-- .../tests/test_causal_relationships.py | 10 ++--- .../tests/test_fact_extraction_quality.py | 42 +++++++++---------- .../hindsight_client_api/api/memory_api.py | 23 ++++++++-- .../models/graph_data_response.py | 6 ++- .../typescript/generated/sdk.gen.ts | 2 +- .../typescript/generated/types.gen.ts | 8 ++++ .../src/app/api/graph/route.ts | 3 ++ .../src/components/data-view.tsx | 28 +++++++++++-- hindsight-control-plane/src/lib/api.ts | 3 +- hindsight-docs/static/openapi.json | 20 ++++++++- 13 files changed, 136 insertions(+), 46 deletions(-) diff --git a/hindsight-api/hindsight_api/api/http.py b/hindsight-api/hindsight_api/api/http.py index 71cf122d..56bd6fd4 100644 --- a/hindsight-api/hindsight_api/api/http.py +++ b/hindsight-api/hindsight_api/api/http.py @@ -647,6 +647,7 @@ class GraphDataResponse(BaseModel): } ], "total_units": 2, + "limit": 1000, } } ) @@ -655,6 +656,7 @@ class GraphDataResponse(BaseModel): edges: list[dict[str, Any]] table_rows: list[dict[str, Any]] total_units: int + limit: int class ListMemoryUnitsResponse(BaseModel): @@ -1066,16 +1068,19 @@ def _register_routes(app: FastAPI): "/v1/default/banks/{bank_id}/graph", response_model=GraphDataResponse, summary="Get memory graph data", - description="Retrieve graph data for visualization, optionally filtered by type (world/experience/opinion). Limited to 1000 most recent items.", + description="Retrieve graph data for visualization, optionally filtered by type (world/experience/opinion).", operation_id="get_graph", tags=["Memory"], ) async def api_graph( - bank_id: str, type: str | None = None, request_context: RequestContext = Depends(get_request_context) + bank_id: str, + type: str | None = None, + limit: int = 1000, + request_context: RequestContext = Depends(get_request_context), ): """Get graph data from database, filtered by bank_id and optionally by type.""" try: - data = await app.state.memory.get_graph_data(bank_id, type, request_context=request_context) + data = await app.state.memory.get_graph_data(bank_id, type, limit=limit, request_context=request_context) return data except (AuthenticationError, HTTPException): raise diff --git a/hindsight-api/hindsight_api/engine/interface.py b/hindsight-api/hindsight_api/engine/interface.py index a861b5b5..2f469dea 100644 --- a/hindsight-api/hindsight_api/engine/interface.py +++ b/hindsight-api/hindsight_api/engine/interface.py @@ -289,6 +289,7 @@ class MemoryEngineInterface(ABC): bank_id: str, *, fact_type: str | None = None, + limit: int = 1000, request_context: "RequestContext", ) -> dict[str, Any]: """ @@ -297,10 +298,11 @@ class MemoryEngineInterface(ABC): Args: bank_id: The memory bank ID. fact_type: Filter by fact type. + limit: Maximum number of items to return (default: 1000). request_context: Request context for authentication. Returns: - Dict with nodes, edges, table_rows, total_units. + Dict with nodes, edges, table_rows, total_units, limit. """ ... diff --git a/hindsight-api/hindsight_api/engine/memory_engine.py b/hindsight-api/hindsight_api/engine/memory_engine.py index a0c43fd7..67e664df 100644 --- a/hindsight-api/hindsight_api/engine/memory_engine.py +++ b/hindsight-api/hindsight_api/engine/memory_engine.py @@ -2264,6 +2264,7 @@ class MemoryEngine(MemoryEngineInterface): bank_id: str | None = None, fact_type: str | None = None, *, + limit: int = 1000, request_context: "RequestContext", ): """ @@ -2272,10 +2273,11 @@ class MemoryEngine(MemoryEngineInterface): Args: bank_id: Filter by bank ID fact_type: Filter by fact type (world, experience, opinion) + limit: Maximum number of items to return (default: 1000) request_context: Request context for authentication. Returns: - Dict with nodes, edges, and table_rows + Dict with nodes, edges, table_rows, total_units, and limit """ await self._authenticate_tenant(request_context) pool = await self._get_pool() @@ -2297,15 +2299,29 @@ class MemoryEngine(MemoryEngineInterface): where_clause = "WHERE " + " AND ".join(query_conditions) if query_conditions else "" + # Get total count first + total_count_result = await conn.fetchrow( + f""" + SELECT COUNT(*) as total + FROM {fq_table("memory_units")} + {where_clause} + """, + *query_params, + ) + total_count = total_count_result["total"] if total_count_result else 0 + + # Get units with limit + param_count += 1 units = await conn.fetch( f""" SELECT id, text, event_date, context, occurred_start, occurred_end, mentioned_at, document_id, chunk_id, fact_type FROM {fq_table("memory_units")} {where_clause} ORDER BY mentioned_at DESC NULLS LAST, event_date DESC - LIMIT 1000 + LIMIT ${param_count} """, *query_params, + limit, ) # Get links, filtering to only include links between units of the selected agent @@ -2442,7 +2458,7 @@ class MemoryEngine(MemoryEngineInterface): } ) - return {"nodes": nodes, "edges": edges, "table_rows": table_rows, "total_units": len(units)} + return {"nodes": nodes, "edges": edges, "table_rows": table_rows, "total_units": total_count, "limit": limit} async def list_memory_units( self, diff --git a/hindsight-api/tests/test_causal_relationships.py b/hindsight-api/tests/test_causal_relationships.py index d8c64ece..6a8f61d3 100644 --- a/hindsight-api/tests/test_causal_relationships.py +++ b/hindsight-api/tests/test_causal_relationships.py @@ -36,7 +36,7 @@ After searching for weeks, I finally found a cheaper apartment in Brooklyn. context = "Personal story about housing change" llm_config = LLMConfig.for_memory() - facts, _ = await extract_facts_from_text( + facts, _, _ = await extract_facts_from_text( text=text, event_date=datetime(2024, 3, 15), context=context, @@ -105,7 +105,7 @@ The renovation took three months and cost $15,000. context = "Home repair story" llm_config = LLMConfig.for_memory() - facts, _ = await extract_facts_from_text( + facts, _, _ = await extract_facts_from_text( text=text, event_date=datetime(2024, 6, 1), context=context, @@ -140,7 +140,7 @@ Machine learning fascinated me so much that I changed my career to data science. context = "Career change story" llm_config = LLMConfig.for_memory() - facts, _ = await extract_facts_from_text( + facts, _, _ = await extract_facts_from_text( text=text, event_date=datetime(2024, 1, 1), context=context, @@ -172,7 +172,7 @@ The new role enabled me to lead a team of engineers. context = "Work promotion story" llm_config = LLMConfig.for_memory() - facts, _ = await extract_facts_from_text( + facts, _, _ = await extract_facts_from_text( text=text, event_date=datetime(2024, 2, 15), context=context, @@ -205,7 +205,7 @@ Reduced spending somewhat affected local businesses. context = "Economic impact story" llm_config = LLMConfig.for_memory() - facts, _ = await extract_facts_from_text( + facts, _, _ = await extract_facts_from_text( text=text, event_date=datetime(2024, 4, 1), context=context, diff --git a/hindsight-api/tests/test_fact_extraction_quality.py b/hindsight-api/tests/test_fact_extraction_quality.py index eebdc533..66da2803 100644 --- a/hindsight-api/tests/test_fact_extraction_quality.py +++ b/hindsight-api/tests/test_fact_extraction_quality.py @@ -43,7 +43,7 @@ Marcus felt anxious about the upcoming interview. context = "Personal journal entry" llm_config = LLMConfig.for_memory() - facts, _ = await extract_facts_from_text( + facts, _, _ = await extract_facts_from_text( text=text, event_date=datetime(2024, 11, 13), context=context, @@ -75,7 +75,7 @@ The music was so loud I could barely hear myself think. context = "Personal experience" llm_config = LLMConfig.for_memory() - facts, _ = await extract_facts_from_text( + facts, _, _ = await extract_facts_from_text( text=text, event_date=datetime(2024, 11, 13), context=context, @@ -108,7 +108,7 @@ Maybe we should reconsider the timeline. context = "Team discussion" llm_config = LLMConfig.for_memory() - facts, _ = await extract_facts_from_text( + facts, _, _ = await extract_facts_from_text( text=text, event_date=datetime(2024, 11, 13), context=context, @@ -141,7 +141,7 @@ I'm unable to attend the conference due to scheduling conflicts. context = "Personal profile discussion" llm_config = LLMConfig.for_memory() - facts, _ = await extract_facts_from_text( + facts, _, _ = await extract_facts_from_text( text=text, event_date=datetime(2024, 11, 13), context=context, @@ -173,7 +173,7 @@ Unlike last year, we're ahead of schedule. context = "Project review" llm_config = LLMConfig.for_memory() - facts, _ = await extract_facts_from_text( + facts, _, _ = await extract_facts_from_text( text=text, event_date=datetime(2024, 11, 13), context=context, @@ -206,7 +206,7 @@ She's enthusiastic about the opportunity. context = "Team meeting" llm_config = LLMConfig.for_memory() - facts, _ = await extract_facts_from_text( + facts, _, _ = await extract_facts_from_text( text=text, event_date=datetime(2024, 11, 13), context=context, @@ -239,7 +239,7 @@ I'm planning to switch careers because I'm not fulfilled in my current role. context = "Personal goals discussion" llm_config = LLMConfig.for_memory() - facts, _ = await extract_facts_from_text( + facts, _, _ = await extract_facts_from_text( text=text, event_date=datetime(2024, 11, 13), context=context, @@ -276,7 +276,7 @@ Family is the most important thing to her. context = "Personal values discussion" llm_config = LLMConfig.for_memory() - facts, _ = await extract_facts_from_text( + facts, _, _ = await extract_facts_from_text( text=text, event_date=datetime(2024, 11, 13), context=context, @@ -310,7 +310,7 @@ I prefer presenting in person rather than virtually because I can read the room event_date = datetime(2024, 11, 13) - facts, _ = await extract_facts_from_text( + facts, _, _ = await extract_facts_from_text( text=text, event_date=event_date, context=context, @@ -366,7 +366,7 @@ I'm planning to visit Tokyo next month. event_date = datetime(2024, 11, 13) - facts, _ = await extract_facts_from_text( + facts, _, _ = await extract_facts_from_text( text=text, event_date=event_date, context=context, @@ -419,7 +419,7 @@ with a concert surrounded by music, joy and the warm summer breeze. for attempt in range(max_retries): try: - facts, _ = await extract_facts_from_text( + facts, _, _ = await extract_facts_from_text( text=text, event_date=event_date, context=context, @@ -488,7 +488,7 @@ Yesterday I went for a morning jog for the first time in a nearby park. event_date = datetime(2024, 11, 13) - facts, _ = await extract_facts_from_text( + facts, _, _ = await extract_facts_from_text( text=text, event_date=event_date, context=context, @@ -537,7 +537,7 @@ Yesterday I went for a morning jog for the first time in a nearby park. This morning I had coffee with Alice. """ - facts, _ = await extract_facts_from_text( + facts, _, _ = await extract_facts_from_text( text=text, event_date=reference_date, llm_config=llm_config, @@ -567,7 +567,7 @@ Yesterday I went for a morning jog for the first time in a nearby park. text = "Alice works at Google. She loves Python programming." - facts, _ = await extract_facts_from_text( + facts, _, _ = await extract_facts_from_text( text=text, event_date=reference_date, llm_config=llm_config, @@ -594,7 +594,7 @@ Yesterday I went for a morning jog for the first time in a nearby park. Bob will start his vacation on April 1st. """ - facts, _ = await extract_facts_from_text( + facts, _, _ = await extract_facts_from_text( text=text, event_date=reference_date, llm_config=llm_config, @@ -645,7 +645,7 @@ great time! Every time I see it, I can't help but smile. event_date = datetime(2023, 2, 23) - facts, _ = await extract_facts_from_text( + facts, _, _ = await extract_facts_from_text( text=text, event_date=event_date, context=context, @@ -695,7 +695,7 @@ I've learned so much from it. context = "Personal update" llm_config = LLMConfig.for_memory() - facts, _ = await extract_facts_from_text( + facts, _, _ = await extract_facts_from_text( text=text, event_date=datetime(2024, 11, 13), context=context, @@ -758,7 +758,7 @@ Jamie: Congratulations! I'd love to read it. llm_config = LLMConfig.for_memory() - facts, _ = await extract_facts_from_text( + facts, _, _ = await extract_facts_from_text( text=transcript, event_date=datetime(2024, 11, 13), llm_config=llm_config, @@ -803,7 +803,7 @@ We presented our findings to the team yesterday. llm_config = LLMConfig.for_memory() - facts, _ = await extract_facts_from_text( + facts, _, _ = await extract_facts_from_text( text=text, event_date=datetime(2024, 11, 13), llm_config=llm_config, @@ -838,7 +838,7 @@ Jamie: [teasing] We'll see who's right, my Niners pick is solid. llm_config = LLMConfig.for_memory() - facts, _ = await extract_facts_from_text( + facts, _, _ = await extract_facts_from_text( text=transcript, event_date=datetime(2024, 11, 14), context=context, @@ -897,7 +897,7 @@ so the algorithm learns to box out. See you next week! llm_config = LLMConfig.for_memory() - facts, _ = await extract_facts_from_text( + facts, _, _ = await extract_facts_from_text( text=transcript, event_date=datetime(2024, 11, 13), llm_config=llm_config, diff --git a/hindsight-clients/python/hindsight_client_api/api/memory_api.py b/hindsight-clients/python/hindsight_client_api/api/memory_api.py index 618d721e..f067094e 100644 --- a/hindsight-clients/python/hindsight_client_api/api/memory_api.py +++ b/hindsight-clients/python/hindsight_client_api/api/memory_api.py @@ -347,6 +347,7 @@ class MemoryApi: self, bank_id: StrictStr, type: Optional[StrictStr] = None, + limit: Optional[StrictInt] = None, authorization: Optional[StrictStr] = None, _request_timeout: Union[ None, @@ -363,12 +364,14 @@ class MemoryApi: ) -> GraphDataResponse: """Get memory graph data - Retrieve graph data for visualization, optionally filtered by type (world/experience/opinion). Limited to 1000 most recent items. + Retrieve graph data for visualization, optionally filtered by type (world/experience/opinion). :param bank_id: (required) :type bank_id: str :param type: :type type: str + :param limit: + :type limit: int :param authorization: :type authorization: str :param _request_timeout: timeout setting for this request. If one @@ -396,6 +399,7 @@ class MemoryApi: _param = self._get_graph_serialize( bank_id=bank_id, type=type, + limit=limit, authorization=authorization, _request_auth=_request_auth, _content_type=_content_type, @@ -423,6 +427,7 @@ class MemoryApi: self, bank_id: StrictStr, type: Optional[StrictStr] = None, + limit: Optional[StrictInt] = None, authorization: Optional[StrictStr] = None, _request_timeout: Union[ None, @@ -439,12 +444,14 @@ class MemoryApi: ) -> ApiResponse[GraphDataResponse]: """Get memory graph data - Retrieve graph data for visualization, optionally filtered by type (world/experience/opinion). Limited to 1000 most recent items. + Retrieve graph data for visualization, optionally filtered by type (world/experience/opinion). :param bank_id: (required) :type bank_id: str :param type: :type type: str + :param limit: + :type limit: int :param authorization: :type authorization: str :param _request_timeout: timeout setting for this request. If one @@ -472,6 +479,7 @@ class MemoryApi: _param = self._get_graph_serialize( bank_id=bank_id, type=type, + limit=limit, authorization=authorization, _request_auth=_request_auth, _content_type=_content_type, @@ -499,6 +507,7 @@ class MemoryApi: self, bank_id: StrictStr, type: Optional[StrictStr] = None, + limit: Optional[StrictInt] = None, authorization: Optional[StrictStr] = None, _request_timeout: Union[ None, @@ -515,12 +524,14 @@ class MemoryApi: ) -> RESTResponseType: """Get memory graph data - Retrieve graph data for visualization, optionally filtered by type (world/experience/opinion). Limited to 1000 most recent items. + Retrieve graph data for visualization, optionally filtered by type (world/experience/opinion). :param bank_id: (required) :type bank_id: str :param type: :type type: str + :param limit: + :type limit: int :param authorization: :type authorization: str :param _request_timeout: timeout setting for this request. If one @@ -548,6 +559,7 @@ class MemoryApi: _param = self._get_graph_serialize( bank_id=bank_id, type=type, + limit=limit, authorization=authorization, _request_auth=_request_auth, _content_type=_content_type, @@ -570,6 +582,7 @@ class MemoryApi: self, bank_id, type, + limit, authorization, _request_auth, _content_type, @@ -599,6 +612,10 @@ class MemoryApi: _query_params.append(('type', type)) + if limit is not None: + + _query_params.append(('limit', limit)) + # process the header parameters if authorization is not None: _header_params['authorization'] = authorization diff --git a/hindsight-clients/python/hindsight_client_api/models/graph_data_response.py b/hindsight-clients/python/hindsight_client_api/models/graph_data_response.py index 60d1e229..e7966ce1 100644 --- a/hindsight-clients/python/hindsight_client_api/models/graph_data_response.py +++ b/hindsight-clients/python/hindsight_client_api/models/graph_data_response.py @@ -30,7 +30,8 @@ class GraphDataResponse(BaseModel): edges: List[Dict[str, Any]] table_rows: List[Dict[str, Any]] total_units: StrictInt - __properties: ClassVar[List[str]] = ["nodes", "edges", "table_rows", "total_units"] + limit: StrictInt + __properties: ClassVar[List[str]] = ["nodes", "edges", "table_rows", "total_units", "limit"] model_config = ConfigDict( populate_by_name=True, @@ -86,7 +87,8 @@ class GraphDataResponse(BaseModel): "nodes": obj.get("nodes"), "edges": obj.get("edges"), "table_rows": obj.get("table_rows"), - "total_units": obj.get("total_units") + "total_units": obj.get("total_units"), + "limit": obj.get("limit") }) return _obj diff --git a/hindsight-clients/typescript/generated/sdk.gen.ts b/hindsight-clients/typescript/generated/sdk.gen.ts index 86cea5dd..18697d27 100644 --- a/hindsight-clients/typescript/generated/sdk.gen.ts +++ b/hindsight-clients/typescript/generated/sdk.gen.ts @@ -123,7 +123,7 @@ export const metricsEndpointMetricsGet = ( /** * Get memory graph data * - * Retrieve graph data for visualization, optionally filtered by type (world/experience/opinion). Limited to 1000 most recent items. + * Retrieve graph data for visualization, optionally filtered by type (world/experience/opinion). */ export const getGraph = ( options: Options, diff --git a/hindsight-clients/typescript/generated/types.gen.ts b/hindsight-clients/typescript/generated/types.gen.ts index 30f160cc..7db1a485 100644 --- a/hindsight-clients/typescript/generated/types.gen.ts +++ b/hindsight-clients/typescript/generated/types.gen.ts @@ -570,6 +570,10 @@ export type GraphDataResponse = { * Total Units */ total_units: number; + /** + * Limit + */ + limit: number; }; /** @@ -1123,6 +1127,10 @@ export type GetGraphData = { * Type */ type?: string | null; + /** + * Limit + */ + limit?: number; }; url: "/v1/default/banks/{bank_id}/graph"; }; diff --git a/hindsight-control-plane/src/app/api/graph/route.ts b/hindsight-control-plane/src/app/api/graph/route.ts index b5c20988..077278d4 100644 --- a/hindsight-control-plane/src/app/api/graph/route.ts +++ b/hindsight-control-plane/src/app/api/graph/route.ts @@ -12,12 +12,15 @@ export async function GET(request: NextRequest) { // Get optional query parameters const type = searchParams.get("type") || searchParams.get("fact_type") || undefined; + const limitParam = searchParams.get("limit"); + const limit = limitParam ? parseInt(limitParam, 10) : undefined; const response = await sdk.getGraph({ client: lowLevelClient, path: { bank_id: bankId }, query: { type: type, + limit: limit, }, }); diff --git a/hindsight-control-plane/src/components/data-view.tsx b/hindsight-control-plane/src/components/data-view.tsx index 5cdfb3a5..06c94194 100644 --- a/hindsight-control-plane/src/components/data-view.tsx +++ b/hindsight-control-plane/src/components/data-view.tsx @@ -52,6 +52,9 @@ export function DataView({ factType }: DataViewProps) { const [selectedTableMemory, setSelectedTableMemory] = useState(null); const itemsPerPage = 100; + // Fetch limit state - how many memories to load from the API + const [fetchLimit, setFetchLimit] = useState(1000); + // Graph controls state const [showLabels, setShowLabels] = useState(true); const [maxNodes, setMaxNodes] = useState(undefined); @@ -93,7 +96,7 @@ export function DataView({ factType }: DataViewProps) { } }; - const loadData = async () => { + const loadData = async (limit?: number) => { if (!currentBank) return; setLoading(true); @@ -101,6 +104,7 @@ export function DataView({ factType }: DataViewProps) { const graphData: any = await client.getGraph({ bank_id: currentBank, type: factType, + limit: limit ?? fetchLimit, }); setData(graphData); } catch (error) { @@ -265,9 +269,25 @@ export function DataView({ factType }: DataViewProps) {
- {searchQuery - ? `${filteredTableRows.length} of ${data.total_units} memories` - : `${data.total_units} total memories`} + {searchQuery ? ( + `${filteredTableRows.length} of ${data.table_rows?.length ?? 0} loaded memories` + ) : data.table_rows?.length < data.total_units ? ( + + Showing {data.table_rows?.length ?? 0} of {data.total_units} total memories + + + ) : ( + `${data.total_units} total memories` + )}