From 26bf5714cdede6b83544bab26b6b87d1e6be7de9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=B2=20Boschi?= Date: Mon, 12 Jan 2026 18:50:53 +0100 Subject: [PATCH] fix: entities list only show 100 entities (#142) * fix: entities list only show 100 entities * fix: update Rust CLI for entities pagination API changes --- hindsight-api/hindsight_api/api/http.py | 24 +++-- .../hindsight_api/engine/interface.py | 8 +- .../hindsight_api/engine/memory_engine.py | 30 +++++-- .../tests/test_http_api_integration.py | 25 +++++- hindsight-cli/src/api.rs | 4 +- hindsight-cli/src/commands/entity.rs | 2 +- hindsight-cli/src/commands/explore.rs | 2 +- .../hindsight_client_api/api/entities_api.py | 23 ++++- .../models/entity_list_response.py | 12 ++- .../python/tests/test_main_operations.py | 32 +++++++ .../typescript/generated/sdk.gen.ts | 2 +- .../typescript/generated/types.gen.ts | 18 ++++ .../src/app/api/entities/route.ts | 3 +- .../src/components/entities-view.tsx | 88 +++++++++++++++++-- hindsight-control-plane/src/lib/api.ts | 10 ++- hindsight-docs/static/openapi.json | 36 +++++++- 16 files changed, 280 insertions(+), 39 deletions(-) diff --git a/hindsight-api/hindsight_api/api/http.py b/hindsight-api/hindsight_api/api/http.py index 898ef322..afdb980e 100644 --- a/hindsight-api/hindsight_api/api/http.py +++ b/hindsight-api/hindsight_api/api/http.py @@ -188,12 +188,18 @@ class EntityListResponse(BaseModel): "first_seen": "2024-01-15T10:30:00Z", "last_seen": "2024-02-01T14:00:00Z", } - ] + ], + "total": 150, + "limit": 100, + "offset": 0, } } ) items: list[EntityListItem] + total: int + limit: int + offset: int class EntityDetailResponse(BaseModel): @@ -1516,19 +1522,27 @@ def _register_routes(app: FastAPI): "/v1/default/banks/{bank_id}/entities", response_model=EntityListResponse, summary="List entities", - description="List all entities (people, organizations, etc.) known by the bank, ordered by mention count.", + description="List all entities (people, organizations, etc.) known by the bank, ordered by mention count. Supports pagination.", operation_id="list_entities", tags=["Entities"], ) async def api_list_entities( bank_id: str, limit: int = Query(default=100, description="Maximum number of entities to return"), + offset: int = Query(default=0, description="Offset for pagination"), request_context: RequestContext = Depends(get_request_context), ): - """List entities for a memory bank.""" + """List entities for a memory bank with pagination.""" try: - entities = await app.state.memory.list_entities(bank_id, limit=limit, request_context=request_context) - return EntityListResponse(items=[EntityListItem(**e) for e in entities]) + data = await app.state.memory.list_entities( + bank_id, limit=limit, offset=offset, request_context=request_context + ) + return EntityListResponse( + items=[EntityListItem(**e) for e in data["items"]], + total=data["total"], + limit=data["limit"], + offset=data["offset"], + ) except (AuthenticationError, HTTPException): raise except Exception as e: diff --git a/hindsight-api/hindsight_api/engine/interface.py b/hindsight-api/hindsight_api/engine/interface.py index 2f469dea..38ce8ce5 100644 --- a/hindsight-api/hindsight_api/engine/interface.py +++ b/hindsight-api/hindsight_api/engine/interface.py @@ -406,18 +406,20 @@ class MemoryEngineInterface(ABC): bank_id: str, *, limit: int = 100, + offset: int = 0, request_context: "RequestContext", - ) -> list[dict[str, Any]]: + ) -> dict[str, Any]: """ - List entities for a bank. + List entities for a bank with pagination. Args: bank_id: The memory bank ID. limit: Maximum results. + offset: Offset for pagination. request_context: Request context for authentication. Returns: - List of entity dicts. + Dict with items, total, limit, offset. """ ... diff --git a/hindsight-api/hindsight_api/engine/memory_engine.py b/hindsight-api/hindsight_api/engine/memory_engine.py index 27f11079..4319fa4c 100644 --- a/hindsight-api/hindsight_api/engine/memory_engine.py +++ b/hindsight-api/hindsight_api/engine/memory_engine.py @@ -3601,32 +3601,47 @@ Guidelines: bank_id: str, *, limit: int = 100, + offset: int = 0, request_context: "RequestContext", - ) -> list[dict[str, Any]]: + ) -> dict[str, Any]: """ - List all entities for a bank. + List all entities for a bank with pagination. Args: bank_id: bank IDentifier limit: Maximum number of entities to return + offset: Offset for pagination request_context: Request context for authentication. Returns: - List of entity dicts with id, canonical_name, mention_count, first_seen, last_seen + Dict with items, total, limit, offset """ await self._authenticate_tenant(request_context) pool = await self._get_pool() async with acquire_with_retry(pool) as conn: + # Get total count + total_row = await conn.fetchrow( + f""" + SELECT COUNT(*) as total + FROM {fq_table("entities")} + WHERE bank_id = $1 + """, + bank_id, + ) + total = total_row["total"] if total_row else 0 + + # Get paginated entities rows = await conn.fetch( f""" SELECT id, canonical_name, mention_count, first_seen, last_seen, metadata FROM {fq_table("entities")} WHERE bank_id = $1 ORDER BY mention_count DESC, last_seen DESC - LIMIT $2 + LIMIT $2 OFFSET $3 """, bank_id, limit, + offset, ) entities = [] @@ -3653,7 +3668,12 @@ Guidelines: "metadata": metadata, } ) - return entities + return { + "items": entities, + "total": total, + "limit": limit, + "offset": offset, + } async def get_entity_state( self, diff --git a/hindsight-api/tests/test_http_api_integration.py b/hindsight-api/tests/test_http_api_integration.py index 28037ed2..b9894aa6 100644 --- a/hindsight-api/tests/test_http_api_integration.py +++ b/hindsight-api/tests/test_http_api_integration.py @@ -250,11 +250,34 @@ async def test_full_api_workflow(api_client, test_bank_id): # 8. Test Entity Endpoints # ================================================================ - # List entities + # List entities with pagination response = await api_client.get(f"/v1/default/banks/{test_bank_id}/entities") assert response.status_code == 200 entities_data = response.json() assert "items" in entities_data + assert "total" in entities_data + assert "limit" in entities_data + assert "offset" in entities_data + assert entities_data["offset"] == 0 + assert entities_data["limit"] == 100 # default limit + + # Test pagination with custom limit and offset + response = await api_client.get(f"/v1/default/banks/{test_bank_id}/entities?limit=5&offset=0") + assert response.status_code == 200 + paginated_data = response.json() + assert paginated_data["limit"] == 5 + assert paginated_data["offset"] == 0 + assert len(paginated_data["items"]) <= 5 + + # Test offset + if entities_data["total"] > 1: + response = await api_client.get(f"/v1/default/banks/{test_bank_id}/entities?limit=1&offset=1") + assert response.status_code == 200 + offset_data = response.json() + assert offset_data["offset"] == 1 + # With offset=1, we should get different entity than first one (if there are multiple) + if len(offset_data["items"]) > 0 and len(entities_data["items"]) > 1: + assert offset_data["items"][0]["id"] != entities_data["items"][0]["id"] # Get specific entity if any exist if len(entities_data['items']) > 0: diff --git a/hindsight-cli/src/api.rs b/hindsight-cli/src/api.rs index e928778d..38efde87 100644 --- a/hindsight-cli/src/api.rs +++ b/hindsight-cli/src/api.rs @@ -241,9 +241,9 @@ impl ApiClient { }) } - pub fn list_entities(&self, bank_id: &str, limit: Option, _verbose: bool) -> Result { + pub fn list_entities(&self, bank_id: &str, limit: Option, offset: Option, _verbose: bool) -> Result { self.runtime.block_on(async { - let response = self.client.list_entities(bank_id, limit, None).await?; + let response = self.client.list_entities(bank_id, limit, offset, None).await?; Ok(response.into_inner()) }) } diff --git a/hindsight-cli/src/commands/entity.rs b/hindsight-cli/src/commands/entity.rs index 48f0ece7..3f8ebfb7 100644 --- a/hindsight-cli/src/commands/entity.rs +++ b/hindsight-cli/src/commands/entity.rs @@ -16,7 +16,7 @@ pub fn list( None }; - let response = client.list_entities(bank_id, Some(limit), verbose)?; + let response = client.list_entities(bank_id, Some(limit), None, verbose)?; if let Some(mut sp) = spinner { sp.finish(); diff --git a/hindsight-cli/src/commands/explore.rs b/hindsight-cli/src/commands/explore.rs index 283d4366..892686ca 100644 --- a/hindsight-cli/src/commands/explore.rs +++ b/hindsight-cli/src/commands/explore.rs @@ -283,7 +283,7 @@ impl App { } fn load_entities(&mut self, bank_id: &str) -> Result<()> { - let response = self.client.list_entities(bank_id, Some(100), false)?; + let response = self.client.list_entities(bank_id, Some(100), None, false)?; self.entities = response.items; if !self.entities.is_empty() && self.entities_state.selected().is_none() { diff --git a/hindsight-clients/python/hindsight_client_api/api/entities_api.py b/hindsight-clients/python/hindsight_client_api/api/entities_api.py index 9d710b56..a8085972 100644 --- a/hindsight-clients/python/hindsight_client_api/api/entities_api.py +++ b/hindsight-clients/python/hindsight_client_api/api/entities_api.py @@ -338,6 +338,7 @@ class EntitiesApi: self, bank_id: StrictStr, limit: Annotated[Optional[StrictInt], Field(description="Maximum number of entities to return")] = None, + offset: Annotated[Optional[StrictInt], Field(description="Offset for pagination")] = None, authorization: Optional[StrictStr] = None, _request_timeout: Union[ None, @@ -354,12 +355,14 @@ class EntitiesApi: ) -> EntityListResponse: """List entities - List all entities (people, organizations, etc.) known by the bank, ordered by mention count. + List all entities (people, organizations, etc.) known by the bank, ordered by mention count. Supports pagination. :param bank_id: (required) :type bank_id: str :param limit: Maximum number of entities to return :type limit: int + :param offset: Offset for pagination + :type offset: int :param authorization: :type authorization: str :param _request_timeout: timeout setting for this request. If one @@ -387,6 +390,7 @@ class EntitiesApi: _param = self._list_entities_serialize( bank_id=bank_id, limit=limit, + offset=offset, authorization=authorization, _request_auth=_request_auth, _content_type=_content_type, @@ -414,6 +418,7 @@ class EntitiesApi: self, bank_id: StrictStr, limit: Annotated[Optional[StrictInt], Field(description="Maximum number of entities to return")] = None, + offset: Annotated[Optional[StrictInt], Field(description="Offset for pagination")] = None, authorization: Optional[StrictStr] = None, _request_timeout: Union[ None, @@ -430,12 +435,14 @@ class EntitiesApi: ) -> ApiResponse[EntityListResponse]: """List entities - List all entities (people, organizations, etc.) known by the bank, ordered by mention count. + List all entities (people, organizations, etc.) known by the bank, ordered by mention count. Supports pagination. :param bank_id: (required) :type bank_id: str :param limit: Maximum number of entities to return :type limit: int + :param offset: Offset for pagination + :type offset: int :param authorization: :type authorization: str :param _request_timeout: timeout setting for this request. If one @@ -463,6 +470,7 @@ class EntitiesApi: _param = self._list_entities_serialize( bank_id=bank_id, limit=limit, + offset=offset, authorization=authorization, _request_auth=_request_auth, _content_type=_content_type, @@ -490,6 +498,7 @@ class EntitiesApi: self, bank_id: StrictStr, limit: Annotated[Optional[StrictInt], Field(description="Maximum number of entities to return")] = None, + offset: Annotated[Optional[StrictInt], Field(description="Offset for pagination")] = None, authorization: Optional[StrictStr] = None, _request_timeout: Union[ None, @@ -506,12 +515,14 @@ class EntitiesApi: ) -> RESTResponseType: """List entities - List all entities (people, organizations, etc.) known by the bank, ordered by mention count. + List all entities (people, organizations, etc.) known by the bank, ordered by mention count. Supports pagination. :param bank_id: (required) :type bank_id: str :param limit: Maximum number of entities to return :type limit: int + :param offset: Offset for pagination + :type offset: int :param authorization: :type authorization: str :param _request_timeout: timeout setting for this request. If one @@ -539,6 +550,7 @@ class EntitiesApi: _param = self._list_entities_serialize( bank_id=bank_id, limit=limit, + offset=offset, authorization=authorization, _request_auth=_request_auth, _content_type=_content_type, @@ -561,6 +573,7 @@ class EntitiesApi: self, bank_id, limit, + offset, authorization, _request_auth, _content_type, @@ -590,6 +603,10 @@ class EntitiesApi: _query_params.append(('limit', limit)) + if offset is not None: + + _query_params.append(('offset', offset)) + # process the header parameters if authorization is not None: _header_params['authorization'] = authorization diff --git a/hindsight-clients/python/hindsight_client_api/models/entity_list_response.py b/hindsight-clients/python/hindsight_client_api/models/entity_list_response.py index a638247b..1e24a942 100644 --- a/hindsight-clients/python/hindsight_client_api/models/entity_list_response.py +++ b/hindsight-clients/python/hindsight_client_api/models/entity_list_response.py @@ -17,7 +17,7 @@ import pprint import re # noqa: F401 import json -from pydantic import BaseModel, ConfigDict +from pydantic import BaseModel, ConfigDict, StrictInt from typing import Any, ClassVar, Dict, List from hindsight_client_api.models.entity_list_item import EntityListItem from typing import Optional, Set @@ -28,7 +28,10 @@ class EntityListResponse(BaseModel): Response model for entity list endpoint. """ # noqa: E501 items: List[EntityListItem] - __properties: ClassVar[List[str]] = ["items"] + total: StrictInt + limit: StrictInt + offset: StrictInt + __properties: ClassVar[List[str]] = ["items", "total", "limit", "offset"] model_config = ConfigDict( populate_by_name=True, @@ -88,7 +91,10 @@ class EntityListResponse(BaseModel): return cls.model_validate(obj) _obj = cls.model_validate({ - "items": [EntityListItem.from_dict(_item) for _item in obj["items"]] if obj.get("items") is not None else None + "items": [EntityListItem.from_dict(_item) for _item in obj["items"]] if obj.get("items") is not None else None, + "total": obj.get("total"), + "limit": obj.get("limit"), + "offset": obj.get("offset") }) return _obj diff --git a/hindsight-clients/python/tests/test_main_operations.py b/hindsight-clients/python/tests/test_main_operations.py index 56861b9f..0a97f2a7 100644 --- a/hindsight-clients/python/tests/test_main_operations.py +++ b/hindsight-clients/python/tests/test_main_operations.py @@ -449,6 +449,38 @@ class TestEntities: assert response is not None assert response.items is not None assert isinstance(response.items, list) + # Verify pagination fields + assert response.total is not None + assert response.limit is not None + assert response.offset is not None + assert response.offset == 0 + assert response.limit == 100 # default limit + + def test_list_entities_with_pagination(self, client, bank_id): + """Test listing entities with pagination parameters.""" + import asyncio + from hindsight_client_api import ApiClient, Configuration + from hindsight_client_api.api import EntitiesApi + + async def do_list_paginated(): + config = Configuration(host=HINDSIGHT_API_URL) + api_client = ApiClient(config) + api = EntitiesApi(api_client) + + # Test with custom limit + response = await api.list_entities(bank_id=bank_id, limit=5, offset=0) + assert response.limit == 5 + assert response.offset == 0 + assert len(response.items) <= 5 + + # Test with offset + response_offset = await api.list_entities(bank_id=bank_id, limit=1, offset=1) + assert response_offset.offset == 1 + assert response_offset.limit == 1 + + return response + + asyncio.get_event_loop().run_until_complete(do_list_paginated()) def test_get_entity(self, client, bank_id): """Test getting a specific entity.""" diff --git a/hindsight-clients/typescript/generated/sdk.gen.ts b/hindsight-clients/typescript/generated/sdk.gen.ts index 18697d27..68cfb145 100644 --- a/hindsight-clients/typescript/generated/sdk.gen.ts +++ b/hindsight-clients/typescript/generated/sdk.gen.ts @@ -236,7 +236,7 @@ export const getAgentStats = ( /** * List entities * - * List all entities (people, organizations, etc.) known by the bank, ordered by mention count. + * List all entities (people, organizations, etc.) known by the bank, ordered by mention count. Supports pagination. */ export const listEntities = ( options: Options, diff --git a/hindsight-clients/typescript/generated/types.gen.ts b/hindsight-clients/typescript/generated/types.gen.ts index d3a2d3b9..cc453acb 100644 --- a/hindsight-clients/typescript/generated/types.gen.ts +++ b/hindsight-clients/typescript/generated/types.gen.ts @@ -495,6 +495,18 @@ export type EntityListResponse = { * Items */ items: Array; + /** + * Total + */ + total: number; + /** + * Limit + */ + limit: number; + /** + * Offset + */ + offset: number; }; /** @@ -1376,6 +1388,12 @@ export type ListEntitiesData = { * Maximum number of entities to return */ limit?: number; + /** + * Offset + * + * Offset for pagination + */ + offset?: number; }; url: "/v1/default/banks/{bank_id}/entities"; }; diff --git a/hindsight-control-plane/src/app/api/entities/route.ts b/hindsight-control-plane/src/app/api/entities/route.ts index dc5c1336..a54628e4 100644 --- a/hindsight-control-plane/src/app/api/entities/route.ts +++ b/hindsight-control-plane/src/app/api/entities/route.ts @@ -11,11 +11,12 @@ export async function GET(request: NextRequest) { } const limit = searchParams.get("limit") ? Number(searchParams.get("limit")) : undefined; + const offset = searchParams.get("offset") ? Number(searchParams.get("offset")) : undefined; const response = await sdk.listEntities({ client: lowLevelClient, path: { bank_id: bankId }, - query: { limit }, + query: { limit, offset }, }); if (response.error) { diff --git a/hindsight-control-plane/src/components/entities-view.tsx b/hindsight-control-plane/src/components/entities-view.tsx index 72704d90..99e240e4 100644 --- a/hindsight-control-plane/src/components/entities-view.tsx +++ b/hindsight-control-plane/src/components/entities-view.tsx @@ -4,6 +4,7 @@ import { useState, useEffect } from "react"; import { client } from "@/lib/api"; import { useBank } from "@/lib/bank-context"; import { Button } from "@/components/ui/button"; +import { ChevronLeft, ChevronRight, ChevronsLeft, ChevronsRight } from "lucide-react"; import { Table, TableBody, @@ -29,6 +30,8 @@ interface EntityDetail extends Entity { }>; } +const ITEMS_PER_PAGE = 50; + export function EntitiesView() { const { currentBank } = useBank(); const [entities, setEntities] = useState([]); @@ -37,16 +40,26 @@ export function EntitiesView() { const [loadingDetail, setLoadingDetail] = useState(false); const [regenerating, setRegenerating] = useState(false); - const loadEntities = async () => { + // Pagination state + const [currentPage, setCurrentPage] = useState(1); + const [total, setTotal] = useState(0); + + const totalPages = Math.ceil(total / ITEMS_PER_PAGE); + const offset = (currentPage - 1) * ITEMS_PER_PAGE; + + const loadEntities = async (page: number = 1) => { if (!currentBank) return; setLoading(true); try { - const result: any = await client.listEntities({ + const pageOffset = (page - 1) * ITEMS_PER_PAGE; + const result = await client.listEntities({ bank_id: currentBank, - limit: 100, + limit: ITEMS_PER_PAGE, + offset: pageOffset, }); setEntities(result.items || []); + setTotal(result.total || 0); } catch (error) { console.error("Error loading entities:", error); alert("Error loading entities: " + (error as Error).message); @@ -86,9 +99,16 @@ export function EntitiesView() { } }; + // Handle page change + const handlePageChange = (newPage: number) => { + setCurrentPage(newPage); + loadEntities(newPage); + }; + useEffect(() => { if (currentBank) { - loadEntities(); + setCurrentPage(1); + loadEntities(1); setSelectedEntity(null); } }, [currentBank]); @@ -105,13 +125,15 @@ export function EntitiesView() { {loading ? (
-
⏳
+
...
Loading entities...
) : entities.length > 0 ? ( <> -
{entities.length} entities
+
+ {total} {total === 1 ? "entity" : "entities"} +
@@ -146,11 +168,61 @@ export function EntitiesView() {
+ + {/* Pagination Controls */} + {totalPages > 1 && ( +
+
+ {offset + 1}-{Math.min(offset + ITEMS_PER_PAGE, total)} of {total} +
+
+ + + + {currentPage} / {totalPages} + + + +
+
+ )} ) : (
-
👥
+
...
No entities found
Entities are extracted from facts when memories are added. @@ -178,7 +250,7 @@ export function EntitiesView() { onClick={() => setSelectedEntity(null)} className="h-8 w-8 p-0" > - × + x
diff --git a/hindsight-control-plane/src/lib/api.ts b/hindsight-control-plane/src/lib/api.ts index f26d680f..f90dae1b 100644 --- a/hindsight-control-plane/src/lib/api.ts +++ b/hindsight-control-plane/src/lib/api.ts @@ -127,11 +127,17 @@ export class ControlPlaneClient { /** * List entities */ - async listEntities(params: { bank_id: string; limit?: number }) { + async listEntities(params: { bank_id: string; limit?: number; offset?: number }) { const queryParams = new URLSearchParams(); queryParams.append("bank_id", params.bank_id); if (params.limit) queryParams.append("limit", params.limit.toString()); - return this.fetchApi(`/api/entities?${queryParams}`); + if (params.offset) queryParams.append("offset", params.offset.toString()); + return this.fetchApi<{ + items: any[]; + total: number; + limit: number; + offset: number; + }>(`/api/entities?${queryParams}`); } /** diff --git a/hindsight-docs/static/openapi.json b/hindsight-docs/static/openapi.json index d1cd6cbe..546523a2 100644 --- a/hindsight-docs/static/openapi.json +++ b/hindsight-docs/static/openapi.json @@ -502,7 +502,7 @@ "Entities" ], "summary": "List entities", - "description": "List all entities (people, organizations, etc.) known by the bank, ordered by mention count.", + "description": "List all entities (people, organizations, etc.) known by the bank, ordered by mention count. Supports pagination.", "operationId": "list_entities", "parameters": [ { @@ -526,6 +526,18 @@ }, "description": "Maximum number of entities to return" }, + { + "name": "offset", + "in": "query", + "required": false, + "schema": { + "type": "integer", + "description": "Offset for pagination", + "default": 0, + "title": "Offset" + }, + "description": "Offset for pagination" + }, { "name": "authorization", "in": "header", @@ -2423,11 +2435,26 @@ }, "type": "array", "title": "Items" + }, + "total": { + "type": "integer", + "title": "Total" + }, + "limit": { + "type": "integer", + "title": "Limit" + }, + "offset": { + "type": "integer", + "title": "Offset" } }, "type": "object", "required": [ - "items" + "items", + "total", + "limit", + "offset" ], "title": "EntityListResponse", "description": "Response model for entity list endpoint.", @@ -2440,7 +2467,10 @@ "last_seen": "2024-02-01T14:00:00Z", "mention_count": 15 } - ] + ], + "limit": 100, + "offset": 0, + "total": 150 } }, "EntityObservationResponse": {